sys/dev/mmc/mmc.c

   1 /*-
   2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
   3  *
   4  * Copyright (c) 2006 Bernd Walter.  All rights reserved.
   5  * Copyright (c) 2006 M. Warner Losh.  All rights reserved.
   6  * Copyright (c) 2017 Marius Strobl <marius@FreeBSD.org>
   7  *
   8  * Redistribution and use in source and binary forms, with or without
   9  * modification, are permitted provided that the following conditions
  10  * are met:
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in the
  15  *    documentation and/or other materials provided with the distribution.
  16  *
  17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
  18  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
  19  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
  20  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
  21  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  22  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  23  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  24  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  25  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  26  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  27  *
  28  * Portions of this software may have been developed with reference to
  29  * the SD Simplified Specification.  The following disclaimer may apply:
  30  *
  31  * The following conditions apply to the release of the simplified
  32  * specification ("Simplified Specification") by the SD Card Association and
  33  * the SD Group. The Simplified Specification is a subset of the complete SD
  34  * Specification which is owned by the SD Card Association and the SD
  35  * Group. This Simplified Specification is provided on a non-confidential
  36  * basis subject to the disclaimers below. Any implementation of the
  37  * Simplified Specification may require a license from the SD Card
  38  * Association, SD Group, SD-3C LLC or other third parties.
  39  *
  40  * Disclaimers:
  41  *
  42  * The information contained in the Simplified Specification is presented only
  43  * as a standard specification for SD Cards and SD Host/Ancillary products and
  44  * is provided "AS-IS" without any representations or warranties of any
  45  * kind. No responsibility is assumed by the SD Group, SD-3C LLC or the SD
  46  * Card Association for any damages, any infringements of patents or other
  47  * right of the SD Group, SD-3C LLC, the SD Card Association or any third
  48  * parties, which may result from its use. No license is granted by
  49  * implication, estoppel or otherwise under any patent or other rights of the
  50  * SD Group, SD-3C LLC, the SD Card Association or any third party. Nothing
  51  * herein shall be construed as an obligation by the SD Group, the SD-3C LLC
  52  * or the SD Card Association to disclose or distribute any technical
  53  * information, know-how or other confidential information to any third party.
  54  */
  55
  56 #include <sys/cdefs.h>
  57 __FBSDID("$FreeBSD$");
  58
  59 #include <sys/param.h>
  60 #include <sys/systm.h>
  61 #include <sys/kernel.h>
  62 #include <sys/malloc.h>
  63 #include <sys/lock.h>
  64 #include <sys/module.h>
  65 #include <sys/mutex.h>
  66 #include <sys/bus.h>
  67 #include <sys/endian.h>
  68 #include <sys/sysctl.h>
  69 #include <sys/time.h>
  70
  71 #include <dev/mmc/bridge.h>
  72 #include <dev/mmc/mmc_private.h>
  73 #include <dev/mmc/mmc_subr.h>
  74 #include <dev/mmc/mmcreg.h>
  75 #include <dev/mmc/mmcbrvar.h>
  76 #include <dev/mmc/mmcvar.h>
  77
  78 #include "mmcbr_if.h"
  79 #include "mmcbus_if.h"
  80
  81 CTASSERT(bus_timing_max <= sizeof(uint32_t) * NBBY);
  82
  83 /*
  84  * Per-card data
  85  */
  86 struct mmc_ivars {
  87         uint32_t raw_cid[4];    /* Raw bits of the CID */
  88         uint32_t raw_csd[4];    /* Raw bits of the CSD */
  89         uint32_t raw_scr[2];    /* Raw bits of the SCR */
  90         uint8_t raw_ext_csd[MMC_EXTCSD_SIZE]; /* Raw bits of the EXT_CSD */
  91         uint32_t raw_sd_status[16];     /* Raw bits of the SD_STATUS */
  92         uint16_t rca;
  93         u_char read_only;       /* True when the device is read-only */
  94         u_char high_cap;        /* High Capacity device (block addressed) */
  95         enum mmc_card_mode mode;
  96         enum mmc_bus_width bus_width;   /* Bus width to use */
  97         struct mmc_cid cid;     /* cid decoded */
  98         struct mmc_csd csd;     /* csd decoded */
  99         struct mmc_scr scr;     /* scr decoded */
 100         struct mmc_sd_status sd_status; /* SD_STATUS decoded */
 101         uint32_t sec_count;     /* Card capacity in 512byte blocks */
 102         uint32_t timings;       /* Mask of bus timings supported */
 103         uint32_t vccq_120;      /* Mask of bus timings at VCCQ of 1.2 V */
 104         uint32_t vccq_180;      /* Mask of bus timings at VCCQ of 1.8 V */
 105         uint32_t tran_speed;    /* Max speed in normal mode */
 106         uint32_t hs_tran_speed; /* Max speed in high speed mode */
 107         uint32_t erase_sector;  /* Card native erase sector size */
 108         uint32_t cmd6_time;     /* Generic switch timeout [us] */
 109         uint32_t quirks;        /* Quirks as per mmc_quirk->quirks */
 110         char card_id_string[64];/* Formatted CID info (serial, MFG, etc) */
 111         char card_sn_string[16];/* Formatted serial # for disk->d_ident */
 112 };
 113
 114 #define CMD_RETRIES     3
 115
 116 static const struct mmc_quirk mmc_quirks[] = {
 117         /*
 118          * For some SanDisk iNAND devices, the CMD38 argument needs to be
 119          * provided in EXT_CSD[113].
 120          */
 121         { 0x2, 0x100,                   "SEM02G", MMC_QUIRK_INAND_CMD38 },
 122         { 0x2, 0x100,                   "SEM04G", MMC_QUIRK_INAND_CMD38 },
 123         { 0x2, 0x100,                   "SEM08G", MMC_QUIRK_INAND_CMD38 },
 124         { 0x2, 0x100,                   "SEM16G", MMC_QUIRK_INAND_CMD38 },
 125         { 0x2, 0x100,                   "SEM32G", MMC_QUIRK_INAND_CMD38 },
 126
 127         /*
 128          * Disable TRIM for Kingston eMMCs where a firmware bug can lead to
 129          * unrecoverable data corruption.
 130          */
 131         { 0x70, MMC_QUIRK_OID_ANY,      "V10008", MMC_QUIRK_BROKEN_TRIM },
 132         { 0x70, MMC_QUIRK_OID_ANY,      "V10016", MMC_QUIRK_BROKEN_TRIM },
 133
 134         { 0x0, 0x0, NULL, 0x0 }
 135 };
 136
 137 static SYSCTL_NODE(_hw, OID_AUTO, mmc, CTLFLAG_RD, NULL, "mmc driver");
 138
 139 static int mmc_debug;
 140 SYSCTL_INT(_hw_mmc, OID_AUTO, debug, CTLFLAG_RWTUN, &mmc_debug, 0,
 141     "Debug level");
 142
 143 /* bus entry points */
 144 static int mmc_acquire_bus(device_t busdev, device_t dev);
 145 static int mmc_attach(device_t dev);
 146 static int mmc_child_location_str(device_t dev, device_t child, char *buf,
 147     size_t buflen);
 148 static int mmc_detach(device_t dev);
 149 static int mmc_probe(device_t dev);
 150 static int mmc_read_ivar(device_t bus, device_t child, int which,
 151     uintptr_t *result);
 152 static int mmc_release_bus(device_t busdev, device_t dev);
 153 static int mmc_resume(device_t dev);
 154 static void mmc_retune_pause(device_t busdev, device_t dev, bool retune);
 155 static void mmc_retune_unpause(device_t busdev, device_t dev);
 156 static int mmc_suspend(device_t dev);
 157 static int mmc_wait_for_request(device_t busdev, device_t dev,
 158     struct mmc_request *req);
 159 static int mmc_write_ivar(device_t bus, device_t child, int which,
 160     uintptr_t value);
 161
 162 #define MMC_LOCK(_sc)           mtx_lock(&(_sc)->sc_mtx)
 163 #define MMC_UNLOCK(_sc)         mtx_unlock(&(_sc)->sc_mtx)
 164 #define MMC_LOCK_INIT(_sc)                                              \
 165         mtx_init(&(_sc)->sc_mtx, device_get_nameunit((_sc)->dev),       \
 166             "mmc", MTX_DEF)
 167 #define MMC_LOCK_DESTROY(_sc)   mtx_destroy(&(_sc)->sc_mtx);
 168 #define MMC_ASSERT_LOCKED(_sc)  mtx_assert(&(_sc)->sc_mtx, MA_OWNED);
 169 #define MMC_ASSERT_UNLOCKED(_sc) mtx_assert(&(_sc)->sc_mtx, MA_NOTOWNED);
 170
 171 static int mmc_all_send_cid(struct mmc_softc *sc, uint32_t *rawcid);
 172 static void mmc_app_decode_scr(uint32_t *raw_scr, struct mmc_scr *scr);
 173 static void mmc_app_decode_sd_status(uint32_t *raw_sd_status,
 174     struct mmc_sd_status *sd_status);
 175 static int mmc_app_sd_status(struct mmc_softc *sc, uint16_t rca,
 176     uint32_t *rawsdstatus);
 177 static int mmc_app_send_scr(struct mmc_softc *sc, uint16_t rca,
 178     uint32_t *rawscr);
 179 static int mmc_calculate_clock(struct mmc_softc *sc);
 180 static void mmc_decode_cid_mmc(uint32_t *raw_cid, struct mmc_cid *cid,
 181     bool is_4_41p);
 182 static void mmc_decode_cid_sd(uint32_t *raw_cid, struct mmc_cid *cid);
 183 static void mmc_decode_csd_mmc(uint32_t *raw_csd, struct mmc_csd *csd);
 184 static int mmc_decode_csd_sd(uint32_t *raw_csd, struct mmc_csd *csd);
 185 static void mmc_delayed_attach(void *xsc);
 186 static int mmc_delete_cards(struct mmc_softc *sc, bool final);
 187 static void mmc_discover_cards(struct mmc_softc *sc);
 188 static void mmc_format_card_id_string(struct mmc_ivars *ivar);
 189 static void mmc_go_discovery(struct mmc_softc *sc);
 190 static uint32_t mmc_get_bits(uint32_t *bits, int bit_len, int start,
 191     int size);
 192 static int mmc_highest_voltage(uint32_t ocr);
 193 static bool mmc_host_timing(device_t dev, enum mmc_bus_timing timing);
 194 static void mmc_idle_cards(struct mmc_softc *sc);
 195 static void mmc_ms_delay(int ms);
 196 static void mmc_log_card(device_t dev, struct mmc_ivars *ivar, int newcard);
 197 static void mmc_power_down(struct mmc_softc *sc);
 198 static void mmc_power_up(struct mmc_softc *sc);
 199 static void mmc_rescan_cards(struct mmc_softc *sc);
 200 static int mmc_retune(device_t busdev, device_t dev, bool reset);
 201 static void mmc_scan(struct mmc_softc *sc);
 202 static int mmc_sd_switch(struct mmc_softc *sc, uint8_t mode, uint8_t grp,
 203     uint8_t value, uint8_t *res);
 204 static int mmc_select_card(struct mmc_softc *sc, uint16_t rca);
 205 static uint32_t mmc_select_vdd(struct mmc_softc *sc, uint32_t ocr);
 206 static int mmc_send_app_op_cond(struct mmc_softc *sc, uint32_t ocr,
 207     uint32_t *rocr);
 208 static int mmc_send_csd(struct mmc_softc *sc, uint16_t rca, uint32_t *rawcsd);
 209 static int mmc_send_if_cond(struct mmc_softc *sc, uint8_t vhs);
 210 static int mmc_send_op_cond(struct mmc_softc *sc, uint32_t ocr,
 211     uint32_t *rocr);
 212 static int mmc_send_relative_addr(struct mmc_softc *sc, uint32_t *resp);
 213 static int mmc_set_blocklen(struct mmc_softc *sc, uint32_t len);
 214 static int mmc_set_card_bus_width(struct mmc_softc *sc, struct mmc_ivars *ivar,
 215     enum mmc_bus_timing timing);
 216 static int mmc_set_power_class(struct mmc_softc *sc, struct mmc_ivars *ivar);
 217 static int mmc_set_relative_addr(struct mmc_softc *sc, uint16_t resp);
 218 static int mmc_set_timing(struct mmc_softc *sc, struct mmc_ivars *ivar,
 219     enum mmc_bus_timing timing);
 220 static int mmc_set_vccq(struct mmc_softc *sc, struct mmc_ivars *ivar,
 221     enum mmc_bus_timing timing);
 222 static int mmc_switch_to_hs200(struct mmc_softc *sc, struct mmc_ivars *ivar,
 223     uint32_t clock);
 224 static int mmc_switch_to_hs400(struct mmc_softc *sc, struct mmc_ivars *ivar,
 225     uint32_t max_dtr, enum mmc_bus_timing max_timing);
 226 static int mmc_test_bus_width(struct mmc_softc *sc);
 227 static uint32_t mmc_timing_to_dtr(struct mmc_ivars *ivar,
 228     enum mmc_bus_timing timing);
 229 static const char *mmc_timing_to_string(enum mmc_bus_timing timing);
 230 static void mmc_update_child_list(struct mmc_softc *sc);
 231 static int mmc_wait_for_command(struct mmc_softc *sc, uint32_t opcode,
 232     uint32_t arg, uint32_t flags, uint32_t *resp, int retries);
 233 static int mmc_wait_for_req(struct mmc_softc *sc, struct mmc_request *req);
 234 static void mmc_wakeup(struct mmc_request *req);
 235
 236 static void
 237 mmc_ms_delay(int ms)
 238 {
 239
 240         DELAY(1000 * ms);       /* XXX BAD */
 241 }
 242
 243 static int
 244 mmc_probe(device_t dev)
 245 {
 246
 247         device_set_desc(dev, "MMC/SD bus");
 248         return (0);
 249 }
 250
 251 static int
 252 mmc_attach(device_t dev)
 253 {
 254         struct mmc_softc *sc;
 255
 256         sc = device_get_softc(dev);
 257         sc->dev = dev;
 258         MMC_LOCK_INIT(sc);
 259
 260         /* We'll probe and attach our children later, but before / mount */
 261         sc->config_intrhook.ich_func = mmc_delayed_attach;
 262         sc->config_intrhook.ich_arg = sc;
 263         if (config_intrhook_establish(&sc->config_intrhook) != 0)
 264                 device_printf(dev, "config_intrhook_establish failed\n");
 265         return (0);
 266 }
 267
 268 static int
 269 mmc_detach(device_t dev)
 270 {
 271         struct mmc_softc *sc = device_get_softc(dev);
 272         int err;
 273
 274         err = mmc_delete_cards(sc, true);
 275         if (err != 0)
 276                 return (err);
 277         mmc_power_down(sc);
 278         MMC_LOCK_DESTROY(sc);
 279
 280         return (0);
 281 }
 282
 283 static int
 284 mmc_suspend(device_t dev)
 285 {
 286         struct mmc_softc *sc = device_get_softc(dev);
 287         int err;
 288
 289         err = bus_generic_suspend(dev);
 290         if (err != 0)
 291                 return (err);
 292         /*
 293          * We power down with the bus acquired here, mainly so that no device
 294          * is selected any longer and sc->last_rca gets set to 0.  Otherwise,
 295          * the deselect as part of the bus acquisition in mmc_scan() may fail
 296          * during resume, as the bus isn't powered up again before later in
 297          * mmc_go_discovery().
 298          */
 299         err = mmc_acquire_bus(dev, dev);
 300         if (err != 0)
 301                 return (err);
 302         mmc_power_down(sc);
 303         err = mmc_release_bus(dev, dev);
 304         return (err);
 305 }
 306
 307 static int
 308 mmc_resume(device_t dev)
 309 {
 310         struct mmc_softc *sc = device_get_softc(dev);
 311
 312         mmc_scan(sc);
 313         return (bus_generic_resume(dev));
 314 }
 315
 316 static int
 317 mmc_acquire_bus(device_t busdev, device_t dev)
 318 {
 319         struct mmc_softc *sc;
 320         struct mmc_ivars *ivar;
 321         int err;
 322         uint16_t rca;
 323         enum mmc_bus_timing timing;
 324
 325         err = MMCBR_ACQUIRE_HOST(device_get_parent(busdev), busdev);
 326         if (err)
 327                 return (err);
 328         sc = device_get_softc(busdev);
 329         MMC_LOCK(sc);
 330         if (sc->owner)
 331                 panic("mmc: host bridge didn't serialize us.");
 332         sc->owner = dev;
 333         MMC_UNLOCK(sc);
 334
 335         if (busdev != dev) {
 336                 /*
 337                  * Keep track of the last rca that we've selected.  If
 338                  * we're asked to do it again, don't.  We never
 339                  * unselect unless the bus code itself wants the mmc
 340                  * bus, and constantly reselecting causes problems.
 341                  */
 342                 ivar = device_get_ivars(dev);
 343                 rca = ivar->rca;
 344                 if (sc->last_rca != rca) {
 345                         if (mmc_select_card(sc, rca) != MMC_ERR_NONE) {
 346                                 device_printf(busdev, "Card at relative "
 347                                     "address %d failed to select\n", rca);
 348                                 return (ENXIO);
 349                         }
 350                         sc->last_rca = rca;
 351                         timing = mmcbr_get_timing(busdev);
 352                         /*
 353                          * For eMMC modes, setting/updating bus width and VCCQ
 354                          * only really is necessary if there actually is more
 355                          * than one device on the bus as generally that already
 356                          * had to be done by mmc_calculate_clock() or one of
 357                          * its calees.  Moreover, setting the bus width anew
 358                          * can trigger re-tuning (via a CRC error on the next
 359                          * CMD), even if not switching between devices an the
 360                          * previously selected one is still tuned.  Obviously,
 361                          * we need to re-tune the host controller if devices
 362                          * are actually switched, though.
 363                          */
 364                         if (timing >= bus_timing_mmc_ddr52 &&
 365                             sc->child_count == 1)
 366                                 return (0);
 367                         /* Prepare bus width for the new card. */
 368                         if (bootverbose || mmc_debug) {
 369                                 device_printf(busdev,
 370                                     "setting bus width to %d bits %s timing\n",
 371                                     (ivar->bus_width == bus_width_4) ? 4 :
 372                                     (ivar->bus_width == bus_width_8) ? 8 : 1,
 373                                     mmc_timing_to_string(timing));
 374                         }
 375                         if (mmc_set_card_bus_width(sc, ivar, timing) !=
 376                             MMC_ERR_NONE) {
 377                                 device_printf(busdev, "Card at relative "
 378                                     "address %d failed to set bus width\n",
 379                                     rca);
 380                                 return (ENXIO);
 381                         }
 382                         mmcbr_set_bus_width(busdev, ivar->bus_width);
 383                         mmcbr_update_ios(busdev);
 384                         if (mmc_set_vccq(sc, ivar, timing) != MMC_ERR_NONE) {
 385                                 device_printf(busdev, "Failed to set VCCQ "
 386                                     "for card at relative address %d\n", rca);
 387                                 return (ENXIO);
 388                         }
 389                         if (timing >= bus_timing_mmc_hs200 &&
 390                             mmc_retune(busdev, dev, true) != 0) {
 391                                 device_printf(busdev, "Card at relative "
 392                                     "address %d failed to re-tune\n", rca);
 393                                 return (ENXIO);
 394                         }
 395                 }
 396         } else {
 397                 /*
 398                  * If there's a card selected, stand down.
 399                  */
 400                 if (sc->last_rca != 0) {
 401                         if (mmc_select_card(sc, 0) != MMC_ERR_NONE)
 402                                 return (ENXIO);
 403                         sc->last_rca = 0;
 404                 }
 405         }
 406
 407         return (0);
 408 }
 409
 410 static int
 411 mmc_release_bus(device_t busdev, device_t dev)
 412 {
 413         struct mmc_softc *sc;
 414         int err;
 415
 416         sc = device_get_softc(busdev);
 417
 418         MMC_LOCK(sc);
 419         if (!sc->owner)
 420                 panic("mmc: releasing unowned bus.");
 421         if (sc->owner != dev)
 422                 panic("mmc: you don't own the bus.  game over.");
 423         MMC_UNLOCK(sc);
 424         err = MMCBR_RELEASE_HOST(device_get_parent(busdev), busdev);
 425         if (err)
 426                 return (err);
 427         MMC_LOCK(sc);
 428         sc->owner = NULL;
 429         MMC_UNLOCK(sc);
 430         return (0);
 431 }
 432
 433 static uint32_t
 434 mmc_select_vdd(struct mmc_softc *sc, uint32_t ocr)
 435 {
 436
 437         return (ocr & MMC_OCR_VOLTAGE);
 438 }
 439
 440 static int
 441 mmc_highest_voltage(uint32_t ocr)
 442 {
 443         int i;
 444
 445         for (i = MMC_OCR_MAX_VOLTAGE_SHIFT;
 446             i >= MMC_OCR_MIN_VOLTAGE_SHIFT; i--)
 447                 if (ocr & (1 << i))
 448                         return (i);
 449         return (-1);
 450 }
 451
 452 static void
 453 mmc_wakeup(struct mmc_request *req)
 454 {
 455         struct mmc_softc *sc;
 456
 457         sc = (struct mmc_softc *)req->done_data;
 458         MMC_LOCK(sc);
 459         req->flags |= MMC_REQ_DONE;
 460         MMC_UNLOCK(sc);
 461         wakeup(req);
 462 }
 463
 464 static int
 465 mmc_wait_for_req(struct mmc_softc *sc, struct mmc_request *req)
 466 {
 467
 468         req->done = mmc_wakeup;
 469         req->done_data = sc;
 470         if (__predict_false(mmc_debug > 1)) {
 471                 device_printf(sc->dev, "REQUEST: CMD%d arg %#x flags %#x",
 472                     req->cmd->opcode, req->cmd->arg, req->cmd->flags);
 473                 if (req->cmd->data) {
 474                         printf(" data %d\n", (int)req->cmd->data->len);
 475                 } else
 476                         printf("\n");
 477         }
 478         MMCBR_REQUEST(device_get_parent(sc->dev), sc->dev, req);
 479         MMC_LOCK(sc);
 480         while ((req->flags & MMC_REQ_DONE) == 0)
 481                 msleep(req, &sc->sc_mtx, 0, "mmcreq", 0);
 482         MMC_UNLOCK(sc);
 483         if (__predict_false(mmc_debug > 2 || (mmc_debug > 0 &&
 484             req->cmd->error != MMC_ERR_NONE)))
 485                 device_printf(sc->dev, "CMD%d RESULT: %d\n",
 486                     req->cmd->opcode, req->cmd->error);
 487         return (0);
 488 }
 489
 490 static int
 491 mmc_wait_for_request(device_t busdev, device_t dev, struct mmc_request *req)
 492 {
 493         struct mmc_softc *sc;
 494         struct mmc_ivars *ivar;
 495         int err, i;
 496         enum mmc_retune_req retune_req;
 497
 498         sc = device_get_softc(busdev);
 499         KASSERT(sc->owner != NULL,
 500             ("%s: Request from %s without bus being acquired.", __func__,
 501             device_get_nameunit(dev)));
 502
 503         /*
 504          * Unless no device is selected or re-tuning is already ongoing,
 505          * execute re-tuning if a) the bridge is requesting to do so and
 506          * re-tuning hasn't been otherwise paused, or b) if a child asked
 507          * to be re-tuned prior to pausing (see also mmc_retune_pause()).
 508          */
 509         if (__predict_false(sc->last_rca != 0 && sc->retune_ongoing == 0 &&
 510             (((retune_req = mmcbr_get_retune_req(busdev)) != retune_req_none &&
 511             sc->retune_paused == 0) || sc->retune_needed == 1))) {
 512                 if (__predict_false(mmc_debug > 1)) {
 513                         device_printf(busdev,
 514                             "Re-tuning with%s circuit reset required\n",
 515                             retune_req == retune_req_reset ? "" : "out");
 516                 }
 517                 if (device_get_parent(dev) == busdev)
 518                         ivar = device_get_ivars(dev);
 519                 else {
 520                         for (i = 0; i < sc->child_count; i++) {
 521                                 ivar = device_get_ivars(sc->child_list[i]);
 522                                 if (ivar->rca == sc->last_rca)
 523                                         break;
 524                         }
 525                         if (ivar->rca != sc->last_rca)
 526                                 return (EINVAL);
 527                 }
 528                 sc->retune_ongoing = 1;
 529                 err = mmc_retune(busdev, dev, retune_req == retune_req_reset);
 530                 sc->retune_ongoing = 0;
 531                 switch (err) {
 532                 case MMC_ERR_NONE:
 533                 case MMC_ERR_FAILED:    /* Re-tune error but still might work */
 534                         break;
 535                 case MMC_ERR_BADCRC:    /* Switch failure on HS400 recovery */
 536                         return (ENXIO);
 537                 case MMC_ERR_INVALID:   /* Driver implementation b0rken */
 538                 default:                /* Unknown error, should not happen */
 539                         return (EINVAL);
 540                 }
 541                 sc->retune_needed = 0;
 542         }
 543         return (mmc_wait_for_req(sc, req));
 544 }
 545
 546 static int
 547 mmc_wait_for_command(struct mmc_softc *sc, uint32_t opcode,
 548     uint32_t arg, uint32_t flags, uint32_t *resp, int retries)
 549 {
 550         struct mmc_command cmd;
 551         int err;
 552
 553         memset(&cmd, 0, sizeof(cmd));
 554         cmd.opcode = opcode;
 555         cmd.arg = arg;
 556         cmd.flags = flags;
 557         cmd.data = NULL;
 558         err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, retries);
 559         if (err)
 560                 return (err);
 561         if (resp) {
 562                 if (flags & MMC_RSP_136)
 563                         memcpy(resp, cmd.resp, 4 * sizeof(uint32_t));
 564                 else
 565                         *resp = cmd.resp[0];
 566         }
 567         return (0);
 568 }
 569
 570 static void
 571 mmc_idle_cards(struct mmc_softc *sc)
 572 {
 573         device_t dev;
 574         struct mmc_command cmd;
 575
 576         dev = sc->dev;
 577         mmcbr_set_chip_select(dev, cs_high);
 578         mmcbr_update_ios(dev);
 579         mmc_ms_delay(1);
 580
 581         memset(&cmd, 0, sizeof(cmd));
 582         cmd.opcode = MMC_GO_IDLE_STATE;
 583         cmd.arg = 0;
 584         cmd.flags = MMC_RSP_NONE | MMC_CMD_BC;
 585         cmd.data = NULL;
 586         mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, CMD_RETRIES);
 587         mmc_ms_delay(1);
 588
 589         mmcbr_set_chip_select(dev, cs_dontcare);
 590         mmcbr_update_ios(dev);
 591         mmc_ms_delay(1);
 592 }
 593
 594 static int
 595 mmc_send_app_op_cond(struct mmc_softc *sc, uint32_t ocr, uint32_t *rocr)
 596 {
 597         struct mmc_command cmd;
 598         int err = MMC_ERR_NONE, i;
 599
 600         memset(&cmd, 0, sizeof(cmd));
 601         cmd.opcode = ACMD_SD_SEND_OP_COND;
 602         cmd.arg = ocr;
 603         cmd.flags = MMC_RSP_R3 | MMC_CMD_BCR;
 604         cmd.data = NULL;
 605
 606         for (i = 0; i < 1000; i++) {
 607                 err = mmc_wait_for_app_cmd(sc->dev, sc->dev, 0, &cmd,
 608                     CMD_RETRIES);
 609                 if (err != MMC_ERR_NONE)
 610                         break;
 611                 if ((cmd.resp[0] & MMC_OCR_CARD_BUSY) ||
 612                     (ocr & MMC_OCR_VOLTAGE) == 0)
 613                         break;
 614                 err = MMC_ERR_TIMEOUT;
 615                 mmc_ms_delay(10);
 616         }
 617         if (rocr && err == MMC_ERR_NONE)
 618                 *rocr = cmd.resp[0];
 619         return (err);
 620 }
 621
 622 static int
 623 mmc_send_op_cond(struct mmc_softc *sc, uint32_t ocr, uint32_t *rocr)
 624 {
 625         struct mmc_command cmd;
 626         int err = MMC_ERR_NONE, i;
 627
 628         memset(&cmd, 0, sizeof(cmd));
 629         cmd.opcode = MMC_SEND_OP_COND;
 630         cmd.arg = ocr;
 631         cmd.flags = MMC_RSP_R3 | MMC_CMD_BCR;
 632         cmd.data = NULL;
 633
 634         for (i = 0; i < 1000; i++) {
 635                 err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, CMD_RETRIES);
 636                 if (err != MMC_ERR_NONE)
 637                         break;
 638                 if ((cmd.resp[0] & MMC_OCR_CARD_BUSY) ||
 639                     (ocr & MMC_OCR_VOLTAGE) == 0)
 640                         break;
 641                 err = MMC_ERR_TIMEOUT;
 642                 mmc_ms_delay(10);
 643         }
 644         if (rocr && err == MMC_ERR_NONE)
 645                 *rocr = cmd.resp[0];
 646         return (err);
 647 }
 648
 649 static int
 650 mmc_send_if_cond(struct mmc_softc *sc, uint8_t vhs)
 651 {
 652         struct mmc_command cmd;
 653         int err;
 654
 655         memset(&cmd, 0, sizeof(cmd));
 656         cmd.opcode = SD_SEND_IF_COND;
 657         cmd.arg = (vhs << 8) + 0xAA;
 658         cmd.flags = MMC_RSP_R7 | MMC_CMD_BCR;
 659         cmd.data = NULL;
 660
 661         err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, CMD_RETRIES);
 662         return (err);
 663 }
 664
 665 static void
 666 mmc_power_up(struct mmc_softc *sc)
 667 {
 668         device_t dev;
 669         enum mmc_vccq vccq;
 670
 671         dev = sc->dev;
 672         mmcbr_set_vdd(dev, mmc_highest_voltage(mmcbr_get_host_ocr(dev)));
 673         mmcbr_set_bus_mode(dev, opendrain);
 674         mmcbr_set_chip_select(dev, cs_dontcare);
 675         mmcbr_set_bus_width(dev, bus_width_1);
 676         mmcbr_set_power_mode(dev, power_up);
 677         mmcbr_set_clock(dev, 0);
 678         mmcbr_update_ios(dev);
 679         for (vccq = vccq_330; ; vccq--) {
 680                 mmcbr_set_vccq(dev, vccq);
 681                 if (mmcbr_switch_vccq(dev) == 0 || vccq == vccq_120)
 682                         break;
 683         }
 684         mmc_ms_delay(1);
 685
 686         mmcbr_set_clock(dev, SD_MMC_CARD_ID_FREQUENCY);
 687         mmcbr_set_timing(dev, bus_timing_normal);
 688         mmcbr_set_power_mode(dev, power_on);
 689         mmcbr_update_ios(dev);
 690         mmc_ms_delay(2);
 691 }
 692
 693 static void
 694 mmc_power_down(struct mmc_softc *sc)
 695 {
 696         device_t dev = sc->dev;
 697
 698         mmcbr_set_bus_mode(dev, opendrain);
 699         mmcbr_set_chip_select(dev, cs_dontcare);
 700         mmcbr_set_bus_width(dev, bus_width_1);
 701         mmcbr_set_power_mode(dev, power_off);
 702         mmcbr_set_clock(dev, 0);
 703         mmcbr_set_timing(dev, bus_timing_normal);
 704         mmcbr_update_ios(dev);
 705 }
 706
 707 static int
 708 mmc_select_card(struct mmc_softc *sc, uint16_t rca)
 709 {
 710         int err, flags;
 711
 712         flags = (rca ? MMC_RSP_R1B : MMC_RSP_NONE) | MMC_CMD_AC;
 713         sc->retune_paused++;
 714         err = mmc_wait_for_command(sc, MMC_SELECT_CARD, (uint32_t)rca << 16,
 715             flags, NULL, CMD_RETRIES);
 716         sc->retune_paused--;
 717         return (err);
 718 }
 719
 720 static int
 721 mmc_sd_switch(struct mmc_softc *sc, uint8_t mode, uint8_t grp, uint8_t value,
 722     uint8_t *res)
 723 {
 724         int err;
 725         struct mmc_command cmd;
 726         struct mmc_data data;
 727
 728         memset(&cmd, 0, sizeof(cmd));
 729         memset(&data, 0, sizeof(data));
 730         memset(res, 0, 64);
 731
 732         cmd.opcode = SD_SWITCH_FUNC;
 733         cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC;
 734         cmd.arg = mode << 31;                   /* 0 - check, 1 - set */
 735         cmd.arg |= 0x00FFFFFF;
 736         cmd.arg &= ~(0xF << (grp * 4));
 737         cmd.arg |= value << (grp * 4);
 738         cmd.data = &data;
 739
 740         data.data = res;
 741         data.len = 64;
 742         data.flags = MMC_DATA_READ;
 743
 744         err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, CMD_RETRIES);
 745         return (err);
 746 }
 747
 748 static int
 749 mmc_set_card_bus_width(struct mmc_softc *sc, struct mmc_ivars *ivar,
 750     enum mmc_bus_timing timing)
 751 {
 752         struct mmc_command cmd;
 753         int err;
 754         uint8_t value;
 755
 756         if (mmcbr_get_mode(sc->dev) == mode_sd) {
 757                 memset(&cmd, 0, sizeof(cmd));
 758                 cmd.opcode = ACMD_SET_CLR_CARD_DETECT;
 759                 cmd.flags = MMC_RSP_R1 | MMC_CMD_AC;
 760                 cmd.arg = SD_CLR_CARD_DETECT;
 761                 err = mmc_wait_for_app_cmd(sc->dev, sc->dev, ivar->rca, &cmd,
 762                     CMD_RETRIES);
 763                 if (err != 0)
 764                         return (err);
 765                 memset(&cmd, 0, sizeof(cmd));
 766                 cmd.opcode = ACMD_SET_BUS_WIDTH;
 767                 cmd.flags = MMC_RSP_R1 | MMC_CMD_AC;
 768                 switch (ivar->bus_width) {
 769                 case bus_width_1:
 770                         cmd.arg = SD_BUS_WIDTH_1;
 771                         break;
 772                 case bus_width_4:
 773                         cmd.arg = SD_BUS_WIDTH_4;
 774                         break;
 775                 default:
 776                         return (MMC_ERR_INVALID);
 777                 }
 778                 err = mmc_wait_for_app_cmd(sc->dev, sc->dev, ivar->rca, &cmd,
 779                     CMD_RETRIES);
 780         } else {
 781                 switch (ivar->bus_width) {
 782                 case bus_width_1:
 783                         if (timing == bus_timing_mmc_hs400 ||
 784                             timing == bus_timing_mmc_hs400es)
 785                                 return (MMC_ERR_INVALID);
 786                         value = EXT_CSD_BUS_WIDTH_1;
 787                         break;
 788                 case bus_width_4:
 789                         switch (timing) {
 790                         case bus_timing_mmc_ddr52:
 791                                 value = EXT_CSD_BUS_WIDTH_4_DDR;
 792                                 break;
 793                         case bus_timing_mmc_hs400:
 794                         case bus_timing_mmc_hs400es:
 795                                 return (MMC_ERR_INVALID);
 796                         default:
 797                                 value = EXT_CSD_BUS_WIDTH_4;
 798                                 break;
 799                         }
 800                         break;
 801                 case bus_width_8:
 802                         value = 0;
 803                         switch (timing) {
 804                         case bus_timing_mmc_hs400es:
 805                                 value = EXT_CSD_BUS_WIDTH_ES;
 806                                 /* FALLTHROUGH */
 807                         case bus_timing_mmc_ddr52:
 808                         case bus_timing_mmc_hs400:
 809                                 value |= EXT_CSD_BUS_WIDTH_8_DDR;
 810                                 break;
 811                         default:
 812                                 value = EXT_CSD_BUS_WIDTH_8;
 813                                 break;
 814                         }
 815                         break;
 816                 default:
 817                         return (MMC_ERR_INVALID);
 818                 }
 819                 err = mmc_switch(sc->dev, sc->dev, ivar->rca,
 820                     EXT_CSD_CMD_SET_NORMAL, EXT_CSD_BUS_WIDTH, value,
 821                     ivar->cmd6_time, true);
 822         }
 823         return (err);
 824 }
 825
 826 static int
 827 mmc_set_power_class(struct mmc_softc *sc, struct mmc_ivars *ivar)
 828 {
 829         device_t dev;
 830         const uint8_t *ext_csd;
 831         uint32_t clock;
 832         uint8_t value;
 833
 834         dev = sc->dev;
 835         if (mmcbr_get_mode(dev) != mode_mmc || ivar->csd.spec_vers < 4)
 836                 return (MMC_ERR_NONE);
 837
 838         value = 0;
 839         ext_csd = ivar->raw_ext_csd;
 840         clock = mmcbr_get_clock(dev);
 841         switch (1 << mmcbr_get_vdd(dev)) {
 842         case MMC_OCR_LOW_VOLTAGE:
 843                 if (clock <= MMC_TYPE_HS_26_MAX)
 844                         value = ext_csd[EXT_CSD_PWR_CL_26_195];
 845                 else if (clock <= MMC_TYPE_HS_52_MAX) {
 846                         if (mmcbr_get_timing(dev) >= bus_timing_mmc_ddr52 &&
 847                             ivar->bus_width >= bus_width_4)
 848                                 value = ext_csd[EXT_CSD_PWR_CL_52_195_DDR];
 849                         else
 850                                 value = ext_csd[EXT_CSD_PWR_CL_52_195];
 851                 } else if (clock <= MMC_TYPE_HS200_HS400ES_MAX)
 852                         value = ext_csd[EXT_CSD_PWR_CL_200_195];
 853                 break;
 854         case MMC_OCR_270_280:
 855         case MMC_OCR_280_290:
 856         case MMC_OCR_290_300:
 857         case MMC_OCR_300_310:
 858         case MMC_OCR_310_320:
 859         case MMC_OCR_320_330:
 860         case MMC_OCR_330_340:
 861         case MMC_OCR_340_350:
 862         case MMC_OCR_350_360:
 863                 if (clock <= MMC_TYPE_HS_26_MAX)
 864                         value = ext_csd[EXT_CSD_PWR_CL_26_360];
 865                 else if (clock <= MMC_TYPE_HS_52_MAX) {
 866                         if (mmcbr_get_timing(dev) == bus_timing_mmc_ddr52 &&
 867                             ivar->bus_width >= bus_width_4)
 868                                 value = ext_csd[EXT_CSD_PWR_CL_52_360_DDR];
 869                         else
 870                                 value = ext_csd[EXT_CSD_PWR_CL_52_360];
 871                 } else if (clock <= MMC_TYPE_HS200_HS400ES_MAX) {
 872                         if (ivar->bus_width == bus_width_8)
 873                                 value = ext_csd[EXT_CSD_PWR_CL_200_360_DDR];
 874                         else
 875                                 value = ext_csd[EXT_CSD_PWR_CL_200_360];
 876                 }
 877                 break;
 878         default:
 879                 device_printf(dev, "No power class support for VDD 0x%x\n",
 880                         1 << mmcbr_get_vdd(dev));
 881                 return (MMC_ERR_INVALID);
 882         }
 883
 884         if (ivar->bus_width == bus_width_8)
 885                 value = (value & EXT_CSD_POWER_CLASS_8BIT_MASK) >>
 886                     EXT_CSD_POWER_CLASS_8BIT_SHIFT;
 887         else
 888                 value = (value & EXT_CSD_POWER_CLASS_4BIT_MASK) >>
 889                     EXT_CSD_POWER_CLASS_4BIT_SHIFT;
 890
 891         if (value == 0)
 892                 return (MMC_ERR_NONE);
 893
 894         return (mmc_switch(dev, dev, ivar->rca, EXT_CSD_CMD_SET_NORMAL,
 895             EXT_CSD_POWER_CLASS, value, ivar->cmd6_time, true));
 896 }
 897
 898 static int
 899 mmc_set_timing(struct mmc_softc *sc, struct mmc_ivars *ivar,
 900     enum mmc_bus_timing timing)
 901 {
 902         u_char switch_res[64];
 903         uint8_t value;
 904         int err;
 905
 906         if (mmcbr_get_mode(sc->dev) == mode_sd) {
 907                 switch (timing) {
 908                 case bus_timing_normal:
 909                         value = SD_SWITCH_NORMAL_MODE;
 910                         break;
 911                 case bus_timing_hs:
 912                         value = SD_SWITCH_HS_MODE;
 913                         break;
 914                 default:
 915                         return (MMC_ERR_INVALID);
 916                 }
 917                 err = mmc_sd_switch(sc, SD_SWITCH_MODE_SET, SD_SWITCH_GROUP1,
 918                     value, switch_res);
 919                 if (err != MMC_ERR_NONE)
 920                         return (err);
 921                 if ((switch_res[16] & 0xf) != value)
 922                         return (MMC_ERR_FAILED);
 923                 mmcbr_set_timing(sc->dev, timing);
 924                 mmcbr_update_ios(sc->dev);
 925         } else {
 926                 switch (timing) {
 927                 case bus_timing_normal:
 928                         value = EXT_CSD_HS_TIMING_BC;
 929                         break;
 930                 case bus_timing_hs:
 931                 case bus_timing_mmc_ddr52:
 932                         value = EXT_CSD_HS_TIMING_HS;
 933                         break;
 934                 case bus_timing_mmc_hs200:
 935                         value = EXT_CSD_HS_TIMING_HS200;
 936                         break;
 937                 case bus_timing_mmc_hs400:
 938                 case bus_timing_mmc_hs400es:
 939                         value = EXT_CSD_HS_TIMING_HS400;
 940                         break;
 941                 default:
 942                         return (MMC_ERR_INVALID);
 943                 }
 944                 err = mmc_switch(sc->dev, sc->dev, ivar->rca,
 945                     EXT_CSD_CMD_SET_NORMAL, EXT_CSD_HS_TIMING, value,
 946                     ivar->cmd6_time, false);
 947                 if (err != MMC_ERR_NONE)
 948                         return (err);
 949                 mmcbr_set_timing(sc->dev, timing);
 950                 mmcbr_update_ios(sc->dev);
 951                 err = mmc_switch_status(sc->dev, sc->dev, ivar->rca,
 952                     ivar->cmd6_time);
 953         }
 954         return (err);
 955 }
 956
 957 static int
 958 mmc_set_vccq(struct mmc_softc *sc, struct mmc_ivars *ivar,
 959     enum mmc_bus_timing timing)
 960 {
 961
 962         if (isset(&ivar->vccq_120, timing))
 963                 mmcbr_set_vccq(sc->dev, vccq_120);
 964         else if (isset(&ivar->vccq_180, timing))
 965                 mmcbr_set_vccq(sc->dev, vccq_180);
 966         else
 967                 mmcbr_set_vccq(sc->dev, vccq_330);
 968         if (mmcbr_switch_vccq(sc->dev) != 0)
 969                 return (MMC_ERR_INVALID);
 970         else
 971                 return (MMC_ERR_NONE);
 972 }
 973
 974 static const uint8_t p8[8] = {
 975         0x55, 0xAA, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
 976 };
 977
 978 static const uint8_t p8ok[8] = {
 979         0xAA, 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00
 980 };
 981
 982 static const uint8_t p4[4] = {
 983         0x5A, 0x00, 0x00, 0x00
 984 };
 985
 986 static const uint8_t p4ok[4] = {
 987         0xA5, 0x00, 0x00, 0x00
 988 };
 989
 990 static int
 991 mmc_test_bus_width(struct mmc_softc *sc)
 992 {
 993         struct mmc_command cmd;
 994         struct mmc_data data;
 995         uint8_t buf[8];
 996         int err;
 997
 998         if (mmcbr_get_caps(sc->dev) & MMC_CAP_8_BIT_DATA) {
 999                 mmcbr_set_bus_width(sc->dev, bus_width_8);
1000                 mmcbr_update_ios(sc->dev);
1001
1002                 sc->squelched++; /* Errors are expected, squelch reporting. */
1003                 memset(&cmd, 0, sizeof(cmd));
1004                 memset(&data, 0, sizeof(data));
1005                 cmd.opcode = MMC_BUSTEST_W;
1006                 cmd.arg = 0;
1007                 cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC;
1008                 cmd.data = &data;
1009
1010                 data.data = __DECONST(void *, p8);
1011                 data.len = 8;
1012                 data.flags = MMC_DATA_WRITE;
1013                 mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, 0);
1014
1015                 memset(&cmd, 0, sizeof(cmd));
1016                 memset(&data, 0, sizeof(data));
1017                 cmd.opcode = MMC_BUSTEST_R;
1018                 cmd.arg = 0;
1019                 cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC;
1020                 cmd.data = &data;
1021
1022                 data.data = buf;
1023                 data.len = 8;
1024                 data.flags = MMC_DATA_READ;
1025                 err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, 0);
1026                 sc->squelched--;
1027
1028                 mmcbr_set_bus_width(sc->dev, bus_width_1);
1029                 mmcbr_update_ios(sc->dev);
1030
1031                 if (err == MMC_ERR_NONE && memcmp(buf, p8ok, 8) == 0)
1032                         return (bus_width_8);
1033         }
1034
1035         if (mmcbr_get_caps(sc->dev) & MMC_CAP_4_BIT_DATA) {
1036                 mmcbr_set_bus_width(sc->dev, bus_width_4);
1037                 mmcbr_update_ios(sc->dev);
1038
1039                 sc->squelched++; /* Errors are expected, squelch reporting. */
1040                 memset(&cmd, 0, sizeof(cmd));
1041                 memset(&data, 0, sizeof(data));
1042                 cmd.opcode = MMC_BUSTEST_W;
1043                 cmd.arg = 0;
1044                 cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC;
1045                 cmd.data = &data;
1046
1047                 data.data = __DECONST(void *, p4);
1048                 data.len = 4;
1049                 data.flags = MMC_DATA_WRITE;
1050                 mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, 0);
1051
1052                 memset(&cmd, 0, sizeof(cmd));
1053                 memset(&data, 0, sizeof(data));
1054                 cmd.opcode = MMC_BUSTEST_R;
1055                 cmd.arg = 0;
1056                 cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC;
1057                 cmd.data = &data;
1058
1059                 data.data = buf;
1060                 data.len = 4;
1061                 data.flags = MMC_DATA_READ;
1062                 err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, 0);
1063                 sc->squelched--;
1064
1065                 mmcbr_set_bus_width(sc->dev, bus_width_1);
1066                 mmcbr_update_ios(sc->dev);
1067
1068                 if (err == MMC_ERR_NONE && memcmp(buf, p4ok, 4) == 0)
1069                         return (bus_width_4);
1070         }
1071         return (bus_width_1);
1072 }
1073
1074 static uint32_t
1075 mmc_get_bits(uint32_t *bits, int bit_len, int start, int size)
1076 {
1077         const int i = (bit_len / 32) - (start / 32) - 1;
1078         const int shift = start & 31;
1079         uint32_t retval = bits[i] >> shift;
1080
1081         if (size + shift > 32)
1082                 retval |= bits[i - 1] << (32 - shift);
1083         return (retval & ((1llu << size) - 1));
1084 }
1085
1086 static void
1087 mmc_decode_cid_sd(uint32_t *raw_cid, struct mmc_cid *cid)
1088 {
1089         int i;
1090
1091         /* There's no version info, so we take it on faith */
1092         memset(cid, 0, sizeof(*cid));
1093         cid->mid = mmc_get_bits(raw_cid, 128, 120, 8);
1094         cid->oid = mmc_get_bits(raw_cid, 128, 104, 16);
1095         for (i = 0; i < 5; i++)
1096                 cid->pnm[i] = mmc_get_bits(raw_cid, 128, 96 - i * 8, 8);
1097         cid->pnm[5] = 0;
1098         cid->prv = mmc_get_bits(raw_cid, 128, 56, 8);
1099         cid->psn = mmc_get_bits(raw_cid, 128, 24, 32);
1100         cid->mdt_year = mmc_get_bits(raw_cid, 128, 12, 8) + 2000;
1101         cid->mdt_month = mmc_get_bits(raw_cid, 128, 8, 4);
1102 }
1103
1104 static void
1105 mmc_decode_cid_mmc(uint32_t *raw_cid, struct mmc_cid *cid, bool is_4_41p)
1106 {
1107         int i;
1108
1109         /* There's no version info, so we take it on faith */
1110         memset(cid, 0, sizeof(*cid));
1111         cid->mid = mmc_get_bits(raw_cid, 128, 120, 8);
1112         cid->oid = mmc_get_bits(raw_cid, 128, 104, 8);
1113         for (i = 0; i < 6; i++)
1114                 cid->pnm[i] = mmc_get_bits(raw_cid, 128, 96 - i * 8, 8);
1115         cid->pnm[6] = 0;
1116         cid->prv = mmc_get_bits(raw_cid, 128, 48, 8);
1117         cid->psn = mmc_get_bits(raw_cid, 128, 16, 32);
1118         cid->mdt_month = mmc_get_bits(raw_cid, 128, 12, 4);
1119         cid->mdt_year = mmc_get_bits(raw_cid, 128, 8, 4);
1120         if (is_4_41p)
1121                 cid->mdt_year += 2013;
1122         else
1123                 cid->mdt_year += 1997;
1124 }
1125
1126 static void
1127 mmc_format_card_id_string(struct mmc_ivars *ivar)
1128 {
1129         char oidstr[8];
1130         uint8_t c1;
1131         uint8_t c2;
1132
1133         /*
1134          * Format a card ID string for use by the mmcsd driver, it's what
1135          * appears between the <> in the following:
1136          * mmcsd0: 968MB <SD SD01G 8.0 SN 2686905 MFG 08/2008 by 3 TN> at mmc0
1137          * 22.5MHz/4bit/128-block
1138          *
1139          * Also format just the card serial number, which the mmcsd driver will
1140          * use as the disk->d_ident string.
1141          *
1142          * The card_id_string in mmc_ivars is currently allocated as 64 bytes,
1143          * and our max formatted length is currently 55 bytes if every field
1144          * contains the largest value.
1145          *
1146          * Sometimes the oid is two printable ascii chars; when it's not,
1147          * format it as 0xnnnn instead.
1148          */
1149         c1 = (ivar->cid.oid >> 8) & 0x0ff;
1150         c2 = ivar->cid.oid & 0x0ff;
1151         if (c1 > 0x1f && c1 < 0x7f && c2 > 0x1f && c2 < 0x7f)
1152                 snprintf(oidstr, sizeof(oidstr), "%c%c", c1, c2);
1153         else
1154                 snprintf(oidstr, sizeof(oidstr), "0x%04x", ivar->cid.oid);
1155         snprintf(ivar->card_sn_string, sizeof(ivar->card_sn_string),
1156             "%08X", ivar->cid.psn);
1157         snprintf(ivar->card_id_string, sizeof(ivar->card_id_string),
1158             "%s%s %s %d.%d SN %08X MFG %02d/%04d by %d %s",
1159             ivar->mode == mode_sd ? "SD" : "MMC", ivar->high_cap ? "HC" : "",
1160             ivar->cid.pnm, ivar->cid.prv >> 4, ivar->cid.prv & 0x0f,
1161             ivar->cid.psn, ivar->cid.mdt_month, ivar->cid.mdt_year,
1162             ivar->cid.mid, oidstr);
1163 }
1164
1165 static const int exp[8] = {
1166         1, 10, 100, 1000, 10000, 100000, 1000000, 10000000
1167 };
1168
1169 static const int mant[16] = {
1170         0, 10, 12, 13, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 70, 80
1171 };
1172
1173 static const int cur_min[8] = {
1174         500, 1000, 5000, 10000, 25000, 35000, 60000, 100000
1175 };
1176
1177 static const int cur_max[8] = {
1178         1000, 5000, 10000, 25000, 35000, 45000, 800000, 200000
1179 };
1180
1181 static int
1182 mmc_decode_csd_sd(uint32_t *raw_csd, struct mmc_csd *csd)
1183 {
1184         int v;
1185         int m;
1186         int e;
1187
1188         memset(csd, 0, sizeof(*csd));
1189         csd->csd_structure = v = mmc_get_bits(raw_csd, 128, 126, 2);
1190         if (v == 0) {
1191                 m = mmc_get_bits(raw_csd, 128, 115, 4);
1192                 e = mmc_get_bits(raw_csd, 128, 112, 3);
1193                 csd->tacc = (exp[e] * mant[m] + 9) / 10;
1194                 csd->nsac = mmc_get_bits(raw_csd, 128, 104, 8) * 100;
1195                 m = mmc_get_bits(raw_csd, 128, 99, 4);
1196                 e = mmc_get_bits(raw_csd, 128, 96, 3);
1197                 csd->tran_speed = exp[e] * 10000 * mant[m];
1198                 csd->ccc = mmc_get_bits(raw_csd, 128, 84, 12);
1199                 csd->read_bl_len = 1 << mmc_get_bits(raw_csd, 128, 80, 4);
1200                 csd->read_bl_partial = mmc_get_bits(raw_csd, 128, 79, 1);
1201                 csd->write_blk_misalign = mmc_get_bits(raw_csd, 128, 78, 1);
1202                 csd->read_blk_misalign = mmc_get_bits(raw_csd, 128, 77, 1);
1203                 csd->dsr_imp = mmc_get_bits(raw_csd, 128, 76, 1);
1204                 csd->vdd_r_curr_min =
1205                     cur_min[mmc_get_bits(raw_csd, 128, 59, 3)];
1206                 csd->vdd_r_curr_max =
1207                     cur_max[mmc_get_bits(raw_csd, 128, 56, 3)];
1208                 csd->vdd_w_curr_min =
1209                     cur_min[mmc_get_bits(raw_csd, 128, 53, 3)];
1210                 csd->vdd_w_curr_max =
1211                     cur_max[mmc_get_bits(raw_csd, 128, 50, 3)];
1212                 m = mmc_get_bits(raw_csd, 128, 62, 12);
1213                 e = mmc_get_bits(raw_csd, 128, 47, 3);
1214                 csd->capacity = ((1 + m) << (e + 2)) * csd->read_bl_len;
1215                 csd->erase_blk_en = mmc_get_bits(raw_csd, 128, 46, 1);
1216                 csd->erase_sector = mmc_get_bits(raw_csd, 128, 39, 7) + 1;
1217                 csd->wp_grp_size = mmc_get_bits(raw_csd, 128, 32, 7);
1218                 csd->wp_grp_enable = mmc_get_bits(raw_csd, 128, 31, 1);
1219                 csd->r2w_factor = 1 << mmc_get_bits(raw_csd, 128, 26, 3);
1220                 csd->write_bl_len = 1 << mmc_get_bits(raw_csd, 128, 22, 4);
1221                 csd->write_bl_partial = mmc_get_bits(raw_csd, 128, 21, 1);
1222                 return (MMC_ERR_NONE);
1223         } else if (v == 1) {
1224                 m = mmc_get_bits(raw_csd, 128, 115, 4);
1225                 e = mmc_get_bits(raw_csd, 128, 112, 3);
1226                 csd->tacc = (exp[e] * mant[m] + 9) / 10;
1227                 csd->nsac = mmc_get_bits(raw_csd, 128, 104, 8) * 100;
1228                 m = mmc_get_bits(raw_csd, 128, 99, 4);
1229                 e = mmc_get_bits(raw_csd, 128, 96, 3);
1230                 csd->tran_speed = exp[e] * 10000 * mant[m];
1231                 csd->ccc = mmc_get_bits(raw_csd, 128, 84, 12);
1232                 csd->read_bl_len = 1 << mmc_get_bits(raw_csd, 128, 80, 4);
1233                 csd->read_bl_partial = mmc_get_bits(raw_csd, 128, 79, 1);
1234                 csd->write_blk_misalign = mmc_get_bits(raw_csd, 128, 78, 1);
1235                 csd->read_blk_misalign = mmc_get_bits(raw_csd, 128, 77, 1);
1236                 csd->dsr_imp = mmc_get_bits(raw_csd, 128, 76, 1);
1237                 csd->capacity = ((uint64_t)mmc_get_bits(raw_csd, 128, 48, 22) +
1238                     1) * 512 * 1024;
1239                 csd->erase_blk_en = mmc_get_bits(raw_csd, 128, 46, 1);
1240                 csd->erase_sector = mmc_get_bits(raw_csd, 128, 39, 7) + 1;
1241                 csd->wp_grp_size = mmc_get_bits(raw_csd, 128, 32, 7);
1242                 csd->wp_grp_enable = mmc_get_bits(raw_csd, 128, 31, 1);
1243                 csd->r2w_factor = 1 << mmc_get_bits(raw_csd, 128, 26, 3);
1244                 csd->write_bl_len = 1 << mmc_get_bits(raw_csd, 128, 22, 4);
1245                 csd->write_bl_partial = mmc_get_bits(raw_csd, 128, 21, 1);
1246                 return (MMC_ERR_NONE);
1247         }
1248         return (MMC_ERR_INVALID);
1249 }
1250
1251 static void
1252 mmc_decode_csd_mmc(uint32_t *raw_csd, struct mmc_csd *csd)
1253 {
1254         int m;
1255         int e;
1256
1257         memset(csd, 0, sizeof(*csd));
1258         csd->csd_structure = mmc_get_bits(raw_csd, 128, 126, 2);
1259         csd->spec_vers = mmc_get_bits(raw_csd, 128, 122, 4);
1260         m = mmc_get_bits(raw_csd, 128, 115, 4);
1261         e = mmc_get_bits(raw_csd, 128, 112, 3);
1262         csd->tacc = exp[e] * mant[m] + 9 / 10;
1263         csd->nsac = mmc_get_bits(raw_csd, 128, 104, 8) * 100;
1264         m = mmc_get_bits(raw_csd, 128, 99, 4);
1265         e = mmc_get_bits(raw_csd, 128, 96, 3);
1266         csd->tran_speed = exp[e] * 10000 * mant[m];
1267         csd->ccc = mmc_get_bits(raw_csd, 128, 84, 12);
1268         csd->read_bl_len = 1 << mmc_get_bits(raw_csd, 128, 80, 4);
1269         csd->read_bl_partial = mmc_get_bits(raw_csd, 128, 79, 1);
1270         csd->write_blk_misalign = mmc_get_bits(raw_csd, 128, 78, 1);
1271         csd->read_blk_misalign = mmc_get_bits(raw_csd, 128, 77, 1);
1272         csd->dsr_imp = mmc_get_bits(raw_csd, 128, 76, 1);
1273         csd->vdd_r_curr_min = cur_min[mmc_get_bits(raw_csd, 128, 59, 3)];
1274         csd->vdd_r_curr_max = cur_max[mmc_get_bits(raw_csd, 128, 56, 3)];
1275         csd->vdd_w_curr_min = cur_min[mmc_get_bits(raw_csd, 128, 53, 3)];
1276         csd->vdd_w_curr_max = cur_max[mmc_get_bits(raw_csd, 128, 50, 3)];
1277         m = mmc_get_bits(raw_csd, 128, 62, 12);
1278         e = mmc_get_bits(raw_csd, 128, 47, 3);
1279         csd->capacity = ((1 + m) << (e + 2)) * csd->read_bl_len;
1280         csd->erase_blk_en = 0;
1281         csd->erase_sector = (mmc_get_bits(raw_csd, 128, 42, 5) + 1) *
1282             (mmc_get_bits(raw_csd, 128, 37, 5) + 1);
1283         csd->wp_grp_size = mmc_get_bits(raw_csd, 128, 32, 5);
1284         csd->wp_grp_enable = mmc_get_bits(raw_csd, 128, 31, 1);
1285         csd->r2w_factor = 1 << mmc_get_bits(raw_csd, 128, 26, 3);
1286         csd->write_bl_len = 1 << mmc_get_bits(raw_csd, 128, 22, 4);
1287         csd->write_bl_partial = mmc_get_bits(raw_csd, 128, 21, 1);
1288 }
1289
1290 static void
1291 mmc_app_decode_scr(uint32_t *raw_scr, struct mmc_scr *scr)
1292 {
1293         unsigned int scr_struct;
1294
1295         memset(scr, 0, sizeof(*scr));
1296
1297         scr_struct = mmc_get_bits(raw_scr, 64, 60, 4);
1298         if (scr_struct != 0) {
1299                 printf("Unrecognised SCR structure version %d\n",
1300                     scr_struct);
1301                 return;
1302         }
1303         scr->sda_vsn = mmc_get_bits(raw_scr, 64, 56, 4);
1304         scr->bus_widths = mmc_get_bits(raw_scr, 64, 48, 4);
1305 }
1306
1307 static void
1308 mmc_app_decode_sd_status(uint32_t *raw_sd_status,
1309     struct mmc_sd_status *sd_status)
1310 {
1311
1312         memset(sd_status, 0, sizeof(*sd_status));
1313
1314         sd_status->bus_width = mmc_get_bits(raw_sd_status, 512, 510, 2);
1315         sd_status->secured_mode = mmc_get_bits(raw_sd_status, 512, 509, 1);
1316         sd_status->card_type = mmc_get_bits(raw_sd_status, 512, 480, 16);
1317         sd_status->prot_area = mmc_get_bits(raw_sd_status, 512, 448, 12);
1318         sd_status->speed_class = mmc_get_bits(raw_sd_status, 512, 440, 8);
1319         sd_status->perf_move = mmc_get_bits(raw_sd_status, 512, 432, 8);
1320         sd_status->au_size = mmc_get_bits(raw_sd_status, 512, 428, 4);
1321         sd_status->erase_size = mmc_get_bits(raw_sd_status, 512, 408, 16);
1322         sd_status->erase_timeout = mmc_get_bits(raw_sd_status, 512, 402, 6);
1323         sd_status->erase_offset = mmc_get_bits(raw_sd_status, 512, 400, 2);
1324 }
1325
1326 static int
1327 mmc_all_send_cid(struct mmc_softc *sc, uint32_t *rawcid)
1328 {
1329         struct mmc_command cmd;
1330         int err;
1331
1332         memset(&cmd, 0, sizeof(cmd));
1333         cmd.opcode = MMC_ALL_SEND_CID;
1334         cmd.arg = 0;
1335         cmd.flags = MMC_RSP_R2 | MMC_CMD_BCR;
1336         cmd.data = NULL;
1337         err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, CMD_RETRIES);
1338         memcpy(rawcid, cmd.resp, 4 * sizeof(uint32_t));
1339         return (err);
1340 }
1341
1342 static int
1343 mmc_send_csd(struct mmc_softc *sc, uint16_t rca, uint32_t *rawcsd)
1344 {
1345         struct mmc_command cmd;
1346         int err;
1347
1348         memset(&cmd, 0, sizeof(cmd));
1349         cmd.opcode = MMC_SEND_CSD;
1350         cmd.arg = rca << 16;
1351         cmd.flags = MMC_RSP_R2 | MMC_CMD_BCR;
1352         cmd.data = NULL;
1353         err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, CMD_RETRIES);
1354         memcpy(rawcsd, cmd.resp, 4 * sizeof(uint32_t));
1355         return (err);
1356 }
1357
1358 static int
1359 mmc_app_send_scr(struct mmc_softc *sc, uint16_t rca, uint32_t *rawscr)
1360 {
1361         int err;
1362         struct mmc_command cmd;
1363         struct mmc_data data;
1364
1365         memset(&cmd, 0, sizeof(cmd));
1366         memset(&data, 0, sizeof(data));
1367
1368         memset(rawscr, 0, 8);
1369         cmd.opcode = ACMD_SEND_SCR;
1370         cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC;
1371         cmd.arg = 0;
1372         cmd.data = &data;
1373
1374         data.data = rawscr;
1375         data.len = 8;
1376         data.flags = MMC_DATA_READ;
1377
1378         err = mmc_wait_for_app_cmd(sc->dev, sc->dev, rca, &cmd, CMD_RETRIES);
1379         rawscr[0] = be32toh(rawscr[0]);
1380         rawscr[1] = be32toh(rawscr[1]);
1381         return (err);
1382 }
1383
1384 static int
1385 mmc_app_sd_status(struct mmc_softc *sc, uint16_t rca, uint32_t *rawsdstatus)
1386 {
1387         struct mmc_command cmd;
1388         struct mmc_data data;
1389         int err, i;
1390
1391         memset(&cmd, 0, sizeof(cmd));
1392         memset(&data, 0, sizeof(data));
1393
1394         memset(rawsdstatus, 0, 64);
1395         cmd.opcode = ACMD_SD_STATUS;
1396         cmd.flags = MMC_RSP_R1 | MMC_CMD_ADTC;
1397         cmd.arg = 0;
1398         cmd.data = &data;
1399
1400         data.data = rawsdstatus;
1401         data.len = 64;
1402         data.flags = MMC_DATA_READ;
1403
1404         err = mmc_wait_for_app_cmd(sc->dev, sc->dev, rca, &cmd, CMD_RETRIES);
1405         for (i = 0; i < 16; i++)
1406             rawsdstatus[i] = be32toh(rawsdstatus[i]);
1407         return (err);
1408 }
1409
1410 static int
1411 mmc_set_relative_addr(struct mmc_softc *sc, uint16_t resp)
1412 {
1413         struct mmc_command cmd;
1414         int err;
1415
1416         memset(&cmd, 0, sizeof(cmd));
1417         cmd.opcode = MMC_SET_RELATIVE_ADDR;
1418         cmd.arg = resp << 16;
1419         cmd.flags = MMC_RSP_R6 | MMC_CMD_BCR;
1420         cmd.data = NULL;
1421         err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, CMD_RETRIES);
1422         return (err);
1423 }
1424
1425 static int
1426 mmc_send_relative_addr(struct mmc_softc *sc, uint32_t *resp)
1427 {
1428         struct mmc_command cmd;
1429         int err;
1430
1431         memset(&cmd, 0, sizeof(cmd));
1432         cmd.opcode = SD_SEND_RELATIVE_ADDR;
1433         cmd.arg = 0;
1434         cmd.flags = MMC_RSP_R6 | MMC_CMD_BCR;
1435         cmd.data = NULL;
1436         err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, CMD_RETRIES);
1437         *resp = cmd.resp[0];
1438         return (err);
1439 }
1440
1441 static int
1442 mmc_set_blocklen(struct mmc_softc *sc, uint32_t len)
1443 {
1444         struct mmc_command cmd;
1445         int err;
1446
1447         memset(&cmd, 0, sizeof(cmd));
1448         cmd.opcode = MMC_SET_BLOCKLEN;
1449         cmd.arg = len;
1450         cmd.flags = MMC_RSP_R1 | MMC_CMD_AC;
1451         cmd.data = NULL;
1452         err = mmc_wait_for_cmd(sc->dev, sc->dev, &cmd, CMD_RETRIES);
1453         return (err);
1454 }
1455
1456 static uint32_t
1457 mmc_timing_to_dtr(struct mmc_ivars *ivar, enum mmc_bus_timing timing)
1458 {
1459
1460         switch (timing) {
1461         case bus_timing_normal:
1462                 return (ivar->tran_speed);
1463         case bus_timing_hs:
1464                 return (ivar->hs_tran_speed);
1465         case bus_timing_uhs_sdr12:
1466                 return (SD_SDR12_MAX);
1467         case bus_timing_uhs_sdr25:
1468                 return (SD_SDR25_MAX);
1469         case bus_timing_uhs_ddr50:
1470                 return (SD_DDR50_MAX);
1471         case bus_timing_uhs_sdr50:
1472                 return (SD_SDR50_MAX);
1473         case bus_timing_uhs_sdr104:
1474                 return (SD_SDR104_MAX);
1475         case bus_timing_mmc_ddr52:
1476                 return (MMC_TYPE_DDR52_MAX);
1477         case bus_timing_mmc_hs200:
1478         case bus_timing_mmc_hs400:
1479         case bus_timing_mmc_hs400es:
1480                 return (MMC_TYPE_HS200_HS400ES_MAX);
1481         }
1482         return (0);
1483 }
1484
1485 static const char *
1486 mmc_timing_to_string(enum mmc_bus_timing timing)
1487 {
1488
1489         switch (timing) {
1490         case bus_timing_normal:
1491                 return ("normal speed");
1492         case bus_timing_hs:
1493                 return ("high speed");
1494         case bus_timing_uhs_sdr12:
1495         case bus_timing_uhs_sdr25:
1496         case bus_timing_uhs_sdr50:
1497         case bus_timing_uhs_sdr104:
1498                 return ("single data rate");
1499         case bus_timing_uhs_ddr50:
1500         case bus_timing_mmc_ddr52:
1501                 return ("dual data rate");
1502         case bus_timing_mmc_hs200:
1503                 return ("HS200");
1504         case bus_timing_mmc_hs400:
1505                 return ("HS400");
1506         case bus_timing_mmc_hs400es:
1507                 return ("HS400 with enhanced strobe");
1508         }
1509         return ("");
1510 }
1511
1512 static bool
1513 mmc_host_timing(device_t dev, enum mmc_bus_timing timing)
1514 {
1515         int host_caps;
1516
1517         host_caps = mmcbr_get_caps(dev);
1518
1519 #define HOST_TIMING_CAP(host_caps, cap) ({                              \
1520         bool retval;                                                    \
1521         if (((host_caps) & (cap)) == (cap))                             \
1522                 retval = true;                                          \
1523         else                                                            \
1524                 retval = false;                                         \
1525         retval;                                                         \
1526 })
1527
1528         switch (timing) {
1529         case bus_timing_normal:
1530                 return (true);
1531         case bus_timing_hs:
1532                 return (HOST_TIMING_CAP(host_caps, MMC_CAP_HSPEED));
1533         case bus_timing_uhs_sdr12:
1534                 return (HOST_TIMING_CAP(host_caps, MMC_CAP_UHS_SDR12));
1535         case bus_timing_uhs_sdr25:
1536                 return (HOST_TIMING_CAP(host_caps, MMC_CAP_UHS_SDR25));
1537         case bus_timing_uhs_ddr50:
1538                 return (HOST_TIMING_CAP(host_caps, MMC_CAP_UHS_DDR50));
1539         case bus_timing_uhs_sdr50:
1540                 return (HOST_TIMING_CAP(host_caps, MMC_CAP_UHS_SDR50));
1541         case bus_timing_uhs_sdr104:
1542                 return (HOST_TIMING_CAP(host_caps, MMC_CAP_UHS_SDR104));
1543         case bus_timing_mmc_ddr52:
1544                 return (HOST_TIMING_CAP(host_caps, MMC_CAP_MMC_DDR52));
1545         case bus_timing_mmc_hs200:
1546                 return (HOST_TIMING_CAP(host_caps, MMC_CAP_MMC_HS200));
1547         case bus_timing_mmc_hs400:
1548                 return (HOST_TIMING_CAP(host_caps, MMC_CAP_MMC_HS400));
1549         case bus_timing_mmc_hs400es:
1550                 return (HOST_TIMING_CAP(host_caps, MMC_CAP_MMC_HS400 |
1551                     MMC_CAP_MMC_ENH_STROBE));
1552         }
1553
1554 #undef HOST_TIMING_CAP
1555
1556         return (false);
1557 }
1558
1559 static void
1560 mmc_log_card(device_t dev, struct mmc_ivars *ivar, int newcard)
1561 {
1562         enum mmc_bus_timing timing;
1563
1564         device_printf(dev, "Card at relative address 0x%04x%s:\n",
1565             ivar->rca, newcard ? " added" : "");
1566         device_printf(dev, " card: %s\n", ivar->card_id_string);
1567         for (timing = bus_timing_max; timing > bus_timing_normal; timing--) {
1568                 if (isset(&ivar->timings, timing))
1569                         break;
1570         }
1571         device_printf(dev, " quirks: %b\n", ivar->quirks, MMC_QUIRKS_FMT);
1572         device_printf(dev, " bus: %ubit, %uMHz (%s timing)\n",
1573             (ivar->bus_width == bus_width_1 ? 1 :
1574             (ivar->bus_width == bus_width_4 ? 4 : 8)),
1575             mmc_timing_to_dtr(ivar, timing) / 1000000,
1576             mmc_timing_to_string(timing));
1577         device_printf(dev, " memory: %u blocks, erase sector %u blocks%s\n",
1578             ivar->sec_count, ivar->erase_sector,
1579             ivar->read_only ? ", read-only" : "");
1580 }
1581
1582 static void
1583 mmc_discover_cards(struct mmc_softc *sc)
1584 {
1585         u_char switch_res[64];
1586         uint32_t raw_cid[4];
1587         struct mmc_ivars *ivar = NULL;
1588         const struct mmc_quirk *quirk;
1589         device_t child;
1590         int err, host_caps, i, newcard;
1591         uint32_t resp, sec_count, status;
1592         uint16_t rca = 2;
1593
1594         host_caps = mmcbr_get_caps(sc->dev);
1595         if (bootverbose || mmc_debug)
1596                 device_printf(sc->dev, "Probing cards\n");
1597         while (1) {
1598                 child = NULL;
1599                 sc->squelched++; /* Errors are expected, squelch reporting. */
1600                 err = mmc_all_send_cid(sc, raw_cid);
1601                 sc->squelched--;
1602                 if (err == MMC_ERR_TIMEOUT)
1603                         break;
1604                 if (err != MMC_ERR_NONE) {
1605                         device_printf(sc->dev, "Error reading CID %d\n", err);
1606                         break;
1607                 }
1608                 newcard = 1;
1609                 for (i = 0; i < sc->child_count; i++) {
1610                         ivar = device_get_ivars(sc->child_list[i]);
1611                         if (memcmp(ivar->raw_cid, raw_cid, sizeof(raw_cid)) ==
1612                             0) {
1613                                 newcard = 0;
1614                                 break;
1615                         }
1616                 }
1617                 if (bootverbose || mmc_debug) {
1618                         device_printf(sc->dev,
1619                             "%sard detected (CID %08x%08x%08x%08x)\n",
1620                             newcard ? "New c" : "C",
1621                             raw_cid[0], raw_cid[1], raw_cid[2], raw_cid[3]);
1622                 }
1623                 if (newcard) {
1624                         ivar = malloc(sizeof(struct mmc_ivars), M_DEVBUF,
1625                             M_WAITOK | M_ZERO);
1626                         memcpy(ivar->raw_cid, raw_cid, sizeof(raw_cid));
1627                 }
1628                 if (mmcbr_get_ro(sc->dev))
1629                         ivar->read_only = 1;
1630                 ivar->bus_width = bus_width_1;
1631                 setbit(&ivar->timings, bus_timing_normal);
1632                 ivar->mode = mmcbr_get_mode(sc->dev);
1633                 if (ivar->mode == mode_sd) {
1634                         mmc_decode_cid_sd(ivar->raw_cid, &ivar->cid);
1635                         err = mmc_send_relative_addr(sc, &resp);
1636                         if (err != MMC_ERR_NONE) {
1637                                 device_printf(sc->dev,
1638                                     "Error getting RCA %d\n", err);
1639                                 goto free_ivar;
1640                         }
1641                         ivar->rca = resp >> 16;
1642                         /* Get card CSD. */
1643                         err = mmc_send_csd(sc, ivar->rca, ivar->raw_csd);
1644                         if (err != MMC_ERR_NONE) {
1645                                 device_printf(sc->dev,
1646                                     "Error getting CSD %d\n", err);
1647                                 goto free_ivar;
1648                         }
1649                         if (bootverbose || mmc_debug)
1650                                 device_printf(sc->dev,
1651                                     "%sard detected (CSD %08x%08x%08x%08x)\n",
1652                                     newcard ? "New c" : "C", ivar->raw_csd[0],
1653                                     ivar->raw_csd[1], ivar->raw_csd[2],
1654                                     ivar->raw_csd[3]);
1655                         err = mmc_decode_csd_sd(ivar->raw_csd, &ivar->csd);
1656                         if (err != MMC_ERR_NONE) {
1657                                 device_printf(sc->dev, "Error decoding CSD\n");
1658                                 goto free_ivar;
1659                         }
1660                         ivar->sec_count = ivar->csd.capacity / MMC_SECTOR_SIZE;
1661                         if (ivar->csd.csd_structure > 0)
1662                                 ivar->high_cap = 1;
1663                         ivar->tran_speed = ivar->csd.tran_speed;
1664                         ivar->erase_sector = ivar->csd.erase_sector *
1665                             ivar->csd.write_bl_len / MMC_SECTOR_SIZE;
1666
1667                         err = mmc_send_status(sc->dev, sc->dev, ivar->rca,
1668                             &status);
1669                         if (err != MMC_ERR_NONE) {
1670                                 device_printf(sc->dev,
1671                                     "Error reading card status %d\n", err);
1672                                 goto free_ivar;
1673                         }
1674                         if ((status & R1_CARD_IS_LOCKED) != 0) {
1675                                 device_printf(sc->dev,
1676                                     "Card is password protected, skipping\n");
1677                                 goto free_ivar;
1678                         }
1679
1680                         /* Get card SCR.  Card must be selected to fetch it. */
1681                         err = mmc_select_card(sc, ivar->rca);
1682                         if (err != MMC_ERR_NONE) {
1683                                 device_printf(sc->dev,
1684                                     "Error selecting card %d\n", err);
1685                                 goto free_ivar;
1686                         }
1687                         err = mmc_app_send_scr(sc, ivar->rca, ivar->raw_scr);
1688                         if (err != MMC_ERR_NONE) {
1689                                 device_printf(sc->dev,
1690                                     "Error reading SCR %d\n", err);
1691                                 goto free_ivar;
1692                         }
1693                         mmc_app_decode_scr(ivar->raw_scr, &ivar->scr);
1694                         /* Get card switch capabilities (command class 10). */
1695                         if ((ivar->scr.sda_vsn >= 1) &&
1696                             (ivar->csd.ccc & (1 << 10))) {
1697                                 err = mmc_sd_switch(sc, SD_SWITCH_MODE_CHECK,
1698                                     SD_SWITCH_GROUP1, SD_SWITCH_NOCHANGE,
1699                                     switch_res);
1700                                 if (err == MMC_ERR_NONE &&
1701                                     switch_res[13] & (1 << SD_SWITCH_HS_MODE)) {
1702                                         setbit(&ivar->timings, bus_timing_hs);
1703                                         ivar->hs_tran_speed = SD_HS_MAX;
1704                                 }
1705                         }
1706
1707                         /*
1708                          * We deselect then reselect the card here.  Some cards
1709                          * become unselected and timeout with the above two
1710                          * commands, although the state tables / diagrams in the
1711                          * standard suggest they go back to the transfer state.
1712                          * Other cards don't become deselected, and if we
1713                          * attempt to blindly re-select them, we get timeout
1714                          * errors from some controllers.  So we deselect then
1715                          * reselect to handle all situations.  The only thing we
1716                          * use from the sd_status is the erase sector size, but
1717                          * it is still nice to get that right.
1718                          */
1719                         (void)mmc_select_card(sc, 0);
1720                         (void)mmc_select_card(sc, ivar->rca);
1721                         (void)mmc_app_sd_status(sc, ivar->rca,
1722                             ivar->raw_sd_status);
1723                         mmc_app_decode_sd_status(ivar->raw_sd_status,
1724                             &ivar->sd_status);
1725                         if (ivar->sd_status.au_size != 0) {
1726                                 ivar->erase_sector =
1727                                     16 << ivar->sd_status.au_size;
1728                         }
1729                         /* Find maximum supported bus width. */
1730                         if ((host_caps & MMC_CAP_4_BIT_DATA) &&
1731                             (ivar->scr.bus_widths & SD_SCR_BUS_WIDTH_4))
1732                                 ivar->bus_width = bus_width_4;
1733
1734                         goto child_common;
1735                 }
1736                 ivar->rca = rca++;
1737                 err = mmc_set_relative_addr(sc, ivar->rca);
1738                 if (err != MMC_ERR_NONE) {
1739                         device_printf(sc->dev, "Error setting RCA %d\n", err);
1740                         goto free_ivar;
1741                 }
1742                 /* Get card CSD. */
1743                 err = mmc_send_csd(sc, ivar->rca, ivar->raw_csd);
1744                 if (err != MMC_ERR_NONE) {
1745                         device_printf(sc->dev, "Error getting CSD %d\n", err);
1746                         goto free_ivar;
1747                 }
1748                 if (bootverbose || mmc_debug)
1749                         device_printf(sc->dev,
1750                             "%sard detected (CSD %08x%08x%08x%08x)\n",
1751                             newcard ? "New c" : "C", ivar->raw_csd[0],
1752                             ivar->raw_csd[1], ivar->raw_csd[2],
1753                             ivar->raw_csd[3]);
1754
1755                 mmc_decode_csd_mmc(ivar->raw_csd, &ivar->csd);
1756                 ivar->sec_count = ivar->csd.capacity / MMC_SECTOR_SIZE;
1757                 ivar->tran_speed = ivar->csd.tran_speed;
1758                 ivar->erase_sector = ivar->csd.erase_sector *
1759                     ivar->csd.write_bl_len / MMC_SECTOR_SIZE;
1760
1761                 err = mmc_send_status(sc->dev, sc->dev, ivar->rca, &status);
1762                 if (err != MMC_ERR_NONE) {
1763                         device_printf(sc->dev,
1764                             "Error reading card status %d\n", err);
1765                         goto free_ivar;
1766                 }
1767                 if ((status & R1_CARD_IS_LOCKED) != 0) {
1768                         device_printf(sc->dev,
1769                             "Card is password protected, skipping\n");
1770                         goto free_ivar;
1771                 }
1772
1773                 err = mmc_select_card(sc, ivar->rca);
1774                 if (err != MMC_ERR_NONE) {
1775                         device_printf(sc->dev, "Error selecting card %d\n",
1776                             err);
1777                         goto free_ivar;
1778                 }
1779
1780                 /* Only MMC >= 4.x devices support EXT_CSD. */
1781                 if (ivar->csd.spec_vers >= 4) {
1782                         err = mmc_send_ext_csd(sc->dev, sc->dev,
1783                             ivar->raw_ext_csd);
1784                         if (err != MMC_ERR_NONE) {
1785                                 device_printf(sc->dev,
1786                                     "Error reading EXT_CSD %d\n", err);
1787                                 goto free_ivar;
1788                         }
1789                         /* Handle extended capacity from EXT_CSD */
1790                         sec_count = ivar->raw_ext_csd[EXT_CSD_SEC_CNT] +
1791                             (ivar->raw_ext_csd[EXT_CSD_SEC_CNT + 1] << 8) +
1792                             (ivar->raw_ext_csd[EXT_CSD_SEC_CNT + 2] << 16) +
1793                             (ivar->raw_ext_csd[EXT_CSD_SEC_CNT + 3] << 24);
1794                         if (sec_count != 0) {
1795                                 ivar->sec_count = sec_count;
1796                                 ivar->high_cap = 1;
1797                         }
1798                         /* Find maximum supported bus width. */
1799                         ivar->bus_width = mmc_test_bus_width(sc);
1800                         /* Get device speeds beyond normal mode. */
1801                         if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] &
1802                             EXT_CSD_CARD_TYPE_HS_52) != 0) {
1803                                 setbit(&ivar->timings, bus_timing_hs);
1804                                 ivar->hs_tran_speed = MMC_TYPE_HS_52_MAX;
1805                         } else if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] &
1806                             EXT_CSD_CARD_TYPE_HS_26) != 0) {
1807                                 setbit(&ivar->timings, bus_timing_hs);
1808                                 ivar->hs_tran_speed = MMC_TYPE_HS_26_MAX;
1809                         }
1810                         if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] &
1811                             EXT_CSD_CARD_TYPE_DDR_52_1_2V) != 0 &&
1812                             (host_caps & MMC_CAP_SIGNALING_120) != 0) {
1813                                 setbit(&ivar->timings, bus_timing_mmc_ddr52);
1814                                 setbit(&ivar->vccq_120, bus_timing_mmc_ddr52);
1815                         }
1816                         if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] &
1817                             EXT_CSD_CARD_TYPE_DDR_52_1_8V) != 0 &&
1818                             (host_caps & MMC_CAP_SIGNALING_180) != 0) {
1819                                 setbit(&ivar->timings, bus_timing_mmc_ddr52);
1820                                 setbit(&ivar->vccq_180, bus_timing_mmc_ddr52);
1821                         }
1822                         if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] &
1823                             EXT_CSD_CARD_TYPE_HS200_1_2V) != 0 &&
1824                             (host_caps & MMC_CAP_SIGNALING_120) != 0) {
1825                                 setbit(&ivar->timings, bus_timing_mmc_hs200);
1826                                 setbit(&ivar->vccq_120, bus_timing_mmc_hs200);
1827                         }
1828                         if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] &
1829                             EXT_CSD_CARD_TYPE_HS200_1_8V) != 0 &&
1830                             (host_caps & MMC_CAP_SIGNALING_180) != 0) {
1831                                 setbit(&ivar->timings, bus_timing_mmc_hs200);
1832                                 setbit(&ivar->vccq_180, bus_timing_mmc_hs200);
1833                         }
1834                         if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] &
1835                             EXT_CSD_CARD_TYPE_HS400_1_2V) != 0 &&
1836                             (host_caps & MMC_CAP_SIGNALING_120) != 0 &&
1837                             ivar->bus_width == bus_width_8) {
1838                                 setbit(&ivar->timings, bus_timing_mmc_hs400);
1839                                 setbit(&ivar->vccq_120, bus_timing_mmc_hs400);
1840                         }
1841                         if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] &
1842                             EXT_CSD_CARD_TYPE_HS400_1_8V) != 0 &&
1843                             (host_caps & MMC_CAP_SIGNALING_180) != 0 &&
1844                             ivar->bus_width == bus_width_8) {
1845                                 setbit(&ivar->timings, bus_timing_mmc_hs400);
1846                                 setbit(&ivar->vccq_180, bus_timing_mmc_hs400);
1847                         }
1848                         if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] &
1849                             EXT_CSD_CARD_TYPE_HS400_1_2V) != 0 &&
1850                             (ivar->raw_ext_csd[EXT_CSD_STROBE_SUPPORT] &
1851                             EXT_CSD_STROBE_SUPPORT_EN) != 0 &&
1852                             (host_caps & MMC_CAP_SIGNALING_120) != 0 &&
1853                             ivar->bus_width == bus_width_8) {
1854                                 setbit(&ivar->timings, bus_timing_mmc_hs400es);
1855                                 setbit(&ivar->vccq_120, bus_timing_mmc_hs400es);
1856                         }
1857                         if ((ivar->raw_ext_csd[EXT_CSD_CARD_TYPE] &
1858                             EXT_CSD_CARD_TYPE_HS400_1_8V) != 0 &&
1859                             (ivar->raw_ext_csd[EXT_CSD_STROBE_SUPPORT] &
1860                             EXT_CSD_STROBE_SUPPORT_EN) != 0 &&
1861                             (host_caps & MMC_CAP_SIGNALING_180) != 0 &&
1862                             ivar->bus_width == bus_width_8) {
1863                                 setbit(&ivar->timings, bus_timing_mmc_hs400es);
1864                                 setbit(&ivar->vccq_180, bus_timing_mmc_hs400es);
1865                         }
1866                         /*
1867                          * Determine generic switch timeout (provided in
1868                          * units of 10 ms), defaulting to 500 ms.
1869                          */
1870                         ivar->cmd6_time = 500 * 1000;
1871                         if (ivar->csd.spec_vers >= 6)
1872                                 ivar->cmd6_time = 10 *
1873                                     ivar->raw_ext_csd[EXT_CSD_GEN_CMD6_TIME];
1874                         /* Handle HC erase sector size. */
1875                         if (ivar->raw_ext_csd[EXT_CSD_ERASE_GRP_SIZE] != 0) {
1876                                 ivar->erase_sector = 1024 *
1877                                     ivar->raw_ext_csd[EXT_CSD_ERASE_GRP_SIZE];
1878                                 err = mmc_switch(sc->dev, sc->dev, ivar->rca,
1879                                     EXT_CSD_CMD_SET_NORMAL,
1880                                     EXT_CSD_ERASE_GRP_DEF,
1881                                     EXT_CSD_ERASE_GRP_DEF_EN,
1882                                     ivar->cmd6_time, true);
1883                                 if (err != MMC_ERR_NONE) {
1884                                         device_printf(sc->dev,
1885                                             "Error setting erase group %d\n",
1886                                             err);
1887                                         goto free_ivar;
1888                                 }
1889                         }
1890                 }
1891
1892                 mmc_decode_cid_mmc(ivar->raw_cid, &ivar->cid,
1893                     ivar->raw_ext_csd[EXT_CSD_REV] >= 5);
1894
1895 child_common:
1896                 for (quirk = &mmc_quirks[0]; quirk->mid != 0x0; quirk++) {
1897                         if ((quirk->mid == MMC_QUIRK_MID_ANY ||
1898                             quirk->mid == ivar->cid.mid) &&
1899                             (quirk->oid == MMC_QUIRK_OID_ANY ||
1900                             quirk->oid == ivar->cid.oid) &&
1901                             strncmp(quirk->pnm, ivar->cid.pnm,
1902                             sizeof(ivar->cid.pnm)) == 0) {
1903                                 ivar->quirks = quirk->quirks;
1904                                 break;
1905                         }
1906                 }
1907
1908                 /*
1909                  * Some cards that report maximum I/O block sizes greater
1910                  * than 512 require the block length to be set to 512, even
1911                  * though that is supposed to be the default.  Example:
1912                  *
1913                  * Transcend 2GB SDSC card, CID:
1914                  * mid=0x1b oid=0x534d pnm="00000" prv=1.0 mdt=00.2000
1915                  */
1916                 if (ivar->csd.read_bl_len != MMC_SECTOR_SIZE ||
1917                     ivar->csd.write_bl_len != MMC_SECTOR_SIZE)
1918                         mmc_set_blocklen(sc, MMC_SECTOR_SIZE);
1919
1920                 mmc_format_card_id_string(ivar);
1921
1922                 if (bootverbose || mmc_debug)
1923                         mmc_log_card(sc->dev, ivar, newcard);
1924                 if (newcard) {
1925                         /* Add device. */
1926                         child = device_add_child(sc->dev, NULL, -1);
1927                         if (child != NULL) {
1928                                 device_set_ivars(child, ivar);
1929                                 sc->child_list = realloc(sc->child_list,
1930                                     sizeof(device_t) * sc->child_count + 1,
1931                                     M_DEVBUF, M_WAITOK);
1932                                 sc->child_list[sc->child_count++] = child;
1933                         } else
1934                                 device_printf(sc->dev, "Error adding child\n");
1935                 }
1936
1937 free_ivar:
1938                 if (newcard && child == NULL)
1939                         free(ivar, M_DEVBUF);
1940                 (void)mmc_select_card(sc, 0);
1941                 /*
1942                  * Not returning here when one MMC device could no be added
1943                  * potentially would mean looping forever when that device
1944                  * is broken (in which case it also may impact the remainder
1945                  * of the bus anyway, though).
1946                  */
1947                 if ((newcard && child == NULL) ||
1948                     mmcbr_get_mode(sc->dev) == mode_sd)
1949                         return;
1950         }
1951 }
1952
1953 static void
1954 mmc_update_child_list(struct mmc_softc *sc)
1955 {
1956         device_t child;
1957         int i, j;
1958
1959         if (sc->child_count == 0) {
1960                 free(sc->child_list, M_DEVBUF);
1961                 return;
1962         }
1963         for (i = j = 0; i < sc->child_count; i++) {
1964                 for (;;) {
1965                         child = sc->child_list[j++];
1966                         if (child != NULL)
1967                                 break;
1968                 }
1969                 if (i != j)
1970                         sc->child_list[i] = child;
1971         }
1972         sc->child_list = realloc(sc->child_list, sizeof(device_t) *
1973             sc->child_count, M_DEVBUF, M_WAITOK);
1974 }
1975
1976 static void
1977 mmc_rescan_cards(struct mmc_softc *sc)
1978 {
1979         struct mmc_ivars *ivar;
1980         int err, i, j;
1981
1982         for (i = j = 0; i < sc->child_count; i++) {
1983                 ivar = device_get_ivars(sc->child_list[i]);
1984                 if (mmc_select_card(sc, ivar->rca) != MMC_ERR_NONE) {
1985                         if (bootverbose || mmc_debug)
1986                                 device_printf(sc->dev,
1987                                     "Card at relative address %d lost\n",
1988                                     ivar->rca);
1989                         err = device_delete_child(sc->dev, sc->child_list[i]);
1990                         if (err != 0) {
1991                                 j++;
1992                                 continue;
1993                         }
1994                         free(ivar, M_DEVBUF);
1995                 } else
1996                         j++;
1997         }
1998         if (sc->child_count == j)
1999                 goto out;
2000         sc->child_count = j;
2001         mmc_update_child_list(sc);
2002 out:
2003         (void)mmc_select_card(sc, 0);
2004 }
2005
2006 static int
2007 mmc_delete_cards(struct mmc_softc *sc, bool final)
2008 {
2009         struct mmc_ivars *ivar;
2010         int err, i, j;
2011
2012         err = 0;
2013         for (i = j = 0; i < sc->child_count; i++) {
2014                 ivar = device_get_ivars(sc->child_list[i]);
2015                 if (bootverbose || mmc_debug)
2016                         device_printf(sc->dev,
2017                             "Card at relative address %d deleted\n",
2018                             ivar->rca);
2019                 err = device_delete_child(sc->dev, sc->child_list[i]);
2020                 if (err != 0) {
2021                         j++;
2022                         if (final == false)
2023                                 continue;
2024                         else
2025                                 break;
2026                 }
2027                 free(ivar, M_DEVBUF);
2028         }
2029         sc->child_count = j;
2030         mmc_update_child_list(sc);
2031         return (err);
2032 }
2033
2034 static void
2035 mmc_go_discovery(struct mmc_softc *sc)
2036 {
2037         uint32_t ocr;
2038         device_t dev;
2039         int err;
2040
2041         dev = sc->dev;
2042         if (mmcbr_get_power_mode(dev) != power_on) {
2043                 /*
2044                  * First, try SD modes
2045                  */
2046                 sc->squelched++; /* Errors are expected, squelch reporting. */
2047                 mmcbr_set_mode(dev, mode_sd);
2048                 mmc_power_up(sc);
2049                 mmcbr_set_bus_mode(dev, pushpull);
2050                 if (bootverbose || mmc_debug)
2051                         device_printf(sc->dev, "Probing bus\n");
2052                 mmc_idle_cards(sc);
2053                 err = mmc_send_if_cond(sc, 1);
2054                 if ((bootverbose || mmc_debug) && err == 0)
2055                         device_printf(sc->dev,
2056                             "SD 2.0 interface conditions: OK\n");
2057                 if (mmc_send_app_op_cond(sc, 0, &ocr) != MMC_ERR_NONE) {
2058                         if (bootverbose || mmc_debug)
2059                                 device_printf(sc->dev, "SD probe: failed\n");
2060                         /*
2061                          * Failed, try MMC
2062                          */
2063                         mmcbr_set_mode(dev, mode_mmc);
2064                         if (mmc_send_op_cond(sc, 0, &ocr) != MMC_ERR_NONE) {
2065                                 if (bootverbose || mmc_debug)
2066                                         device_printf(sc->dev,
2067                                             "MMC probe: failed\n");
2068                                 ocr = 0; /* Failed both, powerdown. */
2069                         } else if (bootverbose || mmc_debug)
2070                                 device_printf(sc->dev,
2071                                     "MMC probe: OK (OCR: 0x%08x)\n", ocr);
2072                 } else if (bootverbose || mmc_debug)
2073                         device_printf(sc->dev, "SD probe: OK (OCR: 0x%08x)\n",
2074                             ocr);
2075                 sc->squelched--;
2076
2077                 mmcbr_set_ocr(dev, mmc_select_vdd(sc, ocr));
2078                 if (mmcbr_get_ocr(dev) != 0)
2079                         mmc_idle_cards(sc);
2080         } else {
2081                 mmcbr_set_bus_mode(dev, opendrain);
2082                 mmcbr_set_clock(dev, SD_MMC_CARD_ID_FREQUENCY);
2083                 mmcbr_update_ios(dev);
2084                 /* XXX recompute vdd based on new cards? */
2085         }
2086         /*
2087          * Make sure that we have a mutually agreeable voltage to at least
2088          * one card on the bus.
2089          */
2090         if (bootverbose || mmc_debug)
2091                 device_printf(sc->dev, "Current OCR: 0x%08x\n",
2092                     mmcbr_get_ocr(dev));
2093         if (mmcbr_get_ocr(dev) == 0) {
2094                 device_printf(sc->dev, "No compatible cards found on bus\n");
2095                 (void)mmc_delete_cards(sc, false);
2096                 mmc_power_down(sc);
2097                 return;
2098         }
2099         /*
2100          * Reselect the cards after we've idled them above.
2101          */
2102         if (mmcbr_get_mode(dev) == mode_sd) {
2103                 err = mmc_send_if_cond(sc, 1);
2104                 mmc_send_app_op_cond(sc,
2105                     (err ? 0 : MMC_OCR_CCS) | mmcbr_get_ocr(dev), NULL);
2106         } else
2107                 mmc_send_op_cond(sc, MMC_OCR_CCS | mmcbr_get_ocr(dev), NULL);
2108         mmc_discover_cards(sc);
2109         mmc_rescan_cards(sc);
2110
2111         mmcbr_set_bus_mode(dev, pushpull);
2112         mmcbr_update_ios(dev);
2113         mmc_calculate_clock(sc);
2114 }
2115
2116 static int
2117 mmc_calculate_clock(struct mmc_softc *sc)
2118 {
2119         device_t dev;
2120         struct mmc_ivars *ivar;
2121         int i;
2122         uint32_t dtr, max_dtr;
2123         uint16_t rca;
2124         enum mmc_bus_timing max_timing, timing;
2125         bool changed, hs400;
2126
2127         dev = sc->dev;
2128         max_dtr = mmcbr_get_f_max(dev);
2129         max_timing = bus_timing_max;
2130         do {
2131                 changed = false;
2132                 for (i = 0; i < sc->child_count; i++) {
2133                         ivar = device_get_ivars(sc->child_list[i]);
2134                         if (isclr(&ivar->timings, max_timing) ||
2135                             !mmc_host_timing(dev, max_timing)) {
2136                                 for (timing = max_timing - 1; timing >=
2137                                     bus_timing_normal; timing--) {
2138                                         if (isset(&ivar->timings, timing) &&
2139                                             mmc_host_timing(dev, timing)) {
2140                                                 max_timing = timing;
2141                                                 break;
2142                                         }
2143                                 }
2144                                 changed = true;
2145                         }
2146                         dtr = mmc_timing_to_dtr(ivar, max_timing);
2147                         if (dtr < max_dtr) {
2148                                 max_dtr = dtr;
2149                                 changed = true;
2150                         }
2151                 }
2152         } while (changed == true);
2153
2154         if (bootverbose || mmc_debug) {
2155                 device_printf(dev,
2156                     "setting transfer rate to %d.%03dMHz (%s timing)\n",
2157                     max_dtr / 1000000, (max_dtr / 1000) % 1000,
2158                     mmc_timing_to_string(max_timing));
2159         }
2160
2161         /*
2162          * HS400 must be tuned in HS200 mode, so in case of HS400 we begin
2163          * with HS200 following the sequence as described in "6.6.2.2 HS200
2164          * timing mode selection" of the eMMC specification v5.1, too, and
2165          * switch to max_timing later.  HS400ES requires no tuning and, thus,
2166          * can be switch to directly, but requires the same detour via high
2167          * speed mode as does HS400 (see mmc_switch_to_hs400()).
2168          */
2169         hs400 = max_timing == bus_timing_mmc_hs400;
2170         timing = hs400 == true ? bus_timing_mmc_hs200 : max_timing;
2171         for (i = 0; i < sc->child_count; i++) {
2172                 ivar = device_get_ivars(sc->child_list[i]);
2173                 if ((ivar->timings & ~(1 << bus_timing_normal)) == 0)
2174                         continue;
2175
2176                 rca = ivar->rca;
2177                 if (mmc_select_card(sc, rca) != MMC_ERR_NONE) {
2178                         device_printf(dev, "Card at relative address %d "
2179                             "failed to select\n", rca);
2180                         continue;
2181                 }
2182
2183                 if (timing == bus_timing_mmc_hs200 ||   /* includes HS400 */
2184                     timing == bus_timing_mmc_hs400es) {
2185                         if (mmc_set_vccq(sc, ivar, timing) != MMC_ERR_NONE) {
2186                                 device_printf(dev, "Failed to set VCCQ for "
2187                                     "card at relative address %d\n", rca);
2188                                 continue;
2189                         }
2190                 }
2191
2192                 if (timing == bus_timing_mmc_hs200) {   /* includes HS400 */
2193                         /* Set bus width (required for initial tuning). */
2194                         if (mmc_set_card_bus_width(sc, ivar, timing) !=
2195                             MMC_ERR_NONE) {
2196                                 device_printf(dev, "Card at relative address "
2197                                     "%d failed to set bus width\n", rca);
2198                                 continue;
2199                         }
2200                         mmcbr_set_bus_width(dev, ivar->bus_width);
2201                         mmcbr_update_ios(dev);
2202                 } else if (timing == bus_timing_mmc_hs400es) {
2203                         if (mmc_switch_to_hs400(sc, ivar, max_dtr, timing) !=
2204                             MMC_ERR_NONE) {
2205                                 device_printf(dev, "Card at relative address "
2206                                     "%d failed to set %s timing\n", rca,
2207                                     mmc_timing_to_string(timing));
2208                                 continue;
2209                         }
2210                         goto power_class;
2211                 }
2212
2213                 if (mmc_set_timing(sc, ivar, timing) != MMC_ERR_NONE) {
2214                         device_printf(dev, "Card at relative address %d "
2215                             "failed to set %s timing\n", rca,
2216                             mmc_timing_to_string(timing));
2217                         continue;
2218                 }
2219
2220                 if (timing == bus_timing_mmc_ddr52) {
2221                         /*
2222                          * Set EXT_CSD_BUS_WIDTH_n_DDR in EXT_CSD_BUS_WIDTH
2223                          * (must be done after switching to EXT_CSD_HS_TIMING).
2224                          */
2225                         if (mmc_set_card_bus_width(sc, ivar, timing) !=
2226                             MMC_ERR_NONE) {
2227                                 device_printf(dev, "Card at relative address "
2228                                     "%d failed to set bus width\n", rca);
2229                                 continue;
2230                         }
2231                         mmcbr_set_bus_width(dev, ivar->bus_width);
2232                         mmcbr_update_ios(dev);
2233                         if (mmc_set_vccq(sc, ivar, timing) != MMC_ERR_NONE) {
2234                                 device_printf(dev, "Failed to set VCCQ for "
2235                                     "card at relative address %d\n", rca);
2236                                 continue;
2237                         }
2238                 }
2239
2240                 /* Set clock (must be done before initial tuning). */
2241                 mmcbr_set_clock(dev, max_dtr);
2242                 mmcbr_update_ios(dev);
2243
2244                 if (mmcbr_tune(dev, hs400) != 0) {
2245                         device_printf(dev, "Card at relative address %d "
2246                             "failed to execute initial tuning\n", rca);
2247                         continue;
2248                 }
2249
2250                 if (hs400 == true && mmc_switch_to_hs400(sc, ivar, max_dtr,
2251                     max_timing) != MMC_ERR_NONE) {
2252                         device_printf(dev, "Card at relative address %d "
2253                             "failed to set %s timing\n", rca,
2254                             mmc_timing_to_string(max_timing));
2255                         continue;
2256                 }
2257
2258 power_class:
2259                 if (mmc_set_power_class(sc, ivar) != MMC_ERR_NONE) {
2260                         device_printf(dev, "Card at relative address %d "
2261                             "failed to set power class\n", rca);
2262                 }
2263         }
2264         (void)mmc_select_card(sc, 0);
2265         return (max_dtr);
2266 }
2267
2268 /*
2269  * Switch from HS200 to HS400 (either initially or for re-tuning) or directly
2270  * to HS400ES.  This follows the sequences described in "6.6.2.3 HS400 timing
2271  * mode selection" of the eMMC specification v5.1.
2272  */
2273 static int
2274 mmc_switch_to_hs400(struct mmc_softc *sc, struct mmc_ivars *ivar,
2275     uint32_t clock, enum mmc_bus_timing max_timing)
2276 {
2277         device_t dev;
2278         int err;
2279         uint16_t rca;
2280
2281         dev = sc->dev;
2282         rca = ivar->rca;
2283
2284         /*
2285          * Both clock and timing must be set as appropriate for high speed
2286          * before eventually switching to HS400/HS400ES; mmc_set_timing()
2287          * will issue mmcbr_update_ios().
2288          */
2289         mmcbr_set_clock(dev, ivar->hs_tran_speed);
2290         err = mmc_set_timing(sc, ivar, bus_timing_hs);
2291         if (err != MMC_ERR_NONE)
2292                 return (err);
2293
2294         /*
2295          * Set EXT_CSD_BUS_WIDTH_8_DDR in EXT_CSD_BUS_WIDTH (and additionally
2296          * EXT_CSD_BUS_WIDTH_ES for HS400ES).
2297          */
2298         err = mmc_set_card_bus_width(sc, ivar, max_timing);
2299         if (err != MMC_ERR_NONE)
2300                 return (err);
2301         mmcbr_set_bus_width(dev, ivar->bus_width);
2302         mmcbr_update_ios(dev);
2303
2304         /* Finally, switch to HS400/HS400ES mode. */
2305         err = mmc_set_timing(sc, ivar, max_timing);
2306         if (err != MMC_ERR_NONE)
2307                 return (err);
2308         mmcbr_set_clock(dev, clock);
2309         mmcbr_update_ios(dev);
2310         return (MMC_ERR_NONE);
2311 }
2312
2313 /*
2314  * Switch from HS400 to HS200 (for re-tuning).
2315  */
2316 static int
2317 mmc_switch_to_hs200(struct mmc_softc *sc, struct mmc_ivars *ivar,
2318     uint32_t clock)
2319 {
2320         device_t dev;
2321         int err;
2322         uint16_t rca;
2323
2324         dev = sc->dev;
2325         rca = ivar->rca;
2326
2327         /*
2328          * Both clock and timing must initially be set as appropriate for
2329          * DDR52 before eventually switching to HS200; mmc_set_timing()
2330          * will issue mmcbr_update_ios().
2331          */
2332         mmcbr_set_clock(dev, ivar->hs_tran_speed);
2333         err = mmc_set_timing(sc, ivar, bus_timing_mmc_ddr52);
2334         if (err != MMC_ERR_NONE)
2335                 return (err);
2336
2337         /*
2338          * Next, switch to high speed.  Thus, clear EXT_CSD_BUS_WIDTH_n_DDR
2339          * in EXT_CSD_BUS_WIDTH and update bus width and timing in ios.
2340          */
2341         err = mmc_set_card_bus_width(sc, ivar, bus_timing_hs);
2342         if (err != MMC_ERR_NONE)
2343                 return (err);
2344         mmcbr_set_bus_width(dev, ivar->bus_width);
2345         mmcbr_set_timing(sc->dev, bus_timing_hs);
2346         mmcbr_update_ios(dev);
2347
2348         /* Finally, switch to HS200 mode. */
2349         err = mmc_set_timing(sc, ivar, bus_timing_mmc_hs200);
2350         if (err != MMC_ERR_NONE)
2351                 return (err);
2352         mmcbr_set_clock(dev, clock);
2353         mmcbr_update_ios(dev);
2354         return (MMC_ERR_NONE);
2355 }
2356
2357 static int
2358 mmc_retune(device_t busdev, device_t dev, bool reset)
2359 {
2360         struct mmc_softc *sc;
2361         struct mmc_ivars *ivar;
2362         int err;
2363         uint32_t clock;
2364         enum mmc_bus_timing timing;
2365
2366         if (device_get_parent(dev) != busdev)
2367                 return (MMC_ERR_INVALID);
2368
2369         sc = device_get_softc(busdev);
2370         if (sc->retune_needed != 1 && sc->retune_paused != 0)
2371                 return (MMC_ERR_INVALID);
2372
2373         timing = mmcbr_get_timing(busdev);
2374         if (timing == bus_timing_mmc_hs400) {
2375                 /*
2376                  * Controllers use the data strobe line to latch data from
2377                  * the devices in HS400 mode so periodic re-tuning isn't
2378                  * expected to be required, i. e. only if a CRC or tuning
2379                  * error is signaled to the bridge.  In these latter cases
2380                  * we are asked to reset the tuning circuit and need to do
2381                  * the switch timing dance.
2382                  */
2383                 if (reset == false)
2384                         return (0);
2385                 ivar = device_get_ivars(dev);
2386                 clock = mmcbr_get_clock(busdev);
2387                 if (mmc_switch_to_hs200(sc, ivar, clock) != MMC_ERR_NONE)
2388                         return (MMC_ERR_BADCRC);
2389         }
2390         err = mmcbr_retune(busdev, reset);
2391         if (err != 0 && timing == bus_timing_mmc_hs400)
2392                 return (MMC_ERR_BADCRC);
2393         switch (err) {
2394         case 0:
2395                 break;
2396         case EIO:
2397                 return (MMC_ERR_FAILED);
2398         default:
2399                 return (MMC_ERR_INVALID);
2400         }
2401         if (timing == bus_timing_mmc_hs400) {
2402                 if (mmc_switch_to_hs400(sc, ivar, clock, timing) !=
2403                     MMC_ERR_NONE)
2404                         return (MMC_ERR_BADCRC);
2405         }
2406         return (MMC_ERR_NONE);
2407 }
2408
2409 static void
2410 mmc_retune_pause(device_t busdev, device_t dev, bool retune)
2411 {
2412         struct mmc_softc *sc;
2413
2414         sc = device_get_softc(busdev);
2415         KASSERT(device_get_parent(dev) == busdev,
2416             ("%s: %s is not a child of %s", __func__, device_get_nameunit(dev),
2417             device_get_nameunit(busdev)));
2418         KASSERT(sc->owner != NULL,
2419             ("%s: Request from %s without bus being acquired.", __func__,
2420             device_get_nameunit(dev)));
2421
2422         if (retune == true && sc->retune_paused == 0)
2423                 sc->retune_needed = 1;
2424         sc->retune_paused++;
2425 }
2426
2427 static void
2428 mmc_retune_unpause(device_t busdev, device_t dev)
2429 {
2430         struct mmc_softc *sc;
2431
2432         sc = device_get_softc(busdev);
2433         KASSERT(device_get_parent(dev) == busdev,
2434             ("%s: %s is not a child of %s", __func__, device_get_nameunit(dev),
2435             device_get_nameunit(busdev)));
2436         KASSERT(sc->owner != NULL,
2437             ("%s: Request from %s without bus being acquired.", __func__,
2438             device_get_nameunit(dev)));
2439         KASSERT(sc->retune_paused != 0,
2440             ("%s: Re-tune pause count already at 0", __func__));
2441
2442         sc->retune_paused--;
2443 }
2444
2445 static void
2446 mmc_scan(struct mmc_softc *sc)
2447 {
2448         device_t dev = sc->dev;
2449         int err;
2450
2451         err = mmc_acquire_bus(dev, dev);
2452         if (err != 0) {
2453                 device_printf(dev, "Failed to acquire bus for scanning\n");
2454                 return;
2455         }
2456         mmc_go_discovery(sc);
2457         err = mmc_release_bus(dev, dev);
2458         if (err != 0) {
2459                 device_printf(dev, "Failed to release bus after scanning\n");
2460                 return;
2461         }
2462         (void)bus_generic_attach(dev);
2463 }
2464
2465 static int
2466 mmc_read_ivar(device_t bus, device_t child, int which, uintptr_t *result)
2467 {
2468         struct mmc_ivars *ivar = device_get_ivars(child);
2469
2470         switch (which) {
2471         default:
2472                 return (EINVAL);
2473         case MMC_IVAR_SPEC_VERS:
2474                 *result = ivar->csd.spec_vers;
2475                 break;
2476         case MMC_IVAR_DSR_IMP:
2477                 *result = ivar->csd.dsr_imp;
2478                 break;
2479         case MMC_IVAR_MEDIA_SIZE:
2480                 *result = ivar->sec_count;
2481                 break;
2482         case MMC_IVAR_RCA:
2483                 *result = ivar->rca;
2484                 break;
2485         case MMC_IVAR_SECTOR_SIZE:
2486                 *result = MMC_SECTOR_SIZE;
2487                 break;
2488         case MMC_IVAR_TRAN_SPEED:
2489                 *result = mmcbr_get_clock(bus);
2490                 break;
2491         case MMC_IVAR_READ_ONLY:
2492                 *result = ivar->read_only;
2493                 break;
2494         case MMC_IVAR_HIGH_CAP:
2495                 *result = ivar->high_cap;
2496                 break;
2497         case MMC_IVAR_CARD_TYPE:
2498                 *result = ivar->mode;
2499                 break;
2500         case MMC_IVAR_BUS_WIDTH:
2501                 *result = ivar->bus_width;
2502                 break;
2503         case MMC_IVAR_ERASE_SECTOR:
2504                 *result = ivar->erase_sector;
2505                 break;
2506         case MMC_IVAR_MAX_DATA:
2507                 *result = mmcbr_get_max_data(bus);
2508                 break;
2509         case MMC_IVAR_CMD6_TIMEOUT:
2510                 *result = ivar->cmd6_time;
2511                 break;
2512         case MMC_IVAR_QUIRKS:
2513                 *result = ivar->quirks;
2514                 break;
2515         case MMC_IVAR_CARD_ID_STRING:
2516                 *(char **)result = ivar->card_id_string;
2517                 break;
2518         case MMC_IVAR_CARD_SN_STRING:
2519                 *(char **)result = ivar->card_sn_string;
2520                 break;
2521         }
2522         return (0);
2523 }
2524
2525 static int
2526 mmc_write_ivar(device_t bus, device_t child, int which, uintptr_t value)
2527 {
2528
2529         /*
2530          * None are writable ATM
2531          */
2532         return (EINVAL);
2533 }
2534
2535 static void
2536 mmc_delayed_attach(void *xsc)
2537 {
2538         struct mmc_softc *sc = xsc;
2539
2540         mmc_scan(sc);
2541         config_intrhook_disestablish(&sc->config_intrhook);
2542 }
2543
2544 static int
2545 mmc_child_location_str(device_t dev, device_t child, char *buf,
2546     size_t buflen)
2547 {
2548
2549         snprintf(buf, buflen, "rca=0x%04x", mmc_get_rca(child));
2550         return (0);
2551 }
2552
2553 static device_method_t mmc_methods[] = {
2554         /* device_if */
2555         DEVMETHOD(device_probe, mmc_probe),
2556         DEVMETHOD(device_attach, mmc_attach),
2557         DEVMETHOD(device_detach, mmc_detach),
2558         DEVMETHOD(device_suspend, mmc_suspend),
2559         DEVMETHOD(device_resume, mmc_resume),
2560
2561         /* Bus interface */
2562         DEVMETHOD(bus_read_ivar, mmc_read_ivar),
2563         DEVMETHOD(bus_write_ivar, mmc_write_ivar),
2564         DEVMETHOD(bus_child_location_str, mmc_child_location_str),
2565
2566         /* MMC Bus interface */
2567         DEVMETHOD(mmcbus_retune_pause, mmc_retune_pause),
2568         DEVMETHOD(mmcbus_retune_unpause, mmc_retune_unpause),
2569         DEVMETHOD(mmcbus_wait_for_request, mmc_wait_for_request),
2570         DEVMETHOD(mmcbus_acquire_bus, mmc_acquire_bus),
2571         DEVMETHOD(mmcbus_release_bus, mmc_release_bus),
2572
2573         DEVMETHOD_END
2574 };
2575
2576 driver_t mmc_driver = {
2577         "mmc",
2578         mmc_methods,
2579         sizeof(struct mmc_softc),
2580 };
2581 devclass_t mmc_devclass;
2582
2583 MODULE_VERSION(mmc, MMC_VERSION);