sys/contrib/dev/iwlwifi/fw/dbg.c

   1 // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
   2 /*
   3  * Copyright (C) 2005-2014, 2018-2021 Intel Corporation
   4  * Copyright (C) 2013-2015 Intel Mobile Communications GmbH
   5  * Copyright (C) 2015-2017 Intel Deutschland GmbH
   6  */
   7 #include <linux/devcoredump.h>
   8 #if defined(__FreeBSD__)
   9 #include <linux/delay.h>
  10 #endif
  11 #include "iwl-drv.h"
  12 #include "runtime.h"
  13 #include "dbg.h"
  14 #include "debugfs.h"
  15 #include "iwl-io.h"
  16 #include "iwl-prph.h"
  17 #include "iwl-csr.h"
  18 #include "iwl-fh.h"
  19 /**
  20  * struct iwl_fw_dump_ptrs - set of pointers needed for the fw-error-dump
  21  *
  22  * @fwrt_ptr: pointer to the buffer coming from fwrt
  23  * @trans_ptr: pointer to struct %iwl_trans_dump_data which contains the
  24  *      transport's data.
  25  * @trans_len: length of the valid data in trans_ptr
  26  * @fwrt_len: length of the valid data in fwrt_ptr
  27  */
  28 struct iwl_fw_dump_ptrs {
  29         struct iwl_trans_dump_data *trans_ptr;
  30         void *fwrt_ptr;
  31         u32 fwrt_len;
  32 };
  33
  34 #define RADIO_REG_MAX_READ 0x2ad
  35 static void iwl_read_radio_regs(struct iwl_fw_runtime *fwrt,
  36                                 struct iwl_fw_error_dump_data **dump_data)
  37 {
  38         u8 *pos = (void *)(*dump_data)->data;
  39         int i;
  40
  41         IWL_DEBUG_INFO(fwrt, "WRT radio registers dump\n");
  42
  43         if (!iwl_trans_grab_nic_access(fwrt->trans))
  44                 return;
  45
  46         (*dump_data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_RADIO_REG);
  47         (*dump_data)->len = cpu_to_le32(RADIO_REG_MAX_READ);
  48
  49         for (i = 0; i < RADIO_REG_MAX_READ; i++) {
  50                 u32 rd_cmd = RADIO_RSP_RD_CMD;
  51
  52                 rd_cmd |= i << RADIO_RSP_ADDR_POS;
  53                 iwl_write_prph_no_grab(fwrt->trans, RSP_RADIO_CMD, rd_cmd);
  54                 *pos = (u8)iwl_read_prph_no_grab(fwrt->trans, RSP_RADIO_RDDAT);
  55
  56                 pos++;
  57         }
  58
  59         *dump_data = iwl_fw_error_next_data(*dump_data);
  60
  61         iwl_trans_release_nic_access(fwrt->trans);
  62 }
  63
  64 static void iwl_fwrt_dump_rxf(struct iwl_fw_runtime *fwrt,
  65                               struct iwl_fw_error_dump_data **dump_data,
  66                               int size, u32 offset, int fifo_num)
  67 {
  68         struct iwl_fw_error_dump_fifo *fifo_hdr;
  69         u32 *fifo_data;
  70         u32 fifo_len;
  71         int i;
  72
  73         fifo_hdr = (void *)(*dump_data)->data;
  74         fifo_data = (void *)fifo_hdr->data;
  75         fifo_len = size;
  76
  77         /* No need to try to read the data if the length is 0 */
  78         if (fifo_len == 0)
  79                 return;
  80
  81         /* Add a TLV for the RXF */
  82         (*dump_data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_RXF);
  83         (*dump_data)->len = cpu_to_le32(fifo_len + sizeof(*fifo_hdr));
  84
  85         fifo_hdr->fifo_num = cpu_to_le32(fifo_num);
  86         fifo_hdr->available_bytes =
  87                 cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
  88                                                 RXF_RD_D_SPACE + offset));
  89         fifo_hdr->wr_ptr =
  90                 cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
  91                                                 RXF_RD_WR_PTR + offset));
  92         fifo_hdr->rd_ptr =
  93                 cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
  94                                                 RXF_RD_RD_PTR + offset));
  95         fifo_hdr->fence_ptr =
  96                 cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
  97                                                 RXF_RD_FENCE_PTR + offset));
  98         fifo_hdr->fence_mode =
  99                 cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 100                                                 RXF_SET_FENCE_MODE + offset));
 101
 102         /* Lock fence */
 103         iwl_trans_write_prph(fwrt->trans, RXF_SET_FENCE_MODE + offset, 0x1);
 104         /* Set fence pointer to the same place like WR pointer */
 105         iwl_trans_write_prph(fwrt->trans, RXF_LD_WR2FENCE + offset, 0x1);
 106         /* Set fence offset */
 107         iwl_trans_write_prph(fwrt->trans,
 108                              RXF_LD_FENCE_OFFSET_ADDR + offset, 0x0);
 109
 110         /* Read FIFO */
 111         fifo_len /= sizeof(u32); /* Size in DWORDS */
 112         for (i = 0; i < fifo_len; i++)
 113                 fifo_data[i] = iwl_trans_read_prph(fwrt->trans,
 114                                                  RXF_FIFO_RD_FENCE_INC +
 115                                                  offset);
 116         *dump_data = iwl_fw_error_next_data(*dump_data);
 117 }
 118
 119 static void iwl_fwrt_dump_txf(struct iwl_fw_runtime *fwrt,
 120                               struct iwl_fw_error_dump_data **dump_data,
 121                               int size, u32 offset, int fifo_num)
 122 {
 123         struct iwl_fw_error_dump_fifo *fifo_hdr;
 124         u32 *fifo_data;
 125         u32 fifo_len;
 126         int i;
 127
 128         fifo_hdr = (void *)(*dump_data)->data;
 129         fifo_data = (void *)fifo_hdr->data;
 130         fifo_len = size;
 131
 132         /* No need to try to read the data if the length is 0 */
 133         if (fifo_len == 0)
 134                 return;
 135
 136         /* Add a TLV for the FIFO */
 137         (*dump_data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_TXF);
 138         (*dump_data)->len = cpu_to_le32(fifo_len + sizeof(*fifo_hdr));
 139
 140         fifo_hdr->fifo_num = cpu_to_le32(fifo_num);
 141         fifo_hdr->available_bytes =
 142                 cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 143                                                 TXF_FIFO_ITEM_CNT + offset));
 144         fifo_hdr->wr_ptr =
 145                 cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 146                                                 TXF_WR_PTR + offset));
 147         fifo_hdr->rd_ptr =
 148                 cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 149                                                 TXF_RD_PTR + offset));
 150         fifo_hdr->fence_ptr =
 151                 cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 152                                                 TXF_FENCE_PTR + offset));
 153         fifo_hdr->fence_mode =
 154                 cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 155                                                 TXF_LOCK_FENCE + offset));
 156
 157         /* Set the TXF_READ_MODIFY_ADDR to TXF_WR_PTR */
 158         iwl_trans_write_prph(fwrt->trans, TXF_READ_MODIFY_ADDR + offset,
 159                              TXF_WR_PTR + offset);
 160
 161         /* Dummy-read to advance the read pointer to the head */
 162         iwl_trans_read_prph(fwrt->trans, TXF_READ_MODIFY_DATA + offset);
 163
 164         /* Read FIFO */
 165         for (i = 0; i < fifo_len / sizeof(u32); i++)
 166                 fifo_data[i] = iwl_trans_read_prph(fwrt->trans,
 167                                                   TXF_READ_MODIFY_DATA +
 168                                                   offset);
 169
 170         if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_txf)
 171                 fwrt->sanitize_ops->frob_txf(fwrt->sanitize_ctx,
 172                                              fifo_data, fifo_len);
 173
 174         *dump_data = iwl_fw_error_next_data(*dump_data);
 175 }
 176
 177 static void iwl_fw_dump_rxf(struct iwl_fw_runtime *fwrt,
 178                             struct iwl_fw_error_dump_data **dump_data)
 179 {
 180         struct iwl_fwrt_shared_mem_cfg *cfg = &fwrt->smem_cfg;
 181
 182         IWL_DEBUG_INFO(fwrt, "WRT RX FIFO dump\n");
 183
 184         if (!iwl_trans_grab_nic_access(fwrt->trans))
 185                 return;
 186
 187         if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_RXF)) {
 188                 /* Pull RXF1 */
 189                 iwl_fwrt_dump_rxf(fwrt, dump_data,
 190                                   cfg->lmac[0].rxfifo1_size, 0, 0);
 191                 /* Pull RXF2 */
 192                 iwl_fwrt_dump_rxf(fwrt, dump_data, cfg->rxfifo2_size,
 193                                   RXF_DIFF_FROM_PREV +
 194                                   fwrt->trans->trans_cfg->umac_prph_offset, 1);
 195                 /* Pull LMAC2 RXF1 */
 196                 if (fwrt->smem_cfg.num_lmacs > 1)
 197                         iwl_fwrt_dump_rxf(fwrt, dump_data,
 198                                           cfg->lmac[1].rxfifo1_size,
 199                                           LMAC2_PRPH_OFFSET, 2);
 200         }
 201
 202         iwl_trans_release_nic_access(fwrt->trans);
 203 }
 204
 205 static void iwl_fw_dump_txf(struct iwl_fw_runtime *fwrt,
 206                             struct iwl_fw_error_dump_data **dump_data)
 207 {
 208         struct iwl_fw_error_dump_fifo *fifo_hdr;
 209         struct iwl_fwrt_shared_mem_cfg *cfg = &fwrt->smem_cfg;
 210         u32 *fifo_data;
 211         u32 fifo_len;
 212         int i, j;
 213
 214         IWL_DEBUG_INFO(fwrt, "WRT TX FIFO dump\n");
 215
 216         if (!iwl_trans_grab_nic_access(fwrt->trans))
 217                 return;
 218
 219         if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_TXF)) {
 220                 /* Pull TXF data from LMAC1 */
 221                 for (i = 0; i < fwrt->smem_cfg.num_txfifo_entries; i++) {
 222                         /* Mark the number of TXF we're pulling now */
 223                         iwl_trans_write_prph(fwrt->trans, TXF_LARC_NUM, i);
 224                         iwl_fwrt_dump_txf(fwrt, dump_data,
 225                                           cfg->lmac[0].txfifo_size[i], 0, i);
 226                 }
 227
 228                 /* Pull TXF data from LMAC2 */
 229                 if (fwrt->smem_cfg.num_lmacs > 1) {
 230                         for (i = 0; i < fwrt->smem_cfg.num_txfifo_entries;
 231                              i++) {
 232                                 /* Mark the number of TXF we're pulling now */
 233                                 iwl_trans_write_prph(fwrt->trans,
 234                                                      TXF_LARC_NUM +
 235                                                      LMAC2_PRPH_OFFSET, i);
 236                                 iwl_fwrt_dump_txf(fwrt, dump_data,
 237                                                   cfg->lmac[1].txfifo_size[i],
 238                                                   LMAC2_PRPH_OFFSET,
 239                                                   i + cfg->num_txfifo_entries);
 240                         }
 241                 }
 242         }
 243
 244         if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_INTERNAL_TXF) &&
 245             fw_has_capa(&fwrt->fw->ucode_capa,
 246                         IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG)) {
 247                 /* Pull UMAC internal TXF data from all TXFs */
 248                 for (i = 0;
 249                      i < ARRAY_SIZE(fwrt->smem_cfg.internal_txfifo_size);
 250                      i++) {
 251                         fifo_hdr = (void *)(*dump_data)->data;
 252                         fifo_data = (void *)fifo_hdr->data;
 253                         fifo_len = fwrt->smem_cfg.internal_txfifo_size[i];
 254
 255                         /* No need to try to read the data if the length is 0 */
 256                         if (fifo_len == 0)
 257                                 continue;
 258
 259                         /* Add a TLV for the internal FIFOs */
 260                         (*dump_data)->type =
 261                                 cpu_to_le32(IWL_FW_ERROR_DUMP_INTERNAL_TXF);
 262                         (*dump_data)->len =
 263                                 cpu_to_le32(fifo_len + sizeof(*fifo_hdr));
 264
 265                         fifo_hdr->fifo_num = cpu_to_le32(i);
 266
 267                         /* Mark the number of TXF we're pulling now */
 268                         iwl_trans_write_prph(fwrt->trans, TXF_CPU2_NUM, i +
 269                                 fwrt->smem_cfg.num_txfifo_entries);
 270
 271                         fifo_hdr->available_bytes =
 272                                 cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 273                                                                 TXF_CPU2_FIFO_ITEM_CNT));
 274                         fifo_hdr->wr_ptr =
 275                                 cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 276                                                                 TXF_CPU2_WR_PTR));
 277                         fifo_hdr->rd_ptr =
 278                                 cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 279                                                                 TXF_CPU2_RD_PTR));
 280                         fifo_hdr->fence_ptr =
 281                                 cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 282                                                                 TXF_CPU2_FENCE_PTR));
 283                         fifo_hdr->fence_mode =
 284                                 cpu_to_le32(iwl_trans_read_prph(fwrt->trans,
 285                                                                 TXF_CPU2_LOCK_FENCE));
 286
 287                         /* Set TXF_CPU2_READ_MODIFY_ADDR to TXF_CPU2_WR_PTR */
 288                         iwl_trans_write_prph(fwrt->trans,
 289                                              TXF_CPU2_READ_MODIFY_ADDR,
 290                                              TXF_CPU2_WR_PTR);
 291
 292                         /* Dummy-read to advance the read pointer to head */
 293                         iwl_trans_read_prph(fwrt->trans,
 294                                             TXF_CPU2_READ_MODIFY_DATA);
 295
 296                         /* Read FIFO */
 297                         fifo_len /= sizeof(u32); /* Size in DWORDS */
 298                         for (j = 0; j < fifo_len; j++)
 299                                 fifo_data[j] =
 300                                         iwl_trans_read_prph(fwrt->trans,
 301                                                             TXF_CPU2_READ_MODIFY_DATA);
 302                         *dump_data = iwl_fw_error_next_data(*dump_data);
 303                 }
 304         }
 305
 306         iwl_trans_release_nic_access(fwrt->trans);
 307 }
 308
 309 struct iwl_prph_range {
 310         u32 start, end;
 311 };
 312
 313 static const struct iwl_prph_range iwl_prph_dump_addr_comm[] = {
 314         { .start = 0x00a00000, .end = 0x00a00000 },
 315         { .start = 0x00a0000c, .end = 0x00a00024 },
 316         { .start = 0x00a0002c, .end = 0x00a0003c },
 317         { .start = 0x00a00410, .end = 0x00a00418 },
 318         { .start = 0x00a00420, .end = 0x00a00420 },
 319         { .start = 0x00a00428, .end = 0x00a00428 },
 320         { .start = 0x00a00430, .end = 0x00a0043c },
 321         { .start = 0x00a00444, .end = 0x00a00444 },
 322         { .start = 0x00a004c0, .end = 0x00a004cc },
 323         { .start = 0x00a004d8, .end = 0x00a004d8 },
 324         { .start = 0x00a004e0, .end = 0x00a004f0 },
 325         { .start = 0x00a00840, .end = 0x00a00840 },
 326         { .start = 0x00a00850, .end = 0x00a00858 },
 327         { .start = 0x00a01004, .end = 0x00a01008 },
 328         { .start = 0x00a01010, .end = 0x00a01010 },
 329         { .start = 0x00a01018, .end = 0x00a01018 },
 330         { .start = 0x00a01024, .end = 0x00a01024 },
 331         { .start = 0x00a0102c, .end = 0x00a01034 },
 332         { .start = 0x00a0103c, .end = 0x00a01040 },
 333         { .start = 0x00a01048, .end = 0x00a01094 },
 334         { .start = 0x00a01c00, .end = 0x00a01c20 },
 335         { .start = 0x00a01c58, .end = 0x00a01c58 },
 336         { .start = 0x00a01c7c, .end = 0x00a01c7c },
 337         { .start = 0x00a01c28, .end = 0x00a01c54 },
 338         { .start = 0x00a01c5c, .end = 0x00a01c5c },
 339         { .start = 0x00a01c60, .end = 0x00a01cdc },
 340         { .start = 0x00a01ce0, .end = 0x00a01d0c },
 341         { .start = 0x00a01d18, .end = 0x00a01d20 },
 342         { .start = 0x00a01d2c, .end = 0x00a01d30 },
 343         { .start = 0x00a01d40, .end = 0x00a01d5c },
 344         { .start = 0x00a01d80, .end = 0x00a01d80 },
 345         { .start = 0x00a01d98, .end = 0x00a01d9c },
 346         { .start = 0x00a01da8, .end = 0x00a01da8 },
 347         { .start = 0x00a01db8, .end = 0x00a01df4 },
 348         { .start = 0x00a01dc0, .end = 0x00a01dfc },
 349         { .start = 0x00a01e00, .end = 0x00a01e2c },
 350         { .start = 0x00a01e40, .end = 0x00a01e60 },
 351         { .start = 0x00a01e68, .end = 0x00a01e6c },
 352         { .start = 0x00a01e74, .end = 0x00a01e74 },
 353         { .start = 0x00a01e84, .end = 0x00a01e90 },
 354         { .start = 0x00a01e9c, .end = 0x00a01ec4 },
 355         { .start = 0x00a01ed0, .end = 0x00a01ee0 },
 356         { .start = 0x00a01f00, .end = 0x00a01f1c },
 357         { .start = 0x00a01f44, .end = 0x00a01ffc },
 358         { .start = 0x00a02000, .end = 0x00a02048 },
 359         { .start = 0x00a02068, .end = 0x00a020f0 },
 360         { .start = 0x00a02100, .end = 0x00a02118 },
 361         { .start = 0x00a02140, .end = 0x00a0214c },
 362         { .start = 0x00a02168, .end = 0x00a0218c },
 363         { .start = 0x00a021c0, .end = 0x00a021c0 },
 364         { .start = 0x00a02400, .end = 0x00a02410 },
 365         { .start = 0x00a02418, .end = 0x00a02420 },
 366         { .start = 0x00a02428, .end = 0x00a0242c },
 367         { .start = 0x00a02434, .end = 0x00a02434 },
 368         { .start = 0x00a02440, .end = 0x00a02460 },
 369         { .start = 0x00a02468, .end = 0x00a024b0 },
 370         { .start = 0x00a024c8, .end = 0x00a024cc },
 371         { .start = 0x00a02500, .end = 0x00a02504 },
 372         { .start = 0x00a0250c, .end = 0x00a02510 },
 373         { .start = 0x00a02540, .end = 0x00a02554 },
 374         { .start = 0x00a02580, .end = 0x00a025f4 },
 375         { .start = 0x00a02600, .end = 0x00a0260c },
 376         { .start = 0x00a02648, .end = 0x00a02650 },
 377         { .start = 0x00a02680, .end = 0x00a02680 },
 378         { .start = 0x00a026c0, .end = 0x00a026d0 },
 379         { .start = 0x00a02700, .end = 0x00a0270c },
 380         { .start = 0x00a02804, .end = 0x00a02804 },
 381         { .start = 0x00a02818, .end = 0x00a0281c },
 382         { .start = 0x00a02c00, .end = 0x00a02db4 },
 383         { .start = 0x00a02df4, .end = 0x00a02fb0 },
 384         { .start = 0x00a03000, .end = 0x00a03014 },
 385         { .start = 0x00a0301c, .end = 0x00a0302c },
 386         { .start = 0x00a03034, .end = 0x00a03038 },
 387         { .start = 0x00a03040, .end = 0x00a03048 },
 388         { .start = 0x00a03060, .end = 0x00a03068 },
 389         { .start = 0x00a03070, .end = 0x00a03074 },
 390         { .start = 0x00a0307c, .end = 0x00a0307c },
 391         { .start = 0x00a03080, .end = 0x00a03084 },
 392         { .start = 0x00a0308c, .end = 0x00a03090 },
 393         { .start = 0x00a03098, .end = 0x00a03098 },
 394         { .start = 0x00a030a0, .end = 0x00a030a0 },
 395         { .start = 0x00a030a8, .end = 0x00a030b4 },
 396         { .start = 0x00a030bc, .end = 0x00a030bc },
 397         { .start = 0x00a030c0, .end = 0x00a0312c },
 398         { .start = 0x00a03c00, .end = 0x00a03c5c },
 399         { .start = 0x00a04400, .end = 0x00a04454 },
 400         { .start = 0x00a04460, .end = 0x00a04474 },
 401         { .start = 0x00a044c0, .end = 0x00a044ec },
 402         { .start = 0x00a04500, .end = 0x00a04504 },
 403         { .start = 0x00a04510, .end = 0x00a04538 },
 404         { .start = 0x00a04540, .end = 0x00a04548 },
 405         { .start = 0x00a04560, .end = 0x00a0457c },
 406         { .start = 0x00a04590, .end = 0x00a04598 },
 407         { .start = 0x00a045c0, .end = 0x00a045f4 },
 408 };
 409
 410 static const struct iwl_prph_range iwl_prph_dump_addr_9000[] = {
 411         { .start = 0x00a05c00, .end = 0x00a05c18 },
 412         { .start = 0x00a05400, .end = 0x00a056e8 },
 413         { .start = 0x00a08000, .end = 0x00a098bc },
 414         { .start = 0x00a02400, .end = 0x00a02758 },
 415         { .start = 0x00a04764, .end = 0x00a0476c },
 416         { .start = 0x00a04770, .end = 0x00a04774 },
 417         { .start = 0x00a04620, .end = 0x00a04624 },
 418 };
 419
 420 static const struct iwl_prph_range iwl_prph_dump_addr_22000[] = {
 421         { .start = 0x00a00000, .end = 0x00a00000 },
 422         { .start = 0x00a0000c, .end = 0x00a00024 },
 423         { .start = 0x00a0002c, .end = 0x00a00034 },
 424         { .start = 0x00a0003c, .end = 0x00a0003c },
 425         { .start = 0x00a00410, .end = 0x00a00418 },
 426         { .start = 0x00a00420, .end = 0x00a00420 },
 427         { .start = 0x00a00428, .end = 0x00a00428 },
 428         { .start = 0x00a00430, .end = 0x00a0043c },
 429         { .start = 0x00a00444, .end = 0x00a00444 },
 430         { .start = 0x00a00840, .end = 0x00a00840 },
 431         { .start = 0x00a00850, .end = 0x00a00858 },
 432         { .start = 0x00a01004, .end = 0x00a01008 },
 433         { .start = 0x00a01010, .end = 0x00a01010 },
 434         { .start = 0x00a01018, .end = 0x00a01018 },
 435         { .start = 0x00a01024, .end = 0x00a01024 },
 436         { .start = 0x00a0102c, .end = 0x00a01034 },
 437         { .start = 0x00a0103c, .end = 0x00a01040 },
 438         { .start = 0x00a01048, .end = 0x00a01050 },
 439         { .start = 0x00a01058, .end = 0x00a01058 },
 440         { .start = 0x00a01060, .end = 0x00a01070 },
 441         { .start = 0x00a0108c, .end = 0x00a0108c },
 442         { .start = 0x00a01c20, .end = 0x00a01c28 },
 443         { .start = 0x00a01d10, .end = 0x00a01d10 },
 444         { .start = 0x00a01e28, .end = 0x00a01e2c },
 445         { .start = 0x00a01e60, .end = 0x00a01e60 },
 446         { .start = 0x00a01e80, .end = 0x00a01e80 },
 447         { .start = 0x00a01ea0, .end = 0x00a01ea0 },
 448         { .start = 0x00a02000, .end = 0x00a0201c },
 449         { .start = 0x00a02024, .end = 0x00a02024 },
 450         { .start = 0x00a02040, .end = 0x00a02048 },
 451         { .start = 0x00a020c0, .end = 0x00a020e0 },
 452         { .start = 0x00a02400, .end = 0x00a02404 },
 453         { .start = 0x00a0240c, .end = 0x00a02414 },
 454         { .start = 0x00a0241c, .end = 0x00a0243c },
 455         { .start = 0x00a02448, .end = 0x00a024bc },
 456         { .start = 0x00a024c4, .end = 0x00a024cc },
 457         { .start = 0x00a02508, .end = 0x00a02508 },
 458         { .start = 0x00a02510, .end = 0x00a02514 },
 459         { .start = 0x00a0251c, .end = 0x00a0251c },
 460         { .start = 0x00a0252c, .end = 0x00a0255c },
 461         { .start = 0x00a02564, .end = 0x00a025a0 },
 462         { .start = 0x00a025a8, .end = 0x00a025b4 },
 463         { .start = 0x00a025c0, .end = 0x00a025c0 },
 464         { .start = 0x00a025e8, .end = 0x00a025f4 },
 465         { .start = 0x00a02c08, .end = 0x00a02c18 },
 466         { .start = 0x00a02c2c, .end = 0x00a02c38 },
 467         { .start = 0x00a02c68, .end = 0x00a02c78 },
 468         { .start = 0x00a03000, .end = 0x00a03000 },
 469         { .start = 0x00a03010, .end = 0x00a03014 },
 470         { .start = 0x00a0301c, .end = 0x00a0302c },
 471         { .start = 0x00a03034, .end = 0x00a03038 },
 472         { .start = 0x00a03040, .end = 0x00a03044 },
 473         { .start = 0x00a03060, .end = 0x00a03068 },
 474         { .start = 0x00a03070, .end = 0x00a03070 },
 475         { .start = 0x00a0307c, .end = 0x00a03084 },
 476         { .start = 0x00a0308c, .end = 0x00a03090 },
 477         { .start = 0x00a03098, .end = 0x00a03098 },
 478         { .start = 0x00a030a0, .end = 0x00a030a0 },
 479         { .start = 0x00a030a8, .end = 0x00a030b4 },
 480         { .start = 0x00a030bc, .end = 0x00a030c0 },
 481         { .start = 0x00a030c8, .end = 0x00a030f4 },
 482         { .start = 0x00a03100, .end = 0x00a0312c },
 483         { .start = 0x00a03c00, .end = 0x00a03c5c },
 484         { .start = 0x00a04400, .end = 0x00a04454 },
 485         { .start = 0x00a04460, .end = 0x00a04474 },
 486         { .start = 0x00a044c0, .end = 0x00a044ec },
 487         { .start = 0x00a04500, .end = 0x00a04504 },
 488         { .start = 0x00a04510, .end = 0x00a04538 },
 489         { .start = 0x00a04540, .end = 0x00a04548 },
 490         { .start = 0x00a04560, .end = 0x00a04560 },
 491         { .start = 0x00a04570, .end = 0x00a0457c },
 492         { .start = 0x00a04590, .end = 0x00a04590 },
 493         { .start = 0x00a04598, .end = 0x00a04598 },
 494         { .start = 0x00a045c0, .end = 0x00a045f4 },
 495         { .start = 0x00a05c18, .end = 0x00a05c1c },
 496         { .start = 0x00a0c000, .end = 0x00a0c018 },
 497         { .start = 0x00a0c020, .end = 0x00a0c028 },
 498         { .start = 0x00a0c038, .end = 0x00a0c094 },
 499         { .start = 0x00a0c0c0, .end = 0x00a0c104 },
 500         { .start = 0x00a0c10c, .end = 0x00a0c118 },
 501         { .start = 0x00a0c150, .end = 0x00a0c174 },
 502         { .start = 0x00a0c17c, .end = 0x00a0c188 },
 503         { .start = 0x00a0c190, .end = 0x00a0c198 },
 504         { .start = 0x00a0c1a0, .end = 0x00a0c1a8 },
 505         { .start = 0x00a0c1b0, .end = 0x00a0c1b8 },
 506 };
 507
 508 static const struct iwl_prph_range iwl_prph_dump_addr_ax210[] = {
 509         { .start = 0x00d03c00, .end = 0x00d03c64 },
 510         { .start = 0x00d05c18, .end = 0x00d05c1c },
 511         { .start = 0x00d0c000, .end = 0x00d0c174 },
 512 };
 513
 514 static void iwl_read_prph_block(struct iwl_trans *trans, u32 start,
 515                                 u32 len_bytes, __le32 *data)
 516 {
 517         u32 i;
 518
 519         for (i = 0; i < len_bytes; i += 4)
 520                 *data++ = cpu_to_le32(iwl_read_prph_no_grab(trans, start + i));
 521 }
 522
 523 static void iwl_dump_prph(struct iwl_fw_runtime *fwrt,
 524                           const struct iwl_prph_range *iwl_prph_dump_addr,
 525                           u32 range_len, void *ptr)
 526 {
 527         struct iwl_fw_error_dump_prph *prph;
 528         struct iwl_trans *trans = fwrt->trans;
 529         struct iwl_fw_error_dump_data **data =
 530                 (struct iwl_fw_error_dump_data **)ptr;
 531         u32 i;
 532
 533         if (!data)
 534                 return;
 535
 536         IWL_DEBUG_INFO(trans, "WRT PRPH dump\n");
 537
 538         if (!iwl_trans_grab_nic_access(trans))
 539                 return;
 540
 541         for (i = 0; i < range_len; i++) {
 542                 /* The range includes both boundaries */
 543                 int num_bytes_in_chunk = iwl_prph_dump_addr[i].end -
 544                          iwl_prph_dump_addr[i].start + 4;
 545
 546                 (*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_PRPH);
 547                 (*data)->len = cpu_to_le32(sizeof(*prph) +
 548                                         num_bytes_in_chunk);
 549                 prph = (void *)(*data)->data;
 550                 prph->prph_start = cpu_to_le32(iwl_prph_dump_addr[i].start);
 551
 552                 iwl_read_prph_block(trans, iwl_prph_dump_addr[i].start,
 553                                     /* our range is inclusive, hence + 4 */
 554                                     iwl_prph_dump_addr[i].end -
 555                                     iwl_prph_dump_addr[i].start + 4,
 556                                     (void *)prph->data);
 557
 558                 *data = iwl_fw_error_next_data(*data);
 559         }
 560
 561         iwl_trans_release_nic_access(trans);
 562 }
 563
 564 /*
 565  * alloc_sgtable - allocates scallerlist table in the given size,
 566  * fills it with pages and returns it
 567  * @size: the size (in bytes) of the table
 568 */
 569 static struct scatterlist *alloc_sgtable(int size)
 570 {
 571         int alloc_size, nents, i;
 572         struct page *new_page;
 573         struct scatterlist *iter;
 574         struct scatterlist *table;
 575
 576         nents = DIV_ROUND_UP(size, PAGE_SIZE);
 577         table = kcalloc(nents, sizeof(*table), GFP_KERNEL);
 578         if (!table)
 579                 return NULL;
 580         sg_init_table(table, nents);
 581         iter = table;
 582         for_each_sg(table, iter, sg_nents(table), i) {
 583                 new_page = alloc_page(GFP_KERNEL);
 584                 if (!new_page) {
 585                         /* release all previous allocated pages in the table */
 586                         iter = table;
 587                         for_each_sg(table, iter, sg_nents(table), i) {
 588                                 new_page = sg_page(iter);
 589                                 if (new_page)
 590                                         __free_page(new_page);
 591                         }
 592                         kfree(table);
 593                         return NULL;
 594                 }
 595                 alloc_size = min_t(int, size, PAGE_SIZE);
 596                 size -= PAGE_SIZE;
 597                 sg_set_page(iter, new_page, alloc_size, 0);
 598         }
 599         return table;
 600 }
 601
 602 static void iwl_fw_get_prph_len(struct iwl_fw_runtime *fwrt,
 603                                 const struct iwl_prph_range *iwl_prph_dump_addr,
 604                                 u32 range_len, void *ptr)
 605 {
 606         u32 *prph_len = (u32 *)ptr;
 607         int i, num_bytes_in_chunk;
 608
 609         if (!prph_len)
 610                 return;
 611
 612         for (i = 0; i < range_len; i++) {
 613                 /* The range includes both boundaries */
 614                 num_bytes_in_chunk =
 615                         iwl_prph_dump_addr[i].end -
 616                         iwl_prph_dump_addr[i].start + 4;
 617
 618                 *prph_len += sizeof(struct iwl_fw_error_dump_data) +
 619                         sizeof(struct iwl_fw_error_dump_prph) +
 620                         num_bytes_in_chunk;
 621         }
 622 }
 623
 624 static void iwl_fw_prph_handler(struct iwl_fw_runtime *fwrt, void *ptr,
 625                                 void (*handler)(struct iwl_fw_runtime *,
 626                                                 const struct iwl_prph_range *,
 627                                                 u32, void *))
 628 {
 629         u32 range_len;
 630
 631         if (fwrt->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
 632                 range_len = ARRAY_SIZE(iwl_prph_dump_addr_ax210);
 633                 handler(fwrt, iwl_prph_dump_addr_ax210, range_len, ptr);
 634         } else if (fwrt->trans->trans_cfg->device_family >=
 635                    IWL_DEVICE_FAMILY_22000) {
 636                 range_len = ARRAY_SIZE(iwl_prph_dump_addr_22000);
 637                 handler(fwrt, iwl_prph_dump_addr_22000, range_len, ptr);
 638         } else {
 639                 range_len = ARRAY_SIZE(iwl_prph_dump_addr_comm);
 640                 handler(fwrt, iwl_prph_dump_addr_comm, range_len, ptr);
 641
 642                 if (fwrt->trans->trans_cfg->mq_rx_supported) {
 643                         range_len = ARRAY_SIZE(iwl_prph_dump_addr_9000);
 644                         handler(fwrt, iwl_prph_dump_addr_9000, range_len, ptr);
 645                 }
 646         }
 647 }
 648
 649 static void iwl_fw_dump_mem(struct iwl_fw_runtime *fwrt,
 650                             struct iwl_fw_error_dump_data **dump_data,
 651                             u32 len, u32 ofs, u32 type)
 652 {
 653         struct iwl_fw_error_dump_mem *dump_mem;
 654
 655         if (!len)
 656                 return;
 657
 658         (*dump_data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM);
 659         (*dump_data)->len = cpu_to_le32(len + sizeof(*dump_mem));
 660         dump_mem = (void *)(*dump_data)->data;
 661         dump_mem->type = cpu_to_le32(type);
 662         dump_mem->offset = cpu_to_le32(ofs);
 663         iwl_trans_read_mem_bytes(fwrt->trans, ofs, dump_mem->data, len);
 664         *dump_data = iwl_fw_error_next_data(*dump_data);
 665
 666         if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_mem)
 667                 fwrt->sanitize_ops->frob_mem(fwrt->sanitize_ctx, ofs,
 668                                              dump_mem->data, len);
 669
 670         IWL_DEBUG_INFO(fwrt, "WRT memory dump. Type=%u\n", dump_mem->type);
 671 }
 672
 673 #define ADD_LEN(len, item_len, const_len) \
 674         do {size_t item = item_len; len += (!!item) * const_len + item; } \
 675         while (0)
 676
 677 static int iwl_fw_rxf_len(struct iwl_fw_runtime *fwrt,
 678                           struct iwl_fwrt_shared_mem_cfg *mem_cfg)
 679 {
 680         size_t hdr_len = sizeof(struct iwl_fw_error_dump_data) +
 681                          sizeof(struct iwl_fw_error_dump_fifo);
 682         u32 fifo_len = 0;
 683         int i;
 684
 685         if (!iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_RXF))
 686                 return 0;
 687
 688         /* Count RXF2 size */
 689         ADD_LEN(fifo_len, mem_cfg->rxfifo2_size, hdr_len);
 690
 691         /* Count RXF1 sizes */
 692         if (WARN_ON(mem_cfg->num_lmacs > MAX_NUM_LMAC))
 693                 mem_cfg->num_lmacs = MAX_NUM_LMAC;
 694
 695         for (i = 0; i < mem_cfg->num_lmacs; i++)
 696                 ADD_LEN(fifo_len, mem_cfg->lmac[i].rxfifo1_size, hdr_len);
 697
 698         return fifo_len;
 699 }
 700
 701 static int iwl_fw_txf_len(struct iwl_fw_runtime *fwrt,
 702                           struct iwl_fwrt_shared_mem_cfg *mem_cfg)
 703 {
 704         size_t hdr_len = sizeof(struct iwl_fw_error_dump_data) +
 705                          sizeof(struct iwl_fw_error_dump_fifo);
 706         u32 fifo_len = 0;
 707         int i;
 708
 709         if (!iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_TXF))
 710                 goto dump_internal_txf;
 711
 712         /* Count TXF sizes */
 713         if (WARN_ON(mem_cfg->num_lmacs > MAX_NUM_LMAC))
 714                 mem_cfg->num_lmacs = MAX_NUM_LMAC;
 715
 716         for (i = 0; i < mem_cfg->num_lmacs; i++) {
 717                 int j;
 718
 719                 for (j = 0; j < mem_cfg->num_txfifo_entries; j++)
 720                         ADD_LEN(fifo_len, mem_cfg->lmac[i].txfifo_size[j],
 721                                 hdr_len);
 722         }
 723
 724 dump_internal_txf:
 725         if (!(iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_INTERNAL_TXF) &&
 726               fw_has_capa(&fwrt->fw->ucode_capa,
 727                           IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG)))
 728                 goto out;
 729
 730         for (i = 0; i < ARRAY_SIZE(mem_cfg->internal_txfifo_size); i++)
 731                 ADD_LEN(fifo_len, mem_cfg->internal_txfifo_size[i], hdr_len);
 732
 733 out:
 734         return fifo_len;
 735 }
 736
 737 static void iwl_dump_paging(struct iwl_fw_runtime *fwrt,
 738                             struct iwl_fw_error_dump_data **data)
 739 {
 740         int i;
 741
 742         IWL_DEBUG_INFO(fwrt, "WRT paging dump\n");
 743         for (i = 1; i < fwrt->num_of_paging_blk + 1; i++) {
 744                 struct iwl_fw_error_dump_paging *paging;
 745                 struct page *pages =
 746                         fwrt->fw_paging_db[i].fw_paging_block;
 747                 dma_addr_t addr = fwrt->fw_paging_db[i].fw_paging_phys;
 748
 749                 (*data)->type = cpu_to_le32(IWL_FW_ERROR_DUMP_PAGING);
 750                 (*data)->len = cpu_to_le32(sizeof(*paging) +
 751                                              PAGING_BLOCK_SIZE);
 752                 paging =  (void *)(*data)->data;
 753                 paging->index = cpu_to_le32(i);
 754                 dma_sync_single_for_cpu(fwrt->trans->dev, addr,
 755                                         PAGING_BLOCK_SIZE,
 756                                         DMA_BIDIRECTIONAL);
 757                 memcpy(paging->data, page_address(pages),
 758                        PAGING_BLOCK_SIZE);
 759                 dma_sync_single_for_device(fwrt->trans->dev, addr,
 760                                            PAGING_BLOCK_SIZE,
 761                                            DMA_BIDIRECTIONAL);
 762                 (*data) = iwl_fw_error_next_data(*data);
 763
 764                 if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_mem)
 765                         fwrt->sanitize_ops->frob_mem(fwrt->sanitize_ctx,
 766                                                      fwrt->fw_paging_db[i].fw_offs,
 767                                                      paging->data,
 768                                                      PAGING_BLOCK_SIZE);
 769         }
 770 }
 771
 772 static struct iwl_fw_error_dump_file *
 773 iwl_fw_error_dump_file(struct iwl_fw_runtime *fwrt,
 774                        struct iwl_fw_dump_ptrs *fw_error_dump,
 775                        struct iwl_fwrt_dump_data *data)
 776 {
 777         struct iwl_fw_error_dump_file *dump_file;
 778         struct iwl_fw_error_dump_data *dump_data;
 779         struct iwl_fw_error_dump_info *dump_info;
 780         struct iwl_fw_error_dump_smem_cfg *dump_smem_cfg;
 781         struct iwl_fw_error_dump_trigger_desc *dump_trig;
 782         u32 sram_len, sram_ofs;
 783         const struct iwl_fw_dbg_mem_seg_tlv *fw_mem = fwrt->fw->dbg.mem_tlv;
 784         struct iwl_fwrt_shared_mem_cfg *mem_cfg = &fwrt->smem_cfg;
 785         u32 file_len, fifo_len = 0, prph_len = 0, radio_len = 0;
 786         u32 smem_len = fwrt->fw->dbg.n_mem_tlv ? 0 : fwrt->trans->cfg->smem_len;
 787         u32 sram2_len = fwrt->fw->dbg.n_mem_tlv ?
 788                                 0 : fwrt->trans->cfg->dccm2_len;
 789         int i;
 790
 791         /* SRAM - include stack CCM if driver knows the values for it */
 792         if (!fwrt->trans->cfg->dccm_offset || !fwrt->trans->cfg->dccm_len) {
 793                 const struct fw_img *img;
 794
 795                 if (fwrt->cur_fw_img >= IWL_UCODE_TYPE_MAX)
 796                         return NULL;
 797                 img = &fwrt->fw->img[fwrt->cur_fw_img];
 798                 sram_ofs = img->sec[IWL_UCODE_SECTION_DATA].offset;
 799                 sram_len = img->sec[IWL_UCODE_SECTION_DATA].len;
 800         } else {
 801                 sram_ofs = fwrt->trans->cfg->dccm_offset;
 802                 sram_len = fwrt->trans->cfg->dccm_len;
 803         }
 804
 805         /* reading RXF/TXF sizes */
 806         if (test_bit(STATUS_FW_ERROR, &fwrt->trans->status)) {
 807                 fifo_len = iwl_fw_rxf_len(fwrt, mem_cfg);
 808                 fifo_len += iwl_fw_txf_len(fwrt, mem_cfg);
 809
 810                 /* Make room for PRPH registers */
 811                 if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_PRPH))
 812                         iwl_fw_prph_handler(fwrt, &prph_len,
 813                                             iwl_fw_get_prph_len);
 814
 815                 if (fwrt->trans->trans_cfg->device_family ==
 816                     IWL_DEVICE_FAMILY_7000 &&
 817                     iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_RADIO_REG))
 818                         radio_len = sizeof(*dump_data) + RADIO_REG_MAX_READ;
 819         }
 820
 821         file_len = sizeof(*dump_file) + fifo_len + prph_len + radio_len;
 822
 823         if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_DEV_FW_INFO))
 824                 file_len += sizeof(*dump_data) + sizeof(*dump_info);
 825         if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM_CFG))
 826                 file_len += sizeof(*dump_data) + sizeof(*dump_smem_cfg);
 827
 828         if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM)) {
 829                 size_t hdr_len = sizeof(*dump_data) +
 830                                  sizeof(struct iwl_fw_error_dump_mem);
 831
 832                 /* Dump SRAM only if no mem_tlvs */
 833                 if (!fwrt->fw->dbg.n_mem_tlv)
 834                         ADD_LEN(file_len, sram_len, hdr_len);
 835
 836                 /* Make room for all mem types that exist */
 837                 ADD_LEN(file_len, smem_len, hdr_len);
 838                 ADD_LEN(file_len, sram2_len, hdr_len);
 839
 840                 for (i = 0; i < fwrt->fw->dbg.n_mem_tlv; i++)
 841                         ADD_LEN(file_len, le32_to_cpu(fw_mem[i].len), hdr_len);
 842         }
 843
 844         /* Make room for fw's virtual image pages, if it exists */
 845         if (iwl_fw_dbg_is_paging_enabled(fwrt))
 846                 file_len += fwrt->num_of_paging_blk *
 847                         (sizeof(*dump_data) +
 848                          sizeof(struct iwl_fw_error_dump_paging) +
 849                          PAGING_BLOCK_SIZE);
 850
 851         if (iwl_fw_dbg_is_d3_debug_enabled(fwrt) && fwrt->dump.d3_debug_data) {
 852                 file_len += sizeof(*dump_data) +
 853                         fwrt->trans->cfg->d3_debug_data_length * 2;
 854         }
 855
 856         /* If we only want a monitor dump, reset the file length */
 857         if (data->monitor_only) {
 858                 file_len = sizeof(*dump_file) + sizeof(*dump_data) * 2 +
 859                            sizeof(*dump_info) + sizeof(*dump_smem_cfg);
 860         }
 861
 862         if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_ERROR_INFO) &&
 863             data->desc)
 864                 file_len += sizeof(*dump_data) + sizeof(*dump_trig) +
 865                         data->desc->len;
 866
 867         dump_file = vzalloc(file_len);
 868         if (!dump_file)
 869                 return NULL;
 870
 871         fw_error_dump->fwrt_ptr = dump_file;
 872
 873         dump_file->barker = cpu_to_le32(IWL_FW_ERROR_DUMP_BARKER);
 874         dump_data = (void *)dump_file->data;
 875
 876         if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_DEV_FW_INFO)) {
 877                 dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_DEV_FW_INFO);
 878                 dump_data->len = cpu_to_le32(sizeof(*dump_info));
 879                 dump_info = (void *)dump_data->data;
 880                 dump_info->hw_type =
 881                         cpu_to_le32(CSR_HW_REV_TYPE(fwrt->trans->hw_rev));
 882                 dump_info->hw_step =
 883                         cpu_to_le32(fwrt->trans->hw_rev_step);
 884                 memcpy(dump_info->fw_human_readable, fwrt->fw->human_readable,
 885                        sizeof(dump_info->fw_human_readable));
 886                 strncpy(dump_info->dev_human_readable, fwrt->trans->name,
 887                         sizeof(dump_info->dev_human_readable) - 1);
 888 #if defined(__linux__)
 889                 strncpy(dump_info->bus_human_readable, fwrt->dev->bus->name,
 890                         sizeof(dump_info->bus_human_readable) - 1);
 891 #elif defined(__FreeBSD__)      /* XXX TODO */
 892                 strncpy(dump_info->bus_human_readable, "<bus>",
 893                         sizeof(dump_info->bus_human_readable) - 1);
 894 #endif
 895                 dump_info->num_of_lmacs = fwrt->smem_cfg.num_lmacs;
 896                 dump_info->lmac_err_id[0] =
 897                         cpu_to_le32(fwrt->dump.lmac_err_id[0]);
 898                 if (fwrt->smem_cfg.num_lmacs > 1)
 899                         dump_info->lmac_err_id[1] =
 900                                 cpu_to_le32(fwrt->dump.lmac_err_id[1]);
 901                 dump_info->umac_err_id = cpu_to_le32(fwrt->dump.umac_err_id);
 902
 903                 dump_data = iwl_fw_error_next_data(dump_data);
 904         }
 905
 906         if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM_CFG)) {
 907                 /* Dump shared memory configuration */
 908                 dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_MEM_CFG);
 909                 dump_data->len = cpu_to_le32(sizeof(*dump_smem_cfg));
 910                 dump_smem_cfg = (void *)dump_data->data;
 911                 dump_smem_cfg->num_lmacs = cpu_to_le32(mem_cfg->num_lmacs);
 912                 dump_smem_cfg->num_txfifo_entries =
 913                         cpu_to_le32(mem_cfg->num_txfifo_entries);
 914                 for (i = 0; i < MAX_NUM_LMAC; i++) {
 915                         int j;
 916                         u32 *txf_size = mem_cfg->lmac[i].txfifo_size;
 917
 918                         for (j = 0; j < TX_FIFO_MAX_NUM; j++)
 919                                 dump_smem_cfg->lmac[i].txfifo_size[j] =
 920                                         cpu_to_le32(txf_size[j]);
 921                         dump_smem_cfg->lmac[i].rxfifo1_size =
 922                                 cpu_to_le32(mem_cfg->lmac[i].rxfifo1_size);
 923                 }
 924                 dump_smem_cfg->rxfifo2_size =
 925                         cpu_to_le32(mem_cfg->rxfifo2_size);
 926                 dump_smem_cfg->internal_txfifo_addr =
 927                         cpu_to_le32(mem_cfg->internal_txfifo_addr);
 928                 for (i = 0; i < TX_FIFO_INTERNAL_MAX_NUM; i++) {
 929                         dump_smem_cfg->internal_txfifo_size[i] =
 930                                 cpu_to_le32(mem_cfg->internal_txfifo_size[i]);
 931                 }
 932
 933                 dump_data = iwl_fw_error_next_data(dump_data);
 934         }
 935
 936         /* We only dump the FIFOs if the FW is in error state */
 937         if (fifo_len) {
 938                 iwl_fw_dump_rxf(fwrt, &dump_data);
 939                 iwl_fw_dump_txf(fwrt, &dump_data);
 940         }
 941
 942         if (radio_len)
 943                 iwl_read_radio_regs(fwrt, &dump_data);
 944
 945         if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_ERROR_INFO) &&
 946             data->desc) {
 947                 dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_ERROR_INFO);
 948                 dump_data->len = cpu_to_le32(sizeof(*dump_trig) +
 949                                              data->desc->len);
 950                 dump_trig = (void *)dump_data->data;
 951                 memcpy(dump_trig, &data->desc->trig_desc,
 952                        sizeof(*dump_trig) + data->desc->len);
 953
 954                 dump_data = iwl_fw_error_next_data(dump_data);
 955         }
 956
 957         /* In case we only want monitor dump, skip to dump trasport data */
 958         if (data->monitor_only)
 959                 goto out;
 960
 961         if (iwl_fw_dbg_type_on(fwrt, IWL_FW_ERROR_DUMP_MEM)) {
 962                 const struct iwl_fw_dbg_mem_seg_tlv *fw_dbg_mem =
 963                         fwrt->fw->dbg.mem_tlv;
 964
 965                 if (!fwrt->fw->dbg.n_mem_tlv)
 966                         iwl_fw_dump_mem(fwrt, &dump_data, sram_len, sram_ofs,
 967                                         IWL_FW_ERROR_DUMP_MEM_SRAM);
 968
 969                 for (i = 0; i < fwrt->fw->dbg.n_mem_tlv; i++) {
 970                         u32 len = le32_to_cpu(fw_dbg_mem[i].len);
 971                         u32 ofs = le32_to_cpu(fw_dbg_mem[i].ofs);
 972
 973                         iwl_fw_dump_mem(fwrt, &dump_data, len, ofs,
 974                                         le32_to_cpu(fw_dbg_mem[i].data_type));
 975                 }
 976
 977                 iwl_fw_dump_mem(fwrt, &dump_data, smem_len,
 978                                 fwrt->trans->cfg->smem_offset,
 979                                 IWL_FW_ERROR_DUMP_MEM_SMEM);
 980
 981                 iwl_fw_dump_mem(fwrt, &dump_data, sram2_len,
 982                                 fwrt->trans->cfg->dccm2_offset,
 983                                 IWL_FW_ERROR_DUMP_MEM_SRAM);
 984         }
 985
 986         if (iwl_fw_dbg_is_d3_debug_enabled(fwrt) && fwrt->dump.d3_debug_data) {
 987                 u32 addr = fwrt->trans->cfg->d3_debug_data_base_addr;
 988                 size_t data_size = fwrt->trans->cfg->d3_debug_data_length;
 989
 990                 dump_data->type = cpu_to_le32(IWL_FW_ERROR_DUMP_D3_DEBUG_DATA);
 991                 dump_data->len = cpu_to_le32(data_size * 2);
 992
 993                 memcpy(dump_data->data, fwrt->dump.d3_debug_data, data_size);
 994
 995                 kfree(fwrt->dump.d3_debug_data);
 996                 fwrt->dump.d3_debug_data = NULL;
 997
 998                 iwl_trans_read_mem_bytes(fwrt->trans, addr,
 999                                          dump_data->data + data_size,
1000                                          data_size);
1001
1002                 if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_mem)
1003                         fwrt->sanitize_ops->frob_mem(fwrt->sanitize_ctx, addr,
1004                                                      dump_data->data + data_size,
1005                                                      data_size);
1006
1007                 dump_data = iwl_fw_error_next_data(dump_data);
1008         }
1009
1010         /* Dump fw's virtual image */
1011         if (iwl_fw_dbg_is_paging_enabled(fwrt))
1012                 iwl_dump_paging(fwrt, &dump_data);
1013
1014         if (prph_len)
1015                 iwl_fw_prph_handler(fwrt, &dump_data, iwl_dump_prph);
1016
1017 out:
1018         dump_file->file_len = cpu_to_le32(file_len);
1019         return dump_file;
1020 }
1021
1022 /**
1023  * struct iwl_dump_ini_region_data - region data
1024  * @reg_tlv: region TLV
1025  * @dump_data: dump data
1026  */
1027 struct iwl_dump_ini_region_data {
1028         struct iwl_ucode_tlv *reg_tlv;
1029         struct iwl_fwrt_dump_data *dump_data;
1030 };
1031
1032 static int
1033 iwl_dump_ini_prph_mac_iter(struct iwl_fw_runtime *fwrt,
1034                            struct iwl_dump_ini_region_data *reg_data,
1035                            void *range_ptr, u32 range_len, int idx)
1036 {
1037         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1038         struct iwl_fw_ini_error_dump_range *range = range_ptr;
1039         __le32 *val = range->data;
1040         u32 prph_val;
1041         u32 addr = le32_to_cpu(reg->addrs[idx]) +
1042                    le32_to_cpu(reg->dev_addr.offset);
1043         int i;
1044
1045         range->internal_base_addr = cpu_to_le32(addr);
1046         range->range_data_size = reg->dev_addr.size;
1047         for (i = 0; i < le32_to_cpu(reg->dev_addr.size); i += 4) {
1048                 prph_val = iwl_read_prph(fwrt->trans, addr + i);
1049                 if (prph_val == 0x5a5a5a5a)
1050                         return -EBUSY;
1051                 *val++ = cpu_to_le32(prph_val);
1052         }
1053
1054         return sizeof(*range) + le32_to_cpu(range->range_data_size);
1055 }
1056
1057 static int
1058 iwl_dump_ini_prph_phy_iter(struct iwl_fw_runtime *fwrt,
1059                            struct iwl_dump_ini_region_data *reg_data,
1060                            void *range_ptr, u32 range_len, int idx)
1061 {
1062         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1063         struct iwl_fw_ini_error_dump_range *range = range_ptr;
1064         __le32 *val = range->data;
1065         u32 indirect_wr_addr = WMAL_INDRCT_RD_CMD1;
1066         u32 indirect_rd_addr = WMAL_MRSPF_1;
1067         u32 prph_val;
1068         u32 addr = le32_to_cpu(reg->addrs[idx]);
1069         u32 dphy_state;
1070         u32 dphy_addr;
1071         int i;
1072
1073         range->internal_base_addr = cpu_to_le32(addr);
1074         range->range_data_size = reg->dev_addr.size;
1075
1076         if (fwrt->trans->trans_cfg->device_family < IWL_DEVICE_FAMILY_AX210)
1077                 indirect_wr_addr = WMAL_INDRCT_CMD1;
1078
1079         indirect_wr_addr += le32_to_cpu(reg->dev_addr.offset);
1080         indirect_rd_addr += le32_to_cpu(reg->dev_addr.offset);
1081
1082         if (!iwl_trans_grab_nic_access(fwrt->trans))
1083                 return -EBUSY;
1084
1085         dphy_addr = (reg->dev_addr.offset) ? WFPM_LMAC2_PS_CTL_RW :
1086                                              WFPM_LMAC1_PS_CTL_RW;
1087         dphy_state = iwl_read_umac_prph_no_grab(fwrt->trans, dphy_addr);
1088
1089         for (i = 0; i < le32_to_cpu(reg->dev_addr.size); i += 4) {
1090                 if (dphy_state == HBUS_TIMEOUT ||
1091                     (dphy_state & WFPM_PS_CTL_RW_PHYRF_PD_FSM_CURSTATE_MSK) !=
1092                     WFPM_PHYRF_STATE_ON) {
1093                         *val++ = cpu_to_le32(WFPM_DPHY_OFF);
1094                         continue;
1095                 }
1096
1097                 iwl_write_prph_no_grab(fwrt->trans, indirect_wr_addr,
1098                                        WMAL_INDRCT_CMD(addr + i));
1099                 prph_val = iwl_read_prph_no_grab(fwrt->trans,
1100                                                  indirect_rd_addr);
1101                 *val++ = cpu_to_le32(prph_val);
1102         }
1103
1104         iwl_trans_release_nic_access(fwrt->trans);
1105         return sizeof(*range) + le32_to_cpu(range->range_data_size);
1106 }
1107
1108 static int iwl_dump_ini_csr_iter(struct iwl_fw_runtime *fwrt,
1109                                  struct iwl_dump_ini_region_data *reg_data,
1110                                  void *range_ptr, u32 range_len, int idx)
1111 {
1112         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1113         struct iwl_fw_ini_error_dump_range *range = range_ptr;
1114         __le32 *val = range->data;
1115         u32 addr = le32_to_cpu(reg->addrs[idx]) +
1116                    le32_to_cpu(reg->dev_addr.offset);
1117         int i;
1118
1119         range->internal_base_addr = cpu_to_le32(addr);
1120         range->range_data_size = reg->dev_addr.size;
1121         for (i = 0; i < le32_to_cpu(reg->dev_addr.size); i += 4)
1122                 *val++ = cpu_to_le32(iwl_trans_read32(fwrt->trans, addr + i));
1123
1124         return sizeof(*range) + le32_to_cpu(range->range_data_size);
1125 }
1126
1127 static int iwl_dump_ini_config_iter(struct iwl_fw_runtime *fwrt,
1128                                     struct iwl_dump_ini_region_data *reg_data,
1129                                     void *range_ptr, u32 range_len, int idx)
1130 {
1131         struct iwl_trans *trans = fwrt->trans;
1132         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1133         struct iwl_fw_ini_error_dump_range *range = range_ptr;
1134         __le32 *val = range->data;
1135         u32 addr = le32_to_cpu(reg->addrs[idx]) +
1136                    le32_to_cpu(reg->dev_addr.offset);
1137         int i;
1138
1139         /* we shouldn't get here if the trans doesn't have read_config32 */
1140         if (WARN_ON_ONCE(!trans->ops->read_config32))
1141                 return -EOPNOTSUPP;
1142
1143         range->internal_base_addr = cpu_to_le32(addr);
1144         range->range_data_size = reg->dev_addr.size;
1145         for (i = 0; i < le32_to_cpu(reg->dev_addr.size); i += 4) {
1146                 int ret;
1147                 u32 tmp;
1148
1149                 ret = trans->ops->read_config32(trans, addr + i, &tmp);
1150                 if (ret < 0)
1151                         return ret;
1152
1153                 *val++ = cpu_to_le32(tmp);
1154         }
1155
1156         return sizeof(*range) + le32_to_cpu(range->range_data_size);
1157 }
1158
1159 static int iwl_dump_ini_dev_mem_iter(struct iwl_fw_runtime *fwrt,
1160                                      struct iwl_dump_ini_region_data *reg_data,
1161                                      void *range_ptr, u32 range_len, int idx)
1162 {
1163         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1164         struct iwl_fw_ini_error_dump_range *range = range_ptr;
1165         u32 addr = le32_to_cpu(reg->addrs[idx]) +
1166                    le32_to_cpu(reg->dev_addr.offset);
1167
1168         range->internal_base_addr = cpu_to_le32(addr);
1169         range->range_data_size = reg->dev_addr.size;
1170         iwl_trans_read_mem_bytes(fwrt->trans, addr, range->data,
1171                                  le32_to_cpu(reg->dev_addr.size));
1172
1173         if (reg->sub_type == IWL_FW_INI_REGION_DEVICE_MEMORY_SUBTYPE_HW_SMEM &&
1174             fwrt->sanitize_ops && fwrt->sanitize_ops->frob_txf)
1175                 fwrt->sanitize_ops->frob_txf(fwrt->sanitize_ctx,
1176                                              range->data,
1177                                              le32_to_cpu(reg->dev_addr.size));
1178
1179         return sizeof(*range) + le32_to_cpu(range->range_data_size);
1180 }
1181
1182 static int _iwl_dump_ini_paging_iter(struct iwl_fw_runtime *fwrt,
1183                                      void *range_ptr, u32 range_len, int idx)
1184 {
1185         struct page *page = fwrt->fw_paging_db[idx].fw_paging_block;
1186         struct iwl_fw_ini_error_dump_range *range = range_ptr;
1187         dma_addr_t addr = fwrt->fw_paging_db[idx].fw_paging_phys;
1188         u32 page_size = fwrt->fw_paging_db[idx].fw_paging_size;
1189
1190         range->page_num = cpu_to_le32(idx);
1191         range->range_data_size = cpu_to_le32(page_size);
1192         dma_sync_single_for_cpu(fwrt->trans->dev, addr, page_size,
1193                                 DMA_BIDIRECTIONAL);
1194         memcpy(range->data, page_address(page), page_size);
1195         dma_sync_single_for_device(fwrt->trans->dev, addr, page_size,
1196                                    DMA_BIDIRECTIONAL);
1197
1198         return sizeof(*range) + le32_to_cpu(range->range_data_size);
1199 }
1200
1201 static int iwl_dump_ini_paging_iter(struct iwl_fw_runtime *fwrt,
1202                                     struct iwl_dump_ini_region_data *reg_data,
1203                                     void *range_ptr, u32 range_len, int idx)
1204 {
1205         struct iwl_fw_ini_error_dump_range *range;
1206         u32 page_size;
1207
1208         /* all paged index start from 1 to skip CSS section */
1209         idx++;
1210
1211         if (!fwrt->trans->trans_cfg->gen2)
1212                 return _iwl_dump_ini_paging_iter(fwrt, range_ptr, range_len, idx);
1213
1214         range = range_ptr;
1215         page_size = fwrt->trans->init_dram.paging[idx].size;
1216
1217         range->page_num = cpu_to_le32(idx);
1218         range->range_data_size = cpu_to_le32(page_size);
1219         memcpy(range->data, fwrt->trans->init_dram.paging[idx].block,
1220                page_size);
1221
1222         return sizeof(*range) + le32_to_cpu(range->range_data_size);
1223 }
1224
1225 static int
1226 iwl_dump_ini_mon_dram_iter(struct iwl_fw_runtime *fwrt,
1227                            struct iwl_dump_ini_region_data *reg_data,
1228                            void *range_ptr, u32 range_len, int idx)
1229 {
1230         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1231         struct iwl_fw_ini_error_dump_range *range = range_ptr;
1232         struct iwl_dram_data *frag;
1233         u32 alloc_id = le32_to_cpu(reg->dram_alloc_id);
1234
1235         frag = &fwrt->trans->dbg.fw_mon_ini[alloc_id].frags[idx];
1236
1237         range->dram_base_addr = cpu_to_le64(frag->physical);
1238         range->range_data_size = cpu_to_le32(frag->size);
1239
1240         memcpy(range->data, frag->block, frag->size);
1241
1242         return sizeof(*range) + le32_to_cpu(range->range_data_size);
1243 }
1244
1245 static int iwl_dump_ini_mon_smem_iter(struct iwl_fw_runtime *fwrt,
1246                                       struct iwl_dump_ini_region_data *reg_data,
1247                                       void *range_ptr, u32 range_len, int idx)
1248 {
1249         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1250         struct iwl_fw_ini_error_dump_range *range = range_ptr;
1251         u32 addr = le32_to_cpu(reg->internal_buffer.base_addr);
1252
1253         range->internal_base_addr = cpu_to_le32(addr);
1254         range->range_data_size = reg->internal_buffer.size;
1255         iwl_trans_read_mem_bytes(fwrt->trans, addr, range->data,
1256                                  le32_to_cpu(reg->internal_buffer.size));
1257
1258         return sizeof(*range) + le32_to_cpu(range->range_data_size);
1259 }
1260
1261 static bool iwl_ini_txf_iter(struct iwl_fw_runtime *fwrt,
1262                              struct iwl_dump_ini_region_data *reg_data, int idx)
1263 {
1264         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1265         struct iwl_txf_iter_data *iter = &fwrt->dump.txf_iter_data;
1266         struct iwl_fwrt_shared_mem_cfg *cfg = &fwrt->smem_cfg;
1267         int txf_num = cfg->num_txfifo_entries;
1268         int int_txf_num = ARRAY_SIZE(cfg->internal_txfifo_size);
1269         u32 lmac_bitmap = le32_to_cpu(reg->fifos.fid[0]);
1270
1271         if (!idx) {
1272                 if (le32_to_cpu(reg->fifos.offset) && cfg->num_lmacs == 1) {
1273                         IWL_ERR(fwrt, "WRT: Invalid lmac offset 0x%x\n",
1274                                 le32_to_cpu(reg->fifos.offset));
1275                         return false;
1276                 }
1277
1278                 iter->internal_txf = 0;
1279                 iter->fifo_size = 0;
1280                 iter->fifo = -1;
1281                 if (le32_to_cpu(reg->fifos.offset))
1282                         iter->lmac = 1;
1283                 else
1284                         iter->lmac = 0;
1285         }
1286
1287         if (!iter->internal_txf) {
1288                 for (iter->fifo++; iter->fifo < txf_num; iter->fifo++) {
1289                         iter->fifo_size =
1290                                 cfg->lmac[iter->lmac].txfifo_size[iter->fifo];
1291                         if (iter->fifo_size && (lmac_bitmap & BIT(iter->fifo)))
1292                                 return true;
1293                 }
1294                 iter->fifo--;
1295         }
1296
1297         iter->internal_txf = 1;
1298
1299         if (!fw_has_capa(&fwrt->fw->ucode_capa,
1300                          IWL_UCODE_TLV_CAPA_EXTEND_SHARED_MEM_CFG))
1301                 return false;
1302
1303         for (iter->fifo++; iter->fifo < int_txf_num + txf_num; iter->fifo++) {
1304                 iter->fifo_size =
1305                         cfg->internal_txfifo_size[iter->fifo - txf_num];
1306                 if (iter->fifo_size && (lmac_bitmap & BIT(iter->fifo)))
1307                         return true;
1308         }
1309
1310         return false;
1311 }
1312
1313 static int iwl_dump_ini_txf_iter(struct iwl_fw_runtime *fwrt,
1314                                  struct iwl_dump_ini_region_data *reg_data,
1315                                  void *range_ptr, u32 range_len, int idx)
1316 {
1317         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1318         struct iwl_fw_ini_error_dump_range *range = range_ptr;
1319         struct iwl_txf_iter_data *iter = &fwrt->dump.txf_iter_data;
1320         struct iwl_fw_ini_error_dump_register *reg_dump = (void *)range->data;
1321         u32 offs = le32_to_cpu(reg->fifos.offset), addr;
1322         u32 registers_num = iwl_tlv_array_len(reg_data->reg_tlv, reg, addrs);
1323         u32 registers_size = registers_num * sizeof(*reg_dump);
1324         __le32 *data;
1325         int i;
1326
1327         if (!iwl_ini_txf_iter(fwrt, reg_data, idx))
1328                 return -EIO;
1329
1330         if (!iwl_trans_grab_nic_access(fwrt->trans))
1331                 return -EBUSY;
1332
1333         range->fifo_hdr.fifo_num = cpu_to_le32(iter->fifo);
1334         range->fifo_hdr.num_of_registers = cpu_to_le32(registers_num);
1335         range->range_data_size = cpu_to_le32(iter->fifo_size + registers_size);
1336
1337         iwl_write_prph_no_grab(fwrt->trans, TXF_LARC_NUM + offs, iter->fifo);
1338
1339         /*
1340          * read txf registers. for each register, write to the dump the
1341          * register address and its value
1342          */
1343         for (i = 0; i < registers_num; i++) {
1344                 addr = le32_to_cpu(reg->addrs[i]) + offs;
1345
1346                 reg_dump->addr = cpu_to_le32(addr);
1347                 reg_dump->data = cpu_to_le32(iwl_read_prph_no_grab(fwrt->trans,
1348                                                                    addr));
1349
1350                 reg_dump++;
1351         }
1352
1353         if (reg->fifos.hdr_only) {
1354                 range->range_data_size = cpu_to_le32(registers_size);
1355                 goto out;
1356         }
1357
1358         /* Set the TXF_READ_MODIFY_ADDR to TXF_WR_PTR */
1359         iwl_write_prph_no_grab(fwrt->trans, TXF_READ_MODIFY_ADDR + offs,
1360                                TXF_WR_PTR + offs);
1361
1362         /* Dummy-read to advance the read pointer to the head */
1363         iwl_read_prph_no_grab(fwrt->trans, TXF_READ_MODIFY_DATA + offs);
1364
1365         /* Read FIFO */
1366         addr = TXF_READ_MODIFY_DATA + offs;
1367         data = (void *)reg_dump;
1368         for (i = 0; i < iter->fifo_size; i += sizeof(*data))
1369                 *data++ = cpu_to_le32(iwl_read_prph_no_grab(fwrt->trans, addr));
1370
1371         if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_txf)
1372                 fwrt->sanitize_ops->frob_txf(fwrt->sanitize_ctx,
1373                                              reg_dump, iter->fifo_size);
1374
1375 out:
1376         iwl_trans_release_nic_access(fwrt->trans);
1377
1378         return sizeof(*range) + le32_to_cpu(range->range_data_size);
1379 }
1380
1381 struct iwl_ini_rxf_data {
1382         u32 fifo_num;
1383         u32 size;
1384         u32 offset;
1385 };
1386
1387 static void iwl_ini_get_rxf_data(struct iwl_fw_runtime *fwrt,
1388                                  struct iwl_dump_ini_region_data *reg_data,
1389                                  struct iwl_ini_rxf_data *data)
1390 {
1391         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1392         u32 fid1 = le32_to_cpu(reg->fifos.fid[0]);
1393         u32 fid2 = le32_to_cpu(reg->fifos.fid[1]);
1394         u8 fifo_idx;
1395
1396         if (!data)
1397                 return;
1398
1399         /* make sure only one bit is set in only one fid */
1400         if (WARN_ONCE(hweight_long(fid1) + hweight_long(fid2) != 1,
1401                       "fid1=%x, fid2=%x\n", fid1, fid2))
1402                 return;
1403
1404         memset(data, 0, sizeof(*data));
1405
1406         if (fid1) {
1407                 fifo_idx = ffs(fid1) - 1;
1408                 if (WARN_ONCE(fifo_idx >= MAX_NUM_LMAC, "fifo_idx=%d\n",
1409                               fifo_idx))
1410                         return;
1411
1412                 data->size = fwrt->smem_cfg.lmac[fifo_idx].rxfifo1_size;
1413                 data->fifo_num = fifo_idx;
1414         } else {
1415                 u8 max_idx;
1416
1417                 fifo_idx = ffs(fid2) - 1;
1418                 if (iwl_fw_lookup_notif_ver(fwrt->fw, SYSTEM_GROUP,
1419                                             SHARED_MEM_CFG_CMD, 0) <= 3)
1420                         max_idx = 0;
1421                 else
1422                         max_idx = 1;
1423
1424                 if (WARN_ONCE(fifo_idx > max_idx,
1425                               "invalid umac fifo idx %d", fifo_idx))
1426                         return;
1427
1428                 /* use bit 31 to distinguish between umac and lmac rxf while
1429                  * parsing the dump
1430                  */
1431                 data->fifo_num = fifo_idx | IWL_RXF_UMAC_BIT;
1432
1433                 switch (fifo_idx) {
1434                 case 0:
1435                         data->size = fwrt->smem_cfg.rxfifo2_size;
1436                         data->offset = iwl_umac_prph(fwrt->trans,
1437                                                      RXF_DIFF_FROM_PREV);
1438                         break;
1439                 case 1:
1440                         data->size = fwrt->smem_cfg.rxfifo2_control_size;
1441                         data->offset = iwl_umac_prph(fwrt->trans,
1442                                                      RXF2C_DIFF_FROM_PREV);
1443                         break;
1444                 }
1445         }
1446 }
1447
1448 static int iwl_dump_ini_rxf_iter(struct iwl_fw_runtime *fwrt,
1449                                  struct iwl_dump_ini_region_data *reg_data,
1450                                  void *range_ptr, u32 range_len, int idx)
1451 {
1452         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1453         struct iwl_fw_ini_error_dump_range *range = range_ptr;
1454         struct iwl_ini_rxf_data rxf_data;
1455         struct iwl_fw_ini_error_dump_register *reg_dump = (void *)range->data;
1456         u32 offs = le32_to_cpu(reg->fifos.offset), addr;
1457         u32 registers_num = iwl_tlv_array_len(reg_data->reg_tlv, reg, addrs);
1458         u32 registers_size = registers_num * sizeof(*reg_dump);
1459         __le32 *data;
1460         int i;
1461
1462         iwl_ini_get_rxf_data(fwrt, reg_data, &rxf_data);
1463         if (!rxf_data.size)
1464                 return -EIO;
1465
1466         if (!iwl_trans_grab_nic_access(fwrt->trans))
1467                 return -EBUSY;
1468
1469         range->fifo_hdr.fifo_num = cpu_to_le32(rxf_data.fifo_num);
1470         range->fifo_hdr.num_of_registers = cpu_to_le32(registers_num);
1471         range->range_data_size = cpu_to_le32(rxf_data.size + registers_size);
1472
1473         /*
1474          * read rxf registers. for each register, write to the dump the
1475          * register address and its value
1476          */
1477         for (i = 0; i < registers_num; i++) {
1478                 addr = le32_to_cpu(reg->addrs[i]) + offs;
1479
1480                 reg_dump->addr = cpu_to_le32(addr);
1481                 reg_dump->data = cpu_to_le32(iwl_read_prph_no_grab(fwrt->trans,
1482                                                                    addr));
1483
1484                 reg_dump++;
1485         }
1486
1487         if (reg->fifos.hdr_only) {
1488                 range->range_data_size = cpu_to_le32(registers_size);
1489                 goto out;
1490         }
1491
1492         offs = rxf_data.offset;
1493
1494         /* Lock fence */
1495         iwl_write_prph_no_grab(fwrt->trans, RXF_SET_FENCE_MODE + offs, 0x1);
1496         /* Set fence pointer to the same place like WR pointer */
1497         iwl_write_prph_no_grab(fwrt->trans, RXF_LD_WR2FENCE + offs, 0x1);
1498         /* Set fence offset */
1499         iwl_write_prph_no_grab(fwrt->trans, RXF_LD_FENCE_OFFSET_ADDR + offs,
1500                                0x0);
1501
1502         /* Read FIFO */
1503         addr =  RXF_FIFO_RD_FENCE_INC + offs;
1504         data = (void *)reg_dump;
1505         for (i = 0; i < rxf_data.size; i += sizeof(*data))
1506                 *data++ = cpu_to_le32(iwl_read_prph_no_grab(fwrt->trans, addr));
1507
1508 out:
1509         iwl_trans_release_nic_access(fwrt->trans);
1510
1511         return sizeof(*range) + le32_to_cpu(range->range_data_size);
1512 }
1513
1514 static int
1515 iwl_dump_ini_err_table_iter(struct iwl_fw_runtime *fwrt,
1516                             struct iwl_dump_ini_region_data *reg_data,
1517                             void *range_ptr, u32 range_len, int idx)
1518 {
1519         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1520         struct iwl_fw_ini_region_err_table *err_table = &reg->err_table;
1521         struct iwl_fw_ini_error_dump_range *range = range_ptr;
1522         u32 addr = le32_to_cpu(err_table->base_addr) +
1523                    le32_to_cpu(err_table->offset);
1524
1525         range->internal_base_addr = cpu_to_le32(addr);
1526         range->range_data_size = err_table->size;
1527         iwl_trans_read_mem_bytes(fwrt->trans, addr, range->data,
1528                                  le32_to_cpu(err_table->size));
1529
1530         return sizeof(*range) + le32_to_cpu(range->range_data_size);
1531 }
1532
1533 static int
1534 iwl_dump_ini_special_mem_iter(struct iwl_fw_runtime *fwrt,
1535                               struct iwl_dump_ini_region_data *reg_data,
1536                               void *range_ptr, u32 range_len, int idx)
1537 {
1538         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1539         struct iwl_fw_ini_region_special_device_memory *special_mem =
1540                 &reg->special_mem;
1541
1542         struct iwl_fw_ini_error_dump_range *range = range_ptr;
1543         u32 addr = le32_to_cpu(special_mem->base_addr) +
1544                    le32_to_cpu(special_mem->offset);
1545
1546         range->internal_base_addr = cpu_to_le32(addr);
1547         range->range_data_size = special_mem->size;
1548         iwl_trans_read_mem_bytes(fwrt->trans, addr, range->data,
1549                                  le32_to_cpu(special_mem->size));
1550
1551         return sizeof(*range) + le32_to_cpu(range->range_data_size);
1552 }
1553
1554 static int
1555 iwl_dump_ini_dbgi_sram_iter(struct iwl_fw_runtime *fwrt,
1556                             struct iwl_dump_ini_region_data *reg_data,
1557                             void *range_ptr, u32 range_len, int idx)
1558 {
1559         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1560         struct iwl_fw_ini_error_dump_range *range = range_ptr;
1561         __le32 *val = range->data;
1562         u32 prph_data;
1563         int i;
1564
1565         if (!iwl_trans_grab_nic_access(fwrt->trans))
1566                 return -EBUSY;
1567
1568         range->range_data_size = reg->dev_addr.size;
1569         for (i = 0; i < (le32_to_cpu(reg->dev_addr.size) / 4); i++) {
1570                 prph_data = iwl_read_prph_no_grab(fwrt->trans, (i % 2) ?
1571                                           DBGI_SRAM_TARGET_ACCESS_RDATA_MSB :
1572                                           DBGI_SRAM_TARGET_ACCESS_RDATA_LSB);
1573                 if (prph_data == 0x5a5a5a5a) {
1574                         iwl_trans_release_nic_access(fwrt->trans);
1575                         return -EBUSY;
1576                 }
1577                 *val++ = cpu_to_le32(prph_data);
1578         }
1579         iwl_trans_release_nic_access(fwrt->trans);
1580         return sizeof(*range) + le32_to_cpu(range->range_data_size);
1581 }
1582
1583 static int iwl_dump_ini_fw_pkt_iter(struct iwl_fw_runtime *fwrt,
1584                                     struct iwl_dump_ini_region_data *reg_data,
1585                                     void *range_ptr, u32 range_len, int idx)
1586 {
1587         struct iwl_fw_ini_error_dump_range *range = range_ptr;
1588         struct iwl_rx_packet *pkt = reg_data->dump_data->fw_pkt;
1589         u32 pkt_len;
1590
1591         if (!pkt)
1592                 return -EIO;
1593
1594         pkt_len = iwl_rx_packet_payload_len(pkt);
1595
1596         memcpy(&range->fw_pkt_hdr, &pkt->hdr, sizeof(range->fw_pkt_hdr));
1597         range->range_data_size = cpu_to_le32(pkt_len);
1598
1599         memcpy(range->data, pkt->data, pkt_len);
1600
1601         return sizeof(*range) + le32_to_cpu(range->range_data_size);
1602 }
1603
1604 static int iwl_dump_ini_imr_iter(struct iwl_fw_runtime *fwrt,
1605                                  struct iwl_dump_ini_region_data *reg_data,
1606                                  void *range_ptr, u32 range_len, int idx)
1607 {
1608         /* read the IMR memory and DMA it to SRAM */
1609         struct iwl_fw_ini_error_dump_range *range = range_ptr;
1610         u64 imr_curr_addr = fwrt->trans->dbg.imr_data.imr_curr_addr;
1611         u32 imr_rem_bytes = fwrt->trans->dbg.imr_data.imr2sram_remainbyte;
1612         u32 sram_addr = fwrt->trans->dbg.imr_data.sram_addr;
1613         u32 sram_size = fwrt->trans->dbg.imr_data.sram_size;
1614         u32 size_to_dump = (imr_rem_bytes > sram_size) ? sram_size : imr_rem_bytes;
1615
1616         range->range_data_size = cpu_to_le32(size_to_dump);
1617         if (iwl_trans_write_imr_mem(fwrt->trans, sram_addr,
1618                                     imr_curr_addr, size_to_dump)) {
1619                 IWL_ERR(fwrt, "WRT_DEBUG: IMR Memory transfer failed\n");
1620                 return -1;
1621         }
1622
1623         fwrt->trans->dbg.imr_data.imr_curr_addr = imr_curr_addr + size_to_dump;
1624         fwrt->trans->dbg.imr_data.imr2sram_remainbyte -= size_to_dump;
1625
1626         iwl_trans_read_mem_bytes(fwrt->trans, sram_addr, range->data,
1627                                  size_to_dump);
1628         return sizeof(*range) + le32_to_cpu(range->range_data_size);
1629 }
1630
1631 static void *
1632 iwl_dump_ini_mem_fill_header(struct iwl_fw_runtime *fwrt,
1633                              struct iwl_dump_ini_region_data *reg_data,
1634                              void *data, u32 data_len)
1635 {
1636         struct iwl_fw_ini_error_dump *dump = data;
1637
1638         dump->header.version = cpu_to_le32(IWL_INI_DUMP_VER);
1639
1640         return dump->data;
1641 }
1642
1643 /**
1644  * mask_apply_and_normalize - applies mask on val and normalize the result
1645  *
1646  * The normalization is based on the first set bit in the mask
1647  *
1648  * @val: value
1649  * @mask: mask to apply and to normalize with
1650  */
1651 static u32 mask_apply_and_normalize(u32 val, u32 mask)
1652 {
1653         return (val & mask) >> (ffs(mask) - 1);
1654 }
1655
1656 static __le32 iwl_get_mon_reg(struct iwl_fw_runtime *fwrt, u32 alloc_id,
1657                               const struct iwl_fw_mon_reg *reg_info)
1658 {
1659         u32 val, offs;
1660
1661         /* The header addresses of DBGCi is calculate as follows:
1662          * DBGC1 address + (0x100 * i)
1663          */
1664         offs = (alloc_id - IWL_FW_INI_ALLOCATION_ID_DBGC1) * 0x100;
1665
1666         if (!reg_info || !reg_info->addr || !reg_info->mask)
1667                 return 0;
1668
1669         val = iwl_read_prph_no_grab(fwrt->trans, reg_info->addr + offs);
1670
1671         return cpu_to_le32(mask_apply_and_normalize(val, reg_info->mask));
1672 }
1673
1674 static void *
1675 iwl_dump_ini_mon_fill_header(struct iwl_fw_runtime *fwrt,
1676                              struct iwl_dump_ini_region_data *reg_data,
1677                              struct iwl_fw_ini_monitor_dump *data,
1678                              const struct iwl_fw_mon_regs *addrs)
1679 {
1680         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1681         u32 alloc_id = le32_to_cpu(reg->dram_alloc_id);
1682
1683         if (!iwl_trans_grab_nic_access(fwrt->trans)) {
1684                 IWL_ERR(fwrt, "Failed to get monitor header\n");
1685                 return NULL;
1686         }
1687
1688         data->write_ptr = iwl_get_mon_reg(fwrt, alloc_id,
1689                                           &addrs->write_ptr);
1690         if (fwrt->trans->trans_cfg->device_family >= IWL_DEVICE_FAMILY_AX210) {
1691                 u32 wrt_ptr = le32_to_cpu(data->write_ptr);
1692
1693                 data->write_ptr = cpu_to_le32(wrt_ptr >> 2);
1694         }
1695         data->cycle_cnt = iwl_get_mon_reg(fwrt, alloc_id,
1696                                           &addrs->cycle_cnt);
1697         data->cur_frag = iwl_get_mon_reg(fwrt, alloc_id,
1698                                          &addrs->cur_frag);
1699
1700         iwl_trans_release_nic_access(fwrt->trans);
1701
1702         data->header.version = cpu_to_le32(IWL_INI_DUMP_VER);
1703
1704         return data->data;
1705 }
1706
1707 static void *
1708 iwl_dump_ini_mon_dram_fill_header(struct iwl_fw_runtime *fwrt,
1709                                   struct iwl_dump_ini_region_data *reg_data,
1710                                   void *data, u32 data_len)
1711 {
1712         struct iwl_fw_ini_monitor_dump *mon_dump = (void *)data;
1713
1714         return iwl_dump_ini_mon_fill_header(fwrt, reg_data, mon_dump,
1715                                             &fwrt->trans->cfg->mon_dram_regs);
1716 }
1717
1718 static void *
1719 iwl_dump_ini_mon_smem_fill_header(struct iwl_fw_runtime *fwrt,
1720                                   struct iwl_dump_ini_region_data *reg_data,
1721                                   void *data, u32 data_len)
1722 {
1723         struct iwl_fw_ini_monitor_dump *mon_dump = (void *)data;
1724
1725         return iwl_dump_ini_mon_fill_header(fwrt, reg_data, mon_dump,
1726                                             &fwrt->trans->cfg->mon_smem_regs);
1727 }
1728
1729 static void *
1730 iwl_dump_ini_mon_dbgi_fill_header(struct iwl_fw_runtime *fwrt,
1731                                   struct iwl_dump_ini_region_data *reg_data,
1732                                   void *data, u32 data_len)
1733 {
1734         struct iwl_fw_ini_monitor_dump *mon_dump = (void *)data;
1735
1736         return iwl_dump_ini_mon_fill_header(fwrt, reg_data, mon_dump,
1737                                             &fwrt->trans->cfg->mon_dbgi_regs);
1738 }
1739
1740 static void *
1741 iwl_dump_ini_err_table_fill_header(struct iwl_fw_runtime *fwrt,
1742                                    struct iwl_dump_ini_region_data *reg_data,
1743                                    void *data, u32 data_len)
1744 {
1745         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1746         struct iwl_fw_ini_err_table_dump *dump = data;
1747
1748         dump->header.version = cpu_to_le32(IWL_INI_DUMP_VER);
1749         dump->version = reg->err_table.version;
1750
1751         return dump->data;
1752 }
1753
1754 static void *
1755 iwl_dump_ini_special_mem_fill_header(struct iwl_fw_runtime *fwrt,
1756                                      struct iwl_dump_ini_region_data *reg_data,
1757                                      void *data, u32 data_len)
1758 {
1759         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1760         struct iwl_fw_ini_special_device_memory *dump = data;
1761
1762         dump->header.version = cpu_to_le32(IWL_INI_DUMP_VER);
1763         dump->type = reg->special_mem.type;
1764         dump->version = reg->special_mem.version;
1765
1766         return dump->data;
1767 }
1768
1769 static void *
1770 iwl_dump_ini_imr_fill_header(struct iwl_fw_runtime *fwrt,
1771                              struct iwl_dump_ini_region_data *reg_data,
1772                              void *data, u32 data_len)
1773 {
1774         struct iwl_fw_ini_error_dump *dump = data;
1775
1776         dump->header.version = cpu_to_le32(IWL_INI_DUMP_VER);
1777
1778         return dump->data;
1779 }
1780
1781 static u32 iwl_dump_ini_mem_ranges(struct iwl_fw_runtime *fwrt,
1782                                    struct iwl_dump_ini_region_data *reg_data)
1783 {
1784         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1785
1786         return iwl_tlv_array_len(reg_data->reg_tlv, reg, addrs);
1787 }
1788
1789 static u32 iwl_dump_ini_paging_ranges(struct iwl_fw_runtime *fwrt,
1790                                       struct iwl_dump_ini_region_data *reg_data)
1791 {
1792         if (fwrt->trans->trans_cfg->gen2) {
1793                 if (fwrt->trans->init_dram.paging_cnt)
1794                         return fwrt->trans->init_dram.paging_cnt - 1;
1795                 else
1796                         return 0;
1797         }
1798
1799         return fwrt->num_of_paging_blk;
1800 }
1801
1802 static u32
1803 iwl_dump_ini_mon_dram_ranges(struct iwl_fw_runtime *fwrt,
1804                              struct iwl_dump_ini_region_data *reg_data)
1805 {
1806         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1807         struct iwl_fw_mon *fw_mon;
1808         u32 ranges = 0, alloc_id = le32_to_cpu(reg->dram_alloc_id);
1809         int i;
1810
1811         fw_mon = &fwrt->trans->dbg.fw_mon_ini[alloc_id];
1812
1813         for (i = 0; i < fw_mon->num_frags; i++) {
1814                 if (!fw_mon->frags[i].size)
1815                         break;
1816
1817                 ranges++;
1818         }
1819
1820         return ranges;
1821 }
1822
1823 static u32 iwl_dump_ini_txf_ranges(struct iwl_fw_runtime *fwrt,
1824                                    struct iwl_dump_ini_region_data *reg_data)
1825 {
1826         u32 num_of_fifos = 0;
1827
1828         while (iwl_ini_txf_iter(fwrt, reg_data, num_of_fifos))
1829                 num_of_fifos++;
1830
1831         return num_of_fifos;
1832 }
1833
1834 static u32 iwl_dump_ini_single_range(struct iwl_fw_runtime *fwrt,
1835                                      struct iwl_dump_ini_region_data *reg_data)
1836 {
1837         return 1;
1838 }
1839
1840 static u32 iwl_dump_ini_imr_ranges(struct iwl_fw_runtime *fwrt,
1841                                    struct iwl_dump_ini_region_data *reg_data)
1842 {
1843         /* range is total number of pages need to copied from
1844          *IMR memory to SRAM and later from SRAM to DRAM
1845          */
1846         u32 imr_enable = fwrt->trans->dbg.imr_data.imr_enable;
1847         u32 imr_size = fwrt->trans->dbg.imr_data.imr_size;
1848         u32 sram_size = fwrt->trans->dbg.imr_data.sram_size;
1849
1850         if (imr_enable == 0 || imr_size == 0 || sram_size == 0) {
1851                 IWL_DEBUG_INFO(fwrt,
1852                                "WRT: Invalid imr data enable: %d, imr_size: %d, sram_size: %d\n",
1853                                imr_enable, imr_size, sram_size);
1854                 return 0;
1855         }
1856
1857         return((imr_size % sram_size) ? (imr_size / sram_size + 1) : (imr_size / sram_size));
1858 }
1859
1860 static u32 iwl_dump_ini_mem_get_size(struct iwl_fw_runtime *fwrt,
1861                                      struct iwl_dump_ini_region_data *reg_data)
1862 {
1863         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1864         u32 size = le32_to_cpu(reg->dev_addr.size);
1865         u32 ranges = iwl_dump_ini_mem_ranges(fwrt, reg_data);
1866
1867         if (!size || !ranges)
1868                 return 0;
1869
1870         return sizeof(struct iwl_fw_ini_error_dump) + ranges *
1871                 (size + sizeof(struct iwl_fw_ini_error_dump_range));
1872 }
1873
1874 static u32
1875 iwl_dump_ini_paging_get_size(struct iwl_fw_runtime *fwrt,
1876                              struct iwl_dump_ini_region_data *reg_data)
1877 {
1878         int i;
1879         u32 range_header_len = sizeof(struct iwl_fw_ini_error_dump_range);
1880         u32 size = sizeof(struct iwl_fw_ini_error_dump);
1881
1882         /* start from 1 to skip CSS section */
1883         for (i = 1; i <= iwl_dump_ini_paging_ranges(fwrt, reg_data); i++) {
1884                 size += range_header_len;
1885                 if (fwrt->trans->trans_cfg->gen2)
1886                         size += fwrt->trans->init_dram.paging[i].size;
1887                 else
1888                         size += fwrt->fw_paging_db[i].fw_paging_size;
1889         }
1890
1891         return size;
1892 }
1893
1894 static u32
1895 iwl_dump_ini_mon_dram_get_size(struct iwl_fw_runtime *fwrt,
1896                                struct iwl_dump_ini_region_data *reg_data)
1897 {
1898         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1899         struct iwl_fw_mon *fw_mon;
1900         u32 size = 0, alloc_id = le32_to_cpu(reg->dram_alloc_id);
1901         int i;
1902
1903         fw_mon = &fwrt->trans->dbg.fw_mon_ini[alloc_id];
1904
1905         for (i = 0; i < fw_mon->num_frags; i++) {
1906                 struct iwl_dram_data *frag = &fw_mon->frags[i];
1907
1908                 if (!frag->size)
1909                         break;
1910
1911                 size += sizeof(struct iwl_fw_ini_error_dump_range) + frag->size;
1912         }
1913
1914         if (size)
1915                 size += sizeof(struct iwl_fw_ini_monitor_dump);
1916
1917         return size;
1918 }
1919
1920 static u32
1921 iwl_dump_ini_mon_smem_get_size(struct iwl_fw_runtime *fwrt,
1922                                struct iwl_dump_ini_region_data *reg_data)
1923 {
1924         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1925         u32 size;
1926
1927         size = le32_to_cpu(reg->internal_buffer.size);
1928         if (!size)
1929                 return 0;
1930
1931         size += sizeof(struct iwl_fw_ini_monitor_dump) +
1932                 sizeof(struct iwl_fw_ini_error_dump_range);
1933
1934         return size;
1935 }
1936
1937 static u32 iwl_dump_ini_mon_dbgi_get_size(struct iwl_fw_runtime *fwrt,
1938                                           struct iwl_dump_ini_region_data *reg_data)
1939 {
1940         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1941         u32 size = le32_to_cpu(reg->dev_addr.size);
1942         u32 ranges = iwl_dump_ini_mem_ranges(fwrt, reg_data);
1943
1944         if (!size || !ranges)
1945                 return 0;
1946
1947         return sizeof(struct iwl_fw_ini_monitor_dump) + ranges *
1948                 (size + sizeof(struct iwl_fw_ini_error_dump_range));
1949 }
1950
1951 static u32 iwl_dump_ini_txf_get_size(struct iwl_fw_runtime *fwrt,
1952                                      struct iwl_dump_ini_region_data *reg_data)
1953 {
1954         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1955         struct iwl_txf_iter_data *iter = &fwrt->dump.txf_iter_data;
1956         u32 registers_num = iwl_tlv_array_len(reg_data->reg_tlv, reg, addrs);
1957         u32 size = 0;
1958         u32 fifo_hdr = sizeof(struct iwl_fw_ini_error_dump_range) +
1959                        registers_num *
1960                        sizeof(struct iwl_fw_ini_error_dump_register);
1961
1962         while (iwl_ini_txf_iter(fwrt, reg_data, size)) {
1963                 size += fifo_hdr;
1964                 if (!reg->fifos.hdr_only)
1965                         size += iter->fifo_size;
1966         }
1967
1968         if (!size)
1969                 return 0;
1970
1971         return size + sizeof(struct iwl_fw_ini_error_dump);
1972 }
1973
1974 static u32 iwl_dump_ini_rxf_get_size(struct iwl_fw_runtime *fwrt,
1975                                      struct iwl_dump_ini_region_data *reg_data)
1976 {
1977         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1978         struct iwl_ini_rxf_data rx_data;
1979         u32 registers_num = iwl_tlv_array_len(reg_data->reg_tlv, reg, addrs);
1980         u32 size = sizeof(struct iwl_fw_ini_error_dump) +
1981                 sizeof(struct iwl_fw_ini_error_dump_range) +
1982                 registers_num * sizeof(struct iwl_fw_ini_error_dump_register);
1983
1984         if (reg->fifos.hdr_only)
1985                 return size;
1986
1987         iwl_ini_get_rxf_data(fwrt, reg_data, &rx_data);
1988         size += rx_data.size;
1989
1990         return size;
1991 }
1992
1993 static u32
1994 iwl_dump_ini_err_table_get_size(struct iwl_fw_runtime *fwrt,
1995                                 struct iwl_dump_ini_region_data *reg_data)
1996 {
1997         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
1998         u32 size = le32_to_cpu(reg->err_table.size);
1999
2000         if (size)
2001                 size += sizeof(struct iwl_fw_ini_err_table_dump) +
2002                         sizeof(struct iwl_fw_ini_error_dump_range);
2003
2004         return size;
2005 }
2006
2007 static u32
2008 iwl_dump_ini_special_mem_get_size(struct iwl_fw_runtime *fwrt,
2009                                   struct iwl_dump_ini_region_data *reg_data)
2010 {
2011         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
2012         u32 size = le32_to_cpu(reg->special_mem.size);
2013
2014         if (size)
2015                 size += sizeof(struct iwl_fw_ini_special_device_memory) +
2016                         sizeof(struct iwl_fw_ini_error_dump_range);
2017
2018         return size;
2019 }
2020
2021 static u32
2022 iwl_dump_ini_fw_pkt_get_size(struct iwl_fw_runtime *fwrt,
2023                              struct iwl_dump_ini_region_data *reg_data)
2024 {
2025         u32 size = 0;
2026
2027         if (!reg_data->dump_data->fw_pkt)
2028                 return 0;
2029
2030         size += iwl_rx_packet_payload_len(reg_data->dump_data->fw_pkt);
2031         if (size)
2032                 size += sizeof(struct iwl_fw_ini_error_dump) +
2033                         sizeof(struct iwl_fw_ini_error_dump_range);
2034
2035         return size;
2036 }
2037
2038 static u32
2039 iwl_dump_ini_imr_get_size(struct iwl_fw_runtime *fwrt,
2040                           struct iwl_dump_ini_region_data *reg_data)
2041 {
2042         u32 size = 0;
2043         u32 ranges = 0;
2044         u32 imr_enable = fwrt->trans->dbg.imr_data.imr_enable;
2045         u32 imr_size = fwrt->trans->dbg.imr_data.imr_size;
2046         u32 sram_size = fwrt->trans->dbg.imr_data.sram_size;
2047
2048         if (imr_enable == 0 || imr_size == 0 || sram_size == 0) {
2049                 IWL_DEBUG_INFO(fwrt,
2050                                "WRT: Invalid imr data enable: %d, imr_size: %d, sram_size: %d\n",
2051                                imr_enable, imr_size, sram_size);
2052                 return size;
2053         }
2054         size = imr_size;
2055         ranges = iwl_dump_ini_imr_ranges(fwrt, reg_data);
2056         if (!size && !ranges) {
2057                 IWL_ERR(fwrt, "WRT: imr_size :=%d, ranges :=%d\n", size, ranges);
2058                 return 0;
2059         }
2060         size += sizeof(struct iwl_fw_ini_error_dump) +
2061                 ranges * sizeof(struct iwl_fw_ini_error_dump_range);
2062         return size;
2063 }
2064
2065 /**
2066  * struct iwl_dump_ini_mem_ops - ini memory dump operations
2067  * @get_num_of_ranges: returns the number of memory ranges in the region.
2068  * @get_size: returns the total size of the region.
2069  * @fill_mem_hdr: fills region type specific headers and returns pointer to
2070  *      the first range or NULL if failed to fill headers.
2071  * @fill_range: copies a given memory range into the dump.
2072  *      Returns the size of the range or negative error value otherwise.
2073  */
2074 struct iwl_dump_ini_mem_ops {
2075         u32 (*get_num_of_ranges)(struct iwl_fw_runtime *fwrt,
2076                                  struct iwl_dump_ini_region_data *reg_data);
2077         u32 (*get_size)(struct iwl_fw_runtime *fwrt,
2078                         struct iwl_dump_ini_region_data *reg_data);
2079         void *(*fill_mem_hdr)(struct iwl_fw_runtime *fwrt,
2080                               struct iwl_dump_ini_region_data *reg_data,
2081                               void *data, u32 data_len);
2082         int (*fill_range)(struct iwl_fw_runtime *fwrt,
2083                           struct iwl_dump_ini_region_data *reg_data,
2084                           void *range, u32 range_len, int idx);
2085 };
2086
2087 /**
2088  * iwl_dump_ini_mem
2089  *
2090  * Creates a dump tlv and copy a memory region into it.
2091  * Returns the size of the current dump tlv or 0 if failed
2092  *
2093  * @fwrt: fw runtime struct
2094  * @list: list to add the dump tlv to
2095  * @reg_data: memory region
2096  * @ops: memory dump operations
2097  */
2098 static u32 iwl_dump_ini_mem(struct iwl_fw_runtime *fwrt, struct list_head *list,
2099                             struct iwl_dump_ini_region_data *reg_data,
2100                             const struct iwl_dump_ini_mem_ops *ops)
2101 {
2102         struct iwl_fw_ini_region_tlv *reg = (void *)reg_data->reg_tlv->data;
2103         struct iwl_fw_ini_dump_entry *entry;
2104         struct iwl_fw_ini_error_dump_data *tlv;
2105         struct iwl_fw_ini_error_dump_header *header;
2106         u32 type = reg->type;
2107         u32 id = le32_get_bits(reg->id, IWL_FW_INI_REGION_ID_MASK);
2108         u32 num_of_ranges, i, size;
2109         u8 *range;
2110         u32 free_size;
2111         u64 header_size;
2112         u32 dump_policy = IWL_FW_INI_DUMP_VERBOSE;
2113
2114         IWL_DEBUG_FW(fwrt, "WRT: Collecting region: dump type=%d, id=%d, type=%d\n",
2115                      dump_policy, id, type);
2116
2117         if (le32_to_cpu(reg->hdr.version) >= 2) {
2118                 u32 dp = le32_get_bits(reg->id,
2119                                        IWL_FW_INI_REGION_DUMP_POLICY_MASK);
2120
2121                 if (dump_policy == IWL_FW_INI_DUMP_VERBOSE &&
2122                     !(dp & IWL_FW_INI_DEBUG_DUMP_POLICY_NO_LIMIT)) {
2123                         IWL_DEBUG_FW(fwrt,
2124                                      "WRT: no dump - type %d and policy mismatch=%d\n",
2125                                      dump_policy, dp);
2126                         return 0;
2127                 } else if (dump_policy == IWL_FW_INI_DUMP_MEDIUM &&
2128                            !(dp & IWL_FW_IWL_DEBUG_DUMP_POLICY_MAX_LIMIT_5MB)) {
2129                         IWL_DEBUG_FW(fwrt,
2130                                      "WRT: no dump - type %d and policy mismatch=%d\n",
2131                                      dump_policy, dp);
2132                         return 0;
2133                 } else if (dump_policy == IWL_FW_INI_DUMP_BRIEF &&
2134                            !(dp & IWL_FW_INI_DEBUG_DUMP_POLICY_MAX_LIMIT_600KB)) {
2135                         IWL_DEBUG_FW(fwrt,
2136                                      "WRT: no dump - type %d and policy mismatch=%d\n",
2137                                      dump_policy, dp);
2138                         return 0;
2139                 }
2140         }
2141
2142         if (!ops->get_num_of_ranges || !ops->get_size || !ops->fill_mem_hdr ||
2143             !ops->fill_range) {
2144                 IWL_DEBUG_FW(fwrt, "WRT: no ops for collecting data\n");
2145                 return 0;
2146         }
2147
2148         size = ops->get_size(fwrt, reg_data);
2149
2150         if (size < sizeof(*header)) {
2151                 IWL_DEBUG_FW(fwrt, "WRT: size didn't include space for header\n");
2152                 return 0;
2153         }
2154
2155         entry = vzalloc(sizeof(*entry) + sizeof(*tlv) + size);
2156         if (!entry)
2157                 return 0;
2158
2159         entry->size = sizeof(*tlv) + size;
2160
2161         tlv = (void *)entry->data;
2162         tlv->type = reg->type;
2163         tlv->sub_type = reg->sub_type;
2164         tlv->sub_type_ver = reg->sub_type_ver;
2165         tlv->reserved = reg->reserved;
2166         tlv->len = cpu_to_le32(size);
2167
2168         num_of_ranges = ops->get_num_of_ranges(fwrt, reg_data);
2169
2170         header = (void *)tlv->data;
2171         header->region_id = cpu_to_le32(id);
2172         header->num_of_ranges = cpu_to_le32(num_of_ranges);
2173         header->name_len = cpu_to_le32(IWL_FW_INI_MAX_NAME);
2174         memcpy(header->name, reg->name, IWL_FW_INI_MAX_NAME);
2175
2176         free_size = size;
2177         range = ops->fill_mem_hdr(fwrt, reg_data, header, free_size);
2178         if (!range) {
2179                 IWL_ERR(fwrt,
2180                         "WRT: Failed to fill region header: id=%d, type=%d\n",
2181                         id, type);
2182                 goto out_err;
2183         }
2184
2185         header_size = range - (u8 *)header;
2186
2187         if (WARN(header_size > free_size,
2188 #if defined(__linux__)
2189                  "header size %llu > free_size %d",
2190                  header_size, free_size)) {
2191 #elif defined(__FreeBSD__)
2192                  "header size %ju > free_size %d",
2193                  (uintmax_t)header_size, free_size)) {
2194 #endif
2195                 IWL_ERR(fwrt,
2196                         "WRT: fill_mem_hdr used more than given free_size\n");
2197                 goto out_err;
2198         }
2199
2200         free_size -= header_size;
2201
2202         for (i = 0; i < num_of_ranges; i++) {
2203                 int range_size = ops->fill_range(fwrt, reg_data, range,
2204                                                  free_size, i);
2205
2206                 if (range_size < 0) {
2207                         IWL_ERR(fwrt,
2208                                 "WRT: Failed to dump region: id=%d, type=%d\n",
2209                                 id, type);
2210                         goto out_err;
2211                 }
2212
2213                 if (WARN(range_size > free_size, "range_size %d > free_size %d",
2214                          range_size, free_size)) {
2215                         IWL_ERR(fwrt,
2216                                 "WRT: fill_raged used more than given free_size\n");
2217                         goto out_err;
2218                 }
2219
2220                 free_size -= range_size;
2221                 range = range + range_size;
2222         }
2223
2224         list_add_tail(&entry->list, list);
2225
2226         return entry->size;
2227
2228 out_err:
2229         vfree(entry);
2230
2231         return 0;
2232 }
2233
2234 static u32 iwl_dump_ini_info(struct iwl_fw_runtime *fwrt,
2235                              struct iwl_fw_ini_trigger_tlv *trigger,
2236                              struct list_head *list)
2237 {
2238         struct iwl_fw_ini_dump_entry *entry;
2239         struct iwl_fw_error_dump_data *tlv;
2240         struct iwl_fw_ini_dump_info *dump;
2241         struct iwl_dbg_tlv_node *node;
2242         struct iwl_fw_ini_dump_cfg_name *cfg_name;
2243         u32 size = sizeof(*tlv) + sizeof(*dump);
2244         u32 num_of_cfg_names = 0;
2245         u32 hw_type;
2246
2247         list_for_each_entry(node, &fwrt->trans->dbg.debug_info_tlv_list, list) {
2248                 size += sizeof(*cfg_name);
2249                 num_of_cfg_names++;
2250         }
2251
2252         entry = vzalloc(sizeof(*entry) + size);
2253         if (!entry)
2254                 return 0;
2255
2256         entry->size = size;
2257
2258         tlv = (void *)entry->data;
2259         tlv->type = cpu_to_le32(IWL_INI_DUMP_INFO_TYPE);
2260         tlv->len = cpu_to_le32(size - sizeof(*tlv));
2261
2262         dump = (void *)tlv->data;
2263
2264         dump->version = cpu_to_le32(IWL_INI_DUMP_VER);
2265         dump->time_point = trigger->time_point;
2266         dump->trigger_reason = trigger->trigger_reason;
2267         dump->external_cfg_state =
2268                 cpu_to_le32(fwrt->trans->dbg.external_ini_cfg);
2269
2270         dump->ver_type = cpu_to_le32(fwrt->dump.fw_ver.type);
2271         dump->ver_subtype = cpu_to_le32(fwrt->dump.fw_ver.subtype);
2272
2273         dump->hw_step = cpu_to_le32(fwrt->trans->hw_rev_step);
2274
2275         /*
2276          * Several HWs all have type == 0x42, so we'll override this value
2277          * according to the detected HW
2278          */
2279         hw_type = CSR_HW_REV_TYPE(fwrt->trans->hw_rev);
2280         if (hw_type == IWL_AX210_HW_TYPE) {
2281                 u32 prph_val = iwl_read_umac_prph(fwrt->trans, WFPM_OTP_CFG1_ADDR);
2282                 u32 is_jacket = !!(prph_val & WFPM_OTP_CFG1_IS_JACKET_BIT);
2283                 u32 is_cdb = !!(prph_val & WFPM_OTP_CFG1_IS_CDB_BIT);
2284                 u32 masked_bits = is_jacket | (is_cdb << 1);
2285
2286                 /*
2287                  * The HW type depends on certain bits in this case, so add
2288                  * these bits to the HW type. We won't have collisions since we
2289                  * add these bits after the highest possible bit in the mask.
2290                  */
2291                 hw_type |= masked_bits << IWL_AX210_HW_TYPE_ADDITION_SHIFT;
2292         }
2293         dump->hw_type = cpu_to_le32(hw_type);
2294
2295         dump->rf_id_flavor =
2296                 cpu_to_le32(CSR_HW_RFID_FLAVOR(fwrt->trans->hw_rf_id));
2297         dump->rf_id_dash = cpu_to_le32(CSR_HW_RFID_DASH(fwrt->trans->hw_rf_id));
2298         dump->rf_id_step = cpu_to_le32(CSR_HW_RFID_STEP(fwrt->trans->hw_rf_id));
2299         dump->rf_id_type = cpu_to_le32(CSR_HW_RFID_TYPE(fwrt->trans->hw_rf_id));
2300
2301         dump->lmac_major = cpu_to_le32(fwrt->dump.fw_ver.lmac_major);
2302         dump->lmac_minor = cpu_to_le32(fwrt->dump.fw_ver.lmac_minor);
2303         dump->umac_major = cpu_to_le32(fwrt->dump.fw_ver.umac_major);
2304         dump->umac_minor = cpu_to_le32(fwrt->dump.fw_ver.umac_minor);
2305
2306         dump->fw_mon_mode = cpu_to_le32(fwrt->trans->dbg.ini_dest);
2307         dump->regions_mask = trigger->regions_mask &
2308                              ~cpu_to_le64(fwrt->trans->dbg.unsupported_region_msk);
2309
2310         dump->build_tag_len = cpu_to_le32(sizeof(dump->build_tag));
2311         memcpy(dump->build_tag, fwrt->fw->human_readable,
2312                sizeof(dump->build_tag));
2313
2314         cfg_name = dump->cfg_names;
2315         dump->num_of_cfg_names = cpu_to_le32(num_of_cfg_names);
2316         list_for_each_entry(node, &fwrt->trans->dbg.debug_info_tlv_list, list) {
2317                 struct iwl_fw_ini_debug_info_tlv *debug_info =
2318                         (void *)node->tlv.data;
2319
2320                 cfg_name->image_type = debug_info->image_type;
2321                 cfg_name->cfg_name_len =
2322                         cpu_to_le32(IWL_FW_INI_MAX_CFG_NAME);
2323                 memcpy(cfg_name->cfg_name, debug_info->debug_cfg_name,
2324                        sizeof(cfg_name->cfg_name));
2325                 cfg_name++;
2326         }
2327
2328         /* add dump info TLV to the beginning of the list since it needs to be
2329          * the first TLV in the dump
2330          */
2331         list_add(&entry->list, list);
2332
2333         return entry->size;
2334 }
2335
2336 static const struct iwl_dump_ini_mem_ops iwl_dump_ini_region_ops[] = {
2337         [IWL_FW_INI_REGION_INVALID] = {},
2338         [IWL_FW_INI_REGION_INTERNAL_BUFFER] = {
2339                 .get_num_of_ranges = iwl_dump_ini_single_range,
2340                 .get_size = iwl_dump_ini_mon_smem_get_size,
2341                 .fill_mem_hdr = iwl_dump_ini_mon_smem_fill_header,
2342                 .fill_range = iwl_dump_ini_mon_smem_iter,
2343         },
2344         [IWL_FW_INI_REGION_DRAM_BUFFER] = {
2345                 .get_num_of_ranges = iwl_dump_ini_mon_dram_ranges,
2346                 .get_size = iwl_dump_ini_mon_dram_get_size,
2347                 .fill_mem_hdr = iwl_dump_ini_mon_dram_fill_header,
2348                 .fill_range = iwl_dump_ini_mon_dram_iter,
2349         },
2350         [IWL_FW_INI_REGION_TXF] = {
2351                 .get_num_of_ranges = iwl_dump_ini_txf_ranges,
2352                 .get_size = iwl_dump_ini_txf_get_size,
2353                 .fill_mem_hdr = iwl_dump_ini_mem_fill_header,
2354                 .fill_range = iwl_dump_ini_txf_iter,
2355         },
2356         [IWL_FW_INI_REGION_RXF] = {
2357                 .get_num_of_ranges = iwl_dump_ini_single_range,
2358                 .get_size = iwl_dump_ini_rxf_get_size,
2359                 .fill_mem_hdr = iwl_dump_ini_mem_fill_header,
2360                 .fill_range = iwl_dump_ini_rxf_iter,
2361         },
2362         [IWL_FW_INI_REGION_LMAC_ERROR_TABLE] = {
2363                 .get_num_of_ranges = iwl_dump_ini_single_range,
2364                 .get_size = iwl_dump_ini_err_table_get_size,
2365                 .fill_mem_hdr = iwl_dump_ini_err_table_fill_header,
2366                 .fill_range = iwl_dump_ini_err_table_iter,
2367         },
2368         [IWL_FW_INI_REGION_UMAC_ERROR_TABLE] = {
2369                 .get_num_of_ranges = iwl_dump_ini_single_range,
2370                 .get_size = iwl_dump_ini_err_table_get_size,
2371                 .fill_mem_hdr = iwl_dump_ini_err_table_fill_header,
2372                 .fill_range = iwl_dump_ini_err_table_iter,
2373         },
2374         [IWL_FW_INI_REGION_RSP_OR_NOTIF] = {
2375                 .get_num_of_ranges = iwl_dump_ini_single_range,
2376                 .get_size = iwl_dump_ini_fw_pkt_get_size,
2377                 .fill_mem_hdr = iwl_dump_ini_mem_fill_header,
2378                 .fill_range = iwl_dump_ini_fw_pkt_iter,
2379         },
2380         [IWL_FW_INI_REGION_DEVICE_MEMORY] = {
2381                 .get_num_of_ranges = iwl_dump_ini_mem_ranges,
2382                 .get_size = iwl_dump_ini_mem_get_size,
2383                 .fill_mem_hdr = iwl_dump_ini_mem_fill_header,
2384                 .fill_range = iwl_dump_ini_dev_mem_iter,
2385         },
2386         [IWL_FW_INI_REGION_PERIPHERY_MAC] = {
2387                 .get_num_of_ranges = iwl_dump_ini_mem_ranges,
2388                 .get_size = iwl_dump_ini_mem_get_size,
2389                 .fill_mem_hdr = iwl_dump_ini_mem_fill_header,
2390                 .fill_range = iwl_dump_ini_prph_mac_iter,
2391         },
2392         [IWL_FW_INI_REGION_PERIPHERY_PHY] = {
2393                 .get_num_of_ranges = iwl_dump_ini_mem_ranges,
2394                 .get_size = iwl_dump_ini_mem_get_size,
2395                 .fill_mem_hdr = iwl_dump_ini_mem_fill_header,
2396                 .fill_range = iwl_dump_ini_prph_phy_iter,
2397         },
2398         [IWL_FW_INI_REGION_PERIPHERY_AUX] = {},
2399         [IWL_FW_INI_REGION_PAGING] = {
2400                 .fill_mem_hdr = iwl_dump_ini_mem_fill_header,
2401                 .get_num_of_ranges = iwl_dump_ini_paging_ranges,
2402                 .get_size = iwl_dump_ini_paging_get_size,
2403                 .fill_range = iwl_dump_ini_paging_iter,
2404         },
2405         [IWL_FW_INI_REGION_CSR] = {
2406                 .get_num_of_ranges = iwl_dump_ini_mem_ranges,
2407                 .get_size = iwl_dump_ini_mem_get_size,
2408                 .fill_mem_hdr = iwl_dump_ini_mem_fill_header,
2409                 .fill_range = iwl_dump_ini_csr_iter,
2410         },
2411         [IWL_FW_INI_REGION_DRAM_IMR] = {
2412                 .get_num_of_ranges = iwl_dump_ini_imr_ranges,
2413                 .get_size = iwl_dump_ini_imr_get_size,
2414                 .fill_mem_hdr = iwl_dump_ini_imr_fill_header,
2415                 .fill_range = iwl_dump_ini_imr_iter,
2416         },
2417         [IWL_FW_INI_REGION_PCI_IOSF_CONFIG] = {
2418                 .get_num_of_ranges = iwl_dump_ini_mem_ranges,
2419                 .get_size = iwl_dump_ini_mem_get_size,
2420                 .fill_mem_hdr = iwl_dump_ini_mem_fill_header,
2421                 .fill_range = iwl_dump_ini_config_iter,
2422         },
2423         [IWL_FW_INI_REGION_SPECIAL_DEVICE_MEMORY] = {
2424                 .get_num_of_ranges = iwl_dump_ini_single_range,
2425                 .get_size = iwl_dump_ini_special_mem_get_size,
2426                 .fill_mem_hdr = iwl_dump_ini_special_mem_fill_header,
2427                 .fill_range = iwl_dump_ini_special_mem_iter,
2428         },
2429         [IWL_FW_INI_REGION_DBGI_SRAM] = {
2430                 .get_num_of_ranges = iwl_dump_ini_mem_ranges,
2431                 .get_size = iwl_dump_ini_mon_dbgi_get_size,
2432                 .fill_mem_hdr = iwl_dump_ini_mon_dbgi_fill_header,
2433                 .fill_range = iwl_dump_ini_dbgi_sram_iter,
2434         },
2435 };
2436
2437 static u32 iwl_dump_ini_trigger(struct iwl_fw_runtime *fwrt,
2438                                 struct iwl_fwrt_dump_data *dump_data,
2439                                 struct list_head *list)
2440 {
2441         struct iwl_fw_ini_trigger_tlv *trigger = dump_data->trig;
2442         enum iwl_fw_ini_time_point tp_id = le32_to_cpu(trigger->time_point);
2443         struct iwl_dump_ini_region_data reg_data = {
2444                 .dump_data = dump_data,
2445         };
2446         int i;
2447         u32 size = 0;
2448         u64 regions_mask = le64_to_cpu(trigger->regions_mask) &
2449                            ~(fwrt->trans->dbg.unsupported_region_msk);
2450
2451         BUILD_BUG_ON(sizeof(trigger->regions_mask) != sizeof(regions_mask));
2452         BUILD_BUG_ON((sizeof(trigger->regions_mask) * BITS_PER_BYTE) <
2453                      ARRAY_SIZE(fwrt->trans->dbg.active_regions));
2454
2455         for (i = 0; i < ARRAY_SIZE(fwrt->trans->dbg.active_regions); i++) {
2456                 u32 reg_type;
2457                 struct iwl_fw_ini_region_tlv *reg;
2458
2459                 if (!(BIT_ULL(i) & regions_mask))
2460                         continue;
2461
2462                 reg_data.reg_tlv = fwrt->trans->dbg.active_regions[i];
2463                 if (!reg_data.reg_tlv) {
2464                         IWL_WARN(fwrt,
2465                                  "WRT: Unassigned region id %d, skipping\n", i);
2466                         continue;
2467                 }
2468
2469                 reg = (void *)reg_data.reg_tlv->data;
2470                 reg_type = reg->type;
2471                 if (reg_type >= ARRAY_SIZE(iwl_dump_ini_region_ops))
2472                         continue;
2473
2474                 if (reg_type == IWL_FW_INI_REGION_PERIPHERY_PHY &&
2475                     tp_id != IWL_FW_INI_TIME_POINT_FW_ASSERT) {
2476                         IWL_WARN(fwrt,
2477                                  "WRT: trying to collect phy prph at time point: %d, skipping\n",
2478                                  tp_id);
2479                         continue;
2480                 }
2481
2482                 size += iwl_dump_ini_mem(fwrt, list, &reg_data,
2483                                          &iwl_dump_ini_region_ops[reg_type]);
2484         }
2485
2486         if (size)
2487                 size += iwl_dump_ini_info(fwrt, trigger, list);
2488
2489         return size;
2490 }
2491
2492 static bool iwl_fw_ini_trigger_on(struct iwl_fw_runtime *fwrt,
2493                                   struct iwl_fw_ini_trigger_tlv *trig)
2494 {
2495         enum iwl_fw_ini_time_point tp_id = le32_to_cpu(trig->time_point);
2496         u32 usec = le32_to_cpu(trig->ignore_consec);
2497
2498         if (!iwl_trans_dbg_ini_valid(fwrt->trans) ||
2499             tp_id == IWL_FW_INI_TIME_POINT_INVALID ||
2500             tp_id >= IWL_FW_INI_TIME_POINT_NUM ||
2501             iwl_fw_dbg_no_trig_window(fwrt, tp_id, usec))
2502                 return false;
2503
2504         return true;
2505 }
2506
2507 static u32 iwl_dump_ini_file_gen(struct iwl_fw_runtime *fwrt,
2508                                  struct iwl_fwrt_dump_data *dump_data,
2509                                  struct list_head *list)
2510 {
2511         struct iwl_fw_ini_trigger_tlv *trigger = dump_data->trig;
2512         struct iwl_fw_ini_dump_entry *entry;
2513         struct iwl_fw_ini_dump_file_hdr *hdr;
2514         u32 size;
2515
2516         if (!trigger || !iwl_fw_ini_trigger_on(fwrt, trigger) ||
2517             !le64_to_cpu(trigger->regions_mask))
2518                 return 0;
2519
2520         entry = vzalloc(sizeof(*entry) + sizeof(*hdr));
2521         if (!entry)
2522                 return 0;
2523
2524         entry->size = sizeof(*hdr);
2525
2526         size = iwl_dump_ini_trigger(fwrt, dump_data, list);
2527         if (!size) {
2528                 vfree(entry);
2529                 return 0;
2530         }
2531
2532         hdr = (void *)entry->data;
2533         hdr->barker = cpu_to_le32(IWL_FW_INI_ERROR_DUMP_BARKER);
2534         hdr->file_len = cpu_to_le32(size + entry->size);
2535
2536         list_add(&entry->list, list);
2537
2538         return le32_to_cpu(hdr->file_len);
2539 }
2540
2541 static inline void iwl_fw_free_dump_desc(struct iwl_fw_runtime *fwrt,
2542                                          const struct iwl_fw_dump_desc *desc)
2543 {
2544         if (desc && desc != &iwl_dump_desc_assert)
2545                 kfree(desc);
2546
2547         fwrt->dump.lmac_err_id[0] = 0;
2548         if (fwrt->smem_cfg.num_lmacs > 1)
2549                 fwrt->dump.lmac_err_id[1] = 0;
2550         fwrt->dump.umac_err_id = 0;
2551 }
2552
2553 static void iwl_fw_error_dump(struct iwl_fw_runtime *fwrt,
2554                               struct iwl_fwrt_dump_data *dump_data)
2555 {
2556         struct iwl_fw_dump_ptrs fw_error_dump = {};
2557         struct iwl_fw_error_dump_file *dump_file;
2558         struct scatterlist *sg_dump_data;
2559         u32 file_len;
2560         u32 dump_mask = fwrt->fw->dbg.dump_mask;
2561
2562         dump_file = iwl_fw_error_dump_file(fwrt, &fw_error_dump, dump_data);
2563         if (!dump_file)
2564                 return;
2565
2566         if (dump_data->monitor_only)
2567                 dump_mask &= BIT(IWL_FW_ERROR_DUMP_FW_MONITOR);
2568
2569         fw_error_dump.trans_ptr = iwl_trans_dump_data(fwrt->trans, dump_mask,
2570                                                       fwrt->sanitize_ops,
2571                                                       fwrt->sanitize_ctx);
2572         file_len = le32_to_cpu(dump_file->file_len);
2573         fw_error_dump.fwrt_len = file_len;
2574
2575         if (fw_error_dump.trans_ptr) {
2576                 file_len += fw_error_dump.trans_ptr->len;
2577                 dump_file->file_len = cpu_to_le32(file_len);
2578         }
2579
2580         sg_dump_data = alloc_sgtable(file_len);
2581         if (sg_dump_data) {
2582                 sg_pcopy_from_buffer(sg_dump_data,
2583                                      sg_nents(sg_dump_data),
2584                                      fw_error_dump.fwrt_ptr,
2585                                      fw_error_dump.fwrt_len, 0);
2586                 if (fw_error_dump.trans_ptr)
2587                         sg_pcopy_from_buffer(sg_dump_data,
2588                                              sg_nents(sg_dump_data),
2589                                              fw_error_dump.trans_ptr->data,
2590                                              fw_error_dump.trans_ptr->len,
2591                                              fw_error_dump.fwrt_len);
2592                 dev_coredumpsg(fwrt->trans->dev, sg_dump_data, file_len,
2593                                GFP_KERNEL);
2594         }
2595         vfree(fw_error_dump.fwrt_ptr);
2596         vfree(fw_error_dump.trans_ptr);
2597 }
2598
2599 static void iwl_dump_ini_list_free(struct list_head *list)
2600 {
2601         while (!list_empty(list)) {
2602                 struct iwl_fw_ini_dump_entry *entry =
2603                         list_entry(list->next, typeof(*entry), list);
2604
2605                 list_del(&entry->list);
2606                 vfree(entry);
2607         }
2608 }
2609
2610 static void iwl_fw_error_dump_data_free(struct iwl_fwrt_dump_data *dump_data)
2611 {
2612         dump_data->trig = NULL;
2613         kfree(dump_data->fw_pkt);
2614         dump_data->fw_pkt = NULL;
2615 }
2616
2617 static void iwl_fw_error_ini_dump(struct iwl_fw_runtime *fwrt,
2618                                   struct iwl_fwrt_dump_data *dump_data)
2619 {
2620 #if defined(__linux__)
2621         LIST_HEAD(dump_list);
2622 #elif defined(__FreeBSD__)
2623         LINUX_LIST_HEAD(dump_list);
2624 #endif
2625         struct scatterlist *sg_dump_data;
2626         u32 file_len = iwl_dump_ini_file_gen(fwrt, dump_data, &dump_list);
2627
2628         if (!file_len)
2629                 return;
2630
2631         sg_dump_data = alloc_sgtable(file_len);
2632         if (sg_dump_data) {
2633                 struct iwl_fw_ini_dump_entry *entry;
2634                 int sg_entries = sg_nents(sg_dump_data);
2635                 u32 offs = 0;
2636
2637                 list_for_each_entry(entry, &dump_list, list) {
2638                         sg_pcopy_from_buffer(sg_dump_data, sg_entries,
2639                                              entry->data, entry->size, offs);
2640                         offs += entry->size;
2641                 }
2642                 dev_coredumpsg(fwrt->trans->dev, sg_dump_data, file_len,
2643                                GFP_KERNEL);
2644         }
2645         iwl_dump_ini_list_free(&dump_list);
2646 }
2647
2648 const struct iwl_fw_dump_desc iwl_dump_desc_assert = {
2649         .trig_desc = {
2650                 .type = cpu_to_le32(FW_DBG_TRIGGER_FW_ASSERT),
2651         },
2652 };
2653 IWL_EXPORT_SYMBOL(iwl_dump_desc_assert);
2654
2655 int iwl_fw_dbg_collect_desc(struct iwl_fw_runtime *fwrt,
2656                             const struct iwl_fw_dump_desc *desc,
2657                             bool monitor_only,
2658                             unsigned int delay)
2659 {
2660         struct iwl_fwrt_wk_data *wk_data;
2661         unsigned long idx;
2662
2663         if (iwl_trans_dbg_ini_valid(fwrt->trans)) {
2664                 iwl_fw_free_dump_desc(fwrt, desc);
2665                 return 0;
2666         }
2667
2668         /*
2669          * Check there is an available worker.
2670          * ffz return value is undefined if no zero exists,
2671          * so check against ~0UL first.
2672          */
2673         if (fwrt->dump.active_wks == ~0UL)
2674                 return -EBUSY;
2675
2676         idx = ffz(fwrt->dump.active_wks);
2677
2678         if (idx >= IWL_FW_RUNTIME_DUMP_WK_NUM ||
2679             test_and_set_bit(fwrt->dump.wks[idx].idx, &fwrt->dump.active_wks))
2680                 return -EBUSY;
2681
2682         wk_data = &fwrt->dump.wks[idx];
2683
2684         if (WARN_ON(wk_data->dump_data.desc))
2685                 iwl_fw_free_dump_desc(fwrt, wk_data->dump_data.desc);
2686
2687         wk_data->dump_data.desc = desc;
2688         wk_data->dump_data.monitor_only = monitor_only;
2689
2690         IWL_WARN(fwrt, "Collecting data: trigger %d fired.\n",
2691                  le32_to_cpu(desc->trig_desc.type));
2692
2693         schedule_delayed_work(&wk_data->wk, usecs_to_jiffies(delay));
2694
2695         return 0;
2696 }
2697 IWL_EXPORT_SYMBOL(iwl_fw_dbg_collect_desc);
2698
2699 int iwl_fw_dbg_error_collect(struct iwl_fw_runtime *fwrt,
2700                              enum iwl_fw_dbg_trigger trig_type)
2701 {
2702         if (!test_bit(STATUS_DEVICE_ENABLED, &fwrt->trans->status))
2703                 return -EIO;
2704
2705         if (iwl_trans_dbg_ini_valid(fwrt->trans)) {
2706                 if (trig_type != FW_DBG_TRIGGER_ALIVE_TIMEOUT &&
2707                     trig_type != FW_DBG_TRIGGER_DRIVER)
2708                         return -EIO;
2709
2710                 iwl_dbg_tlv_time_point(fwrt,
2711                                        IWL_FW_INI_TIME_POINT_HOST_ALIVE_TIMEOUT,
2712                                        NULL);
2713         } else {
2714                 struct iwl_fw_dump_desc *iwl_dump_error_desc;
2715                 int ret;
2716
2717                 iwl_dump_error_desc =
2718                         kmalloc(sizeof(*iwl_dump_error_desc), GFP_KERNEL);
2719
2720                 if (!iwl_dump_error_desc)
2721                         return -ENOMEM;
2722
2723                 iwl_dump_error_desc->trig_desc.type = cpu_to_le32(trig_type);
2724                 iwl_dump_error_desc->len = 0;
2725
2726                 ret = iwl_fw_dbg_collect_desc(fwrt, iwl_dump_error_desc,
2727                                               false, 0);
2728                 if (ret) {
2729                         kfree(iwl_dump_error_desc);
2730                         return ret;
2731                 }
2732         }
2733
2734         iwl_trans_sync_nmi(fwrt->trans);
2735
2736         return 0;
2737 }
2738 IWL_EXPORT_SYMBOL(iwl_fw_dbg_error_collect);
2739
2740 int iwl_fw_dbg_collect(struct iwl_fw_runtime *fwrt,
2741                        enum iwl_fw_dbg_trigger trig,
2742                        const char *str, size_t len,
2743                        struct iwl_fw_dbg_trigger_tlv *trigger)
2744 {
2745         struct iwl_fw_dump_desc *desc;
2746         unsigned int delay = 0;
2747         bool monitor_only = false;
2748
2749         if (trigger) {
2750                 u16 occurrences = le16_to_cpu(trigger->occurrences) - 1;
2751
2752                 if (!le16_to_cpu(trigger->occurrences))
2753                         return 0;
2754
2755                 if (trigger->flags & IWL_FW_DBG_FORCE_RESTART) {
2756                         IWL_WARN(fwrt, "Force restart: trigger %d fired.\n",
2757                                  trig);
2758                         iwl_force_nmi(fwrt->trans);
2759                         return 0;
2760                 }
2761
2762                 trigger->occurrences = cpu_to_le16(occurrences);
2763                 monitor_only = trigger->mode & IWL_FW_DBG_TRIGGER_MONITOR_ONLY;
2764
2765                 /* convert msec to usec */
2766                 delay = le32_to_cpu(trigger->stop_delay) * USEC_PER_MSEC;
2767         }
2768
2769         desc = kzalloc(struct_size(desc, trig_desc.data, len), GFP_ATOMIC);
2770         if (!desc)
2771                 return -ENOMEM;
2772
2773
2774         desc->len = len;
2775         desc->trig_desc.type = cpu_to_le32(trig);
2776         memcpy(desc->trig_desc.data, str, len);
2777
2778         return iwl_fw_dbg_collect_desc(fwrt, desc, monitor_only, delay);
2779 }
2780 IWL_EXPORT_SYMBOL(iwl_fw_dbg_collect);
2781
2782 int iwl_fw_dbg_collect_trig(struct iwl_fw_runtime *fwrt,
2783                             struct iwl_fw_dbg_trigger_tlv *trigger,
2784                             const char *fmt, ...)
2785 {
2786         int ret, len = 0;
2787         char buf[64];
2788
2789         if (iwl_trans_dbg_ini_valid(fwrt->trans))
2790                 return 0;
2791
2792         if (fmt) {
2793                 va_list ap;
2794
2795                 buf[sizeof(buf) - 1] = '\0';
2796
2797                 va_start(ap, fmt);
2798                 vsnprintf(buf, sizeof(buf), fmt, ap);
2799                 va_end(ap);
2800
2801                 /* check for truncation */
2802                 if (WARN_ON_ONCE(buf[sizeof(buf) - 1]))
2803                         buf[sizeof(buf) - 1] = '\0';
2804
2805                 len = strlen(buf) + 1;
2806         }
2807
2808         ret = iwl_fw_dbg_collect(fwrt, le32_to_cpu(trigger->id), buf, len,
2809                                  trigger);
2810
2811         if (ret)
2812                 return ret;
2813
2814         return 0;
2815 }
2816 IWL_EXPORT_SYMBOL(iwl_fw_dbg_collect_trig);
2817
2818 int iwl_fw_start_dbg_conf(struct iwl_fw_runtime *fwrt, u8 conf_id)
2819 {
2820         u8 *ptr;
2821         int ret;
2822         int i;
2823
2824         if (WARN_ONCE(conf_id >= ARRAY_SIZE(fwrt->fw->dbg.conf_tlv),
2825                       "Invalid configuration %d\n", conf_id))
2826                 return -EINVAL;
2827
2828         /* EARLY START - firmware's configuration is hard coded */
2829         if ((!fwrt->fw->dbg.conf_tlv[conf_id] ||
2830              !fwrt->fw->dbg.conf_tlv[conf_id]->num_of_hcmds) &&
2831             conf_id == FW_DBG_START_FROM_ALIVE)
2832                 return 0;
2833
2834         if (!fwrt->fw->dbg.conf_tlv[conf_id])
2835                 return -EINVAL;
2836
2837         if (fwrt->dump.conf != FW_DBG_INVALID)
2838                 IWL_INFO(fwrt, "FW already configured (%d) - re-configuring\n",
2839                          fwrt->dump.conf);
2840
2841         /* Send all HCMDs for configuring the FW debug */
2842         ptr = (void *)&fwrt->fw->dbg.conf_tlv[conf_id]->hcmd;
2843         for (i = 0; i < fwrt->fw->dbg.conf_tlv[conf_id]->num_of_hcmds; i++) {
2844                 struct iwl_fw_dbg_conf_hcmd *cmd = (void *)ptr;
2845                 struct iwl_host_cmd hcmd = {
2846                         .id = cmd->id,
2847                         .len = { le16_to_cpu(cmd->len), },
2848                         .data = { cmd->data, },
2849                 };
2850
2851                 ret = iwl_trans_send_cmd(fwrt->trans, &hcmd);
2852                 if (ret)
2853                         return ret;
2854
2855                 ptr += sizeof(*cmd);
2856                 ptr += le16_to_cpu(cmd->len);
2857         }
2858
2859         fwrt->dump.conf = conf_id;
2860
2861         return 0;
2862 }
2863 IWL_EXPORT_SYMBOL(iwl_fw_start_dbg_conf);
2864
2865 void iwl_send_dbg_dump_complete_cmd(struct iwl_fw_runtime *fwrt,
2866                                     u32 timepoint,
2867                                     u32 timepoint_data)
2868 {
2869         struct iwl_dbg_dump_complete_cmd hcmd_data;
2870         struct iwl_host_cmd hcmd = {
2871                 .id = WIDE_ID(DEBUG_GROUP, FW_DUMP_COMPLETE_CMD),
2872                 .data[0] = &hcmd_data,
2873                 .len[0] = sizeof(hcmd_data),
2874         };
2875
2876         if (test_bit(STATUS_FW_ERROR, &fwrt->trans->status))
2877                 return;
2878
2879         if (fw_has_capa(&fwrt->fw->ucode_capa,
2880                         IWL_UCODE_TLV_CAPA_DUMP_COMPLETE_SUPPORT)) {
2881                 hcmd_data.tp = cpu_to_le32(timepoint);
2882                 hcmd_data.tp_data = cpu_to_le32(timepoint_data);
2883                 iwl_trans_send_cmd(fwrt->trans, &hcmd);
2884         }
2885 }
2886
2887 /* this function assumes dump_start was called beforehand and dump_end will be
2888  * called afterwards
2889  */
2890 static void iwl_fw_dbg_collect_sync(struct iwl_fw_runtime *fwrt, u8 wk_idx)
2891 {
2892         struct iwl_fw_dbg_params params = {0};
2893         struct iwl_fwrt_dump_data *dump_data =
2894                 &fwrt->dump.wks[wk_idx].dump_data;
2895         u32 policy;
2896         u32 time_point;
2897         if (!test_bit(wk_idx, &fwrt->dump.active_wks))
2898                 return;
2899
2900         if (!test_bit(STATUS_DEVICE_ENABLED, &fwrt->trans->status)) {
2901                 IWL_ERR(fwrt, "Device is not enabled - cannot dump error\n");
2902                 goto out;
2903         }
2904
2905         /* there's no point in fw dump if the bus is dead */
2906         if (test_bit(STATUS_TRANS_DEAD, &fwrt->trans->status)) {
2907                 IWL_ERR(fwrt, "Skip fw error dump since bus is dead\n");
2908                 goto out;
2909         }
2910
2911         iwl_fw_dbg_stop_restart_recording(fwrt, &params, true);
2912
2913         IWL_DEBUG_FW_INFO(fwrt, "WRT: Data collection start\n");
2914         if (iwl_trans_dbg_ini_valid(fwrt->trans))
2915                 iwl_fw_error_ini_dump(fwrt, &fwrt->dump.wks[wk_idx].dump_data);
2916         else
2917                 iwl_fw_error_dump(fwrt, &fwrt->dump.wks[wk_idx].dump_data);
2918         IWL_DEBUG_FW_INFO(fwrt, "WRT: Data collection done\n");
2919
2920         iwl_fw_dbg_stop_restart_recording(fwrt, &params, false);
2921
2922         policy = le32_to_cpu(dump_data->trig->apply_policy);
2923         time_point = le32_to_cpu(dump_data->trig->time_point);
2924
2925         if (policy & IWL_FW_INI_APPLY_POLICY_DUMP_COMPLETE_CMD) {
2926                 IWL_DEBUG_FW_INFO(fwrt, "WRT: sending dump complete\n");
2927                 iwl_send_dbg_dump_complete_cmd(fwrt, time_point, 0);
2928         }
2929         if (fwrt->trans->dbg.last_tp_resetfw == IWL_FW_INI_RESET_FW_MODE_STOP_FW_ONLY)
2930                 iwl_force_nmi(fwrt->trans);
2931
2932 out:
2933         if (iwl_trans_dbg_ini_valid(fwrt->trans)) {
2934                 iwl_fw_error_dump_data_free(dump_data);
2935         } else {
2936                 iwl_fw_free_dump_desc(fwrt, dump_data->desc);
2937                 dump_data->desc = NULL;
2938         }
2939
2940         clear_bit(wk_idx, &fwrt->dump.active_wks);
2941 }
2942
2943 int iwl_fw_dbg_ini_collect(struct iwl_fw_runtime *fwrt,
2944                            struct iwl_fwrt_dump_data *dump_data,
2945                            bool sync)
2946 {
2947         struct iwl_fw_ini_trigger_tlv *trig = dump_data->trig;
2948         enum iwl_fw_ini_time_point tp_id = le32_to_cpu(trig->time_point);
2949         u32 occur, delay;
2950         unsigned long idx;
2951
2952         if (!iwl_fw_ini_trigger_on(fwrt, trig)) {
2953                 IWL_WARN(fwrt, "WRT: Trigger %d is not active, aborting dump\n",
2954                          tp_id);
2955                 return -EINVAL;
2956         }
2957
2958         delay = le32_to_cpu(trig->dump_delay);
2959         occur = le32_to_cpu(trig->occurrences);
2960         if (!occur)
2961                 return 0;
2962
2963         trig->occurrences = cpu_to_le32(--occur);
2964
2965         /* Check there is an available worker.
2966          * ffz return value is undefined if no zero exists,
2967          * so check against ~0UL first.
2968          */
2969         if (fwrt->dump.active_wks == ~0UL)
2970                 return -EBUSY;
2971
2972         idx = ffz(fwrt->dump.active_wks);
2973
2974         if (idx >= IWL_FW_RUNTIME_DUMP_WK_NUM ||
2975             test_and_set_bit(fwrt->dump.wks[idx].idx, &fwrt->dump.active_wks))
2976                 return -EBUSY;
2977
2978         fwrt->dump.wks[idx].dump_data = *dump_data;
2979
2980         if (sync)
2981                 delay = 0;
2982
2983         IWL_WARN(fwrt,
2984                  "WRT: Collecting data: ini trigger %d fired (delay=%dms).\n",
2985                  tp_id, (u32)(delay / USEC_PER_MSEC));
2986
2987         schedule_delayed_work(&fwrt->dump.wks[idx].wk, usecs_to_jiffies(delay));
2988
2989         if (sync)
2990                 iwl_fw_dbg_collect_sync(fwrt, idx);
2991
2992         return 0;
2993 }
2994
2995 void iwl_fw_error_dump_wk(struct work_struct *work)
2996 {
2997         struct iwl_fwrt_wk_data *wks =
2998                 container_of(work, typeof(*wks), wk.work);
2999         struct iwl_fw_runtime *fwrt =
3000                 container_of(wks, typeof(*fwrt), dump.wks[wks->idx]);
3001
3002         /* assumes the op mode mutex is locked in dump_start since
3003          * iwl_fw_dbg_collect_sync can't run in parallel
3004          */
3005         if (fwrt->ops && fwrt->ops->dump_start)
3006                 fwrt->ops->dump_start(fwrt->ops_ctx);
3007
3008         iwl_fw_dbg_collect_sync(fwrt, wks->idx);
3009
3010         if (fwrt->ops && fwrt->ops->dump_end)
3011                 fwrt->ops->dump_end(fwrt->ops_ctx);
3012 }
3013
3014 void iwl_fw_dbg_read_d3_debug_data(struct iwl_fw_runtime *fwrt)
3015 {
3016         const struct iwl_cfg *cfg = fwrt->trans->cfg;
3017
3018         if (!iwl_fw_dbg_is_d3_debug_enabled(fwrt))
3019                 return;
3020
3021         if (!fwrt->dump.d3_debug_data) {
3022                 fwrt->dump.d3_debug_data = kmalloc(cfg->d3_debug_data_length,
3023                                                    GFP_KERNEL);
3024                 if (!fwrt->dump.d3_debug_data) {
3025                         IWL_ERR(fwrt,
3026                                 "failed to allocate memory for D3 debug data\n");
3027                         return;
3028                 }
3029         }
3030
3031         /* if the buffer holds previous debug data it is overwritten */
3032         iwl_trans_read_mem_bytes(fwrt->trans, cfg->d3_debug_data_base_addr,
3033                                  fwrt->dump.d3_debug_data,
3034                                  cfg->d3_debug_data_length);
3035
3036         if (fwrt->sanitize_ops && fwrt->sanitize_ops->frob_mem)
3037                 fwrt->sanitize_ops->frob_mem(fwrt->sanitize_ctx,
3038                                              cfg->d3_debug_data_base_addr,
3039                                              fwrt->dump.d3_debug_data,
3040                                              cfg->d3_debug_data_length);
3041 }
3042 IWL_EXPORT_SYMBOL(iwl_fw_dbg_read_d3_debug_data);
3043
3044 void iwl_fw_dbg_stop_sync(struct iwl_fw_runtime *fwrt)
3045 {
3046         int i;
3047
3048         iwl_dbg_tlv_del_timers(fwrt->trans);
3049         for (i = 0; i < IWL_FW_RUNTIME_DUMP_WK_NUM; i++)
3050                 iwl_fw_dbg_collect_sync(fwrt, i);
3051
3052         iwl_fw_dbg_stop_restart_recording(fwrt, NULL, true);
3053 }
3054 IWL_EXPORT_SYMBOL(iwl_fw_dbg_stop_sync);
3055
3056 static int iwl_fw_dbg_suspend_resume_hcmd(struct iwl_trans *trans, bool suspend)
3057 {
3058         struct iwl_dbg_suspend_resume_cmd cmd = {
3059                 .operation = suspend ?
3060                         cpu_to_le32(DBGC_SUSPEND_CMD) :
3061                         cpu_to_le32(DBGC_RESUME_CMD),
3062         };
3063         struct iwl_host_cmd hcmd = {
3064                 .id = WIDE_ID(DEBUG_GROUP, DBGC_SUSPEND_RESUME),
3065                 .data[0] = &cmd,
3066                 .len[0] = sizeof(cmd),
3067         };
3068
3069         return iwl_trans_send_cmd(trans, &hcmd);
3070 }
3071
3072 static void iwl_fw_dbg_stop_recording(struct iwl_trans *trans,
3073                                       struct iwl_fw_dbg_params *params)
3074 {
3075         if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_7000) {
3076                 iwl_set_bits_prph(trans, MON_BUFF_SAMPLE_CTL, 0x100);
3077                 return;
3078         }
3079
3080         if (params) {
3081                 params->in_sample = iwl_read_umac_prph(trans, DBGC_IN_SAMPLE);
3082                 params->out_ctrl = iwl_read_umac_prph(trans, DBGC_OUT_CTRL);
3083         }
3084
3085         iwl_write_umac_prph(trans, DBGC_IN_SAMPLE, 0);
3086         /* wait for the DBGC to finish writing the internal buffer to DRAM to
3087          * avoid halting the HW while writing
3088          */
3089         usleep_range(700, 1000);
3090         iwl_write_umac_prph(trans, DBGC_OUT_CTRL, 0);
3091 }
3092
3093 static int iwl_fw_dbg_restart_recording(struct iwl_trans *trans,
3094                                         struct iwl_fw_dbg_params *params)
3095 {
3096         if (!params)
3097                 return -EIO;
3098
3099         if (trans->trans_cfg->device_family == IWL_DEVICE_FAMILY_7000) {
3100                 iwl_clear_bits_prph(trans, MON_BUFF_SAMPLE_CTL, 0x100);
3101                 iwl_clear_bits_prph(trans, MON_BUFF_SAMPLE_CTL, 0x1);
3102                 iwl_set_bits_prph(trans, MON_BUFF_SAMPLE_CTL, 0x1);
3103         } else {
3104                 iwl_write_umac_prph(trans, DBGC_IN_SAMPLE, params->in_sample);
3105                 iwl_write_umac_prph(trans, DBGC_OUT_CTRL, params->out_ctrl);
3106         }
3107
3108         return 0;
3109 }
3110
3111 void iwl_fw_dbg_stop_restart_recording(struct iwl_fw_runtime *fwrt,
3112                                        struct iwl_fw_dbg_params *params,
3113                                        bool stop)
3114 {
3115         int ret __maybe_unused = 0;
3116
3117         if (test_bit(STATUS_FW_ERROR, &fwrt->trans->status))
3118                 return;
3119
3120         if (fw_has_capa(&fwrt->fw->ucode_capa,
3121                         IWL_UCODE_TLV_CAPA_DBG_SUSPEND_RESUME_CMD_SUPP))
3122                 ret = iwl_fw_dbg_suspend_resume_hcmd(fwrt->trans, stop);
3123         else if (stop)
3124                 iwl_fw_dbg_stop_recording(fwrt->trans, params);
3125         else
3126                 ret = iwl_fw_dbg_restart_recording(fwrt->trans, params);
3127 #ifdef CONFIG_IWLWIFI_DEBUGFS
3128         if (!ret) {
3129                 if (stop)
3130                         fwrt->trans->dbg.rec_on = false;
3131                 else
3132                         iwl_fw_set_dbg_rec_on(fwrt);
3133         }
3134 #endif
3135 }
3136 IWL_EXPORT_SYMBOL(iwl_fw_dbg_stop_restart_recording);