sys/dev/iwm/if_iwm.c

   1 /*      $OpenBSD: if_iwm.c,v 1.167 2017/04/04 00:40:52 claudio Exp $    */
   2
   3 /*
   4  * Copyright (c) 2014 genua mbh <info@genua.de>
   5  * Copyright (c) 2014 Fixup Software Ltd.
   6  *
   7  * Permission to use, copy, modify, and distribute this software for any
   8  * purpose with or without fee is hereby granted, provided that the above
   9  * copyright notice and this permission notice appear in all copies.
  10  *
  11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  18  */
  19
  20 /*-
  21  * Based on BSD-licensed source modules in the Linux iwlwifi driver,
  22  * which were used as the reference documentation for this implementation.
  23  *
  24  * Driver version we are currently based off of is
  25  * Linux 3.14.3 (tag id a2df521e42b1d9a23f620ac79dbfe8655a8391dd)
  26  *
  27  ***********************************************************************
  28  *
  29  * This file is provided under a dual BSD/GPLv2 license.  When using or
  30  * redistributing this file, you may do so under either license.
  31  *
  32  * GPL LICENSE SUMMARY
  33  *
  34  * Copyright(c) 2007 - 2013 Intel Corporation. All rights reserved.
  35  *
  36  * This program is free software; you can redistribute it and/or modify
  37  * it under the terms of version 2 of the GNU General Public License as
  38  * published by the Free Software Foundation.
  39  *
  40  * This program is distributed in the hope that it will be useful, but
  41  * WITHOUT ANY WARRANTY; without even the implied warranty of
  42  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  43  * General Public License for more details.
  44  *
  45  * You should have received a copy of the GNU General Public License
  46  * along with this program; if not, write to the Free Software
  47  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
  48  * USA
  49  *
  50  * The full GNU General Public License is included in this distribution
  51  * in the file called COPYING.
  52  *
  53  * Contact Information:
  54  *  Intel Linux Wireless <ilw@linux.intel.com>
  55  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  56  *
  57  *
  58  * BSD LICENSE
  59  *
  60  * Copyright(c) 2005 - 2013 Intel Corporation. All rights reserved.
  61  * All rights reserved.
  62  *
  63  * Redistribution and use in source and binary forms, with or without
  64  * modification, are permitted provided that the following conditions
  65  * are met:
  66  *
  67  *  * Redistributions of source code must retain the above copyright
  68  *    notice, this list of conditions and the following disclaimer.
  69  *  * Redistributions in binary form must reproduce the above copyright
  70  *    notice, this list of conditions and the following disclaimer in
  71  *    the documentation and/or other materials provided with the
  72  *    distribution.
  73  *  * Neither the name Intel Corporation nor the names of its
  74  *    contributors may be used to endorse or promote products derived
  75  *    from this software without specific prior written permission.
  76  *
  77  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  78  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  79  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  80  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  81  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  82  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  83  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  84  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  85  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  86  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  87  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  88  */
  89
  90 /*-
  91  * Copyright (c) 2007-2010 Damien Bergamini <damien.bergamini@free.fr>
  92  *
  93  * Permission to use, copy, modify, and distribute this software for any
  94  * purpose with or without fee is hereby granted, provided that the above
  95  * copyright notice and this permission notice appear in all copies.
  96  *
  97  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  98  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  99  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 100  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 101  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 102  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 103  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 104  */
 105 #include <sys/cdefs.h>
 106 __FBSDID("$FreeBSD$");
 107
 108 #include "opt_wlan.h"
 109 #include "opt_iwm.h"
 110
 111 #include <sys/param.h>
 112 #include <sys/bus.h>
 113 #include <sys/conf.h>
 114 #include <sys/endian.h>
 115 #include <sys/firmware.h>
 116 #include <sys/kernel.h>
 117 #include <sys/malloc.h>
 118 #include <sys/mbuf.h>
 119 #include <sys/mutex.h>
 120 #include <sys/module.h>
 121 #include <sys/proc.h>
 122 #include <sys/rman.h>
 123 #include <sys/socket.h>
 124 #include <sys/sockio.h>
 125 #include <sys/sysctl.h>
 126 #include <sys/linker.h>
 127
 128 #include <machine/bus.h>
 129 #include <machine/endian.h>
 130 #include <machine/resource.h>
 131
 132 #include <dev/pci/pcivar.h>
 133 #include <dev/pci/pcireg.h>
 134
 135 #include <net/bpf.h>
 136
 137 #include <net/if.h>
 138 #include <net/if_var.h>
 139 #include <net/if_arp.h>
 140 #include <net/if_dl.h>
 141 #include <net/if_media.h>
 142 #include <net/if_types.h>
 143
 144 #include <netinet/in.h>
 145 #include <netinet/in_systm.h>
 146 #include <netinet/if_ether.h>
 147 #include <netinet/ip.h>
 148
 149 #include <net80211/ieee80211_var.h>
 150 #include <net80211/ieee80211_regdomain.h>
 151 #include <net80211/ieee80211_ratectl.h>
 152 #include <net80211/ieee80211_radiotap.h>
 153
 154 #include <dev/iwm/if_iwmreg.h>
 155 #include <dev/iwm/if_iwmvar.h>
 156 #include <dev/iwm/if_iwm_config.h>
 157 #include <dev/iwm/if_iwm_debug.h>
 158 #include <dev/iwm/if_iwm_notif_wait.h>
 159 #include <dev/iwm/if_iwm_util.h>
 160 #include <dev/iwm/if_iwm_binding.h>
 161 #include <dev/iwm/if_iwm_phy_db.h>
 162 #include <dev/iwm/if_iwm_mac_ctxt.h>
 163 #include <dev/iwm/if_iwm_phy_ctxt.h>
 164 #include <dev/iwm/if_iwm_time_event.h>
 165 #include <dev/iwm/if_iwm_power.h>
 166 #include <dev/iwm/if_iwm_scan.h>
 167 #include <dev/iwm/if_iwm_sf.h>
 168 #include <dev/iwm/if_iwm_sta.h>
 169
 170 #include <dev/iwm/if_iwm_pcie_trans.h>
 171 #include <dev/iwm/if_iwm_led.h>
 172 #include <dev/iwm/if_iwm_fw.h>
 173
 174 /* From DragonflyBSD */
 175 #define mtodoff(m, t, off)      ((t)((m)->m_data + (off)))
 176
 177 const uint8_t iwm_nvm_channels[] = {
 178         /* 2.4 GHz */
 179         1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
 180         /* 5 GHz */
 181         36, 40, 44, 48, 52, 56, 60, 64,
 182         100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144,
 183         149, 153, 157, 161, 165
 184 };
 185 _Static_assert(nitems(iwm_nvm_channels) <= IWM_NUM_CHANNELS,
 186     "IWM_NUM_CHANNELS is too small");
 187
 188 const uint8_t iwm_nvm_channels_8000[] = {
 189         /* 2.4 GHz */
 190         1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
 191         /* 5 GHz */
 192         36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92,
 193         96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144,
 194         149, 153, 157, 161, 165, 169, 173, 177, 181
 195 };
 196 _Static_assert(nitems(iwm_nvm_channels_8000) <= IWM_NUM_CHANNELS_8000,
 197     "IWM_NUM_CHANNELS_8000 is too small");
 198
 199 #define IWM_NUM_2GHZ_CHANNELS   14
 200 #define IWM_N_HW_ADDR_MASK      0xF
 201
 202 /*
 203  * XXX For now, there's simply a fixed set of rate table entries
 204  * that are populated.
 205  */
 206 const struct iwm_rate {
 207         uint8_t rate;
 208         uint8_t plcp;
 209 } iwm_rates[] = {
 210         {   2,  IWM_RATE_1M_PLCP  },
 211         {   4,  IWM_RATE_2M_PLCP  },
 212         {  11,  IWM_RATE_5M_PLCP  },
 213         {  22,  IWM_RATE_11M_PLCP },
 214         {  12,  IWM_RATE_6M_PLCP  },
 215         {  18,  IWM_RATE_9M_PLCP  },
 216         {  24,  IWM_RATE_12M_PLCP },
 217         {  36,  IWM_RATE_18M_PLCP },
 218         {  48,  IWM_RATE_24M_PLCP },
 219         {  72,  IWM_RATE_36M_PLCP },
 220         {  96,  IWM_RATE_48M_PLCP },
 221         { 108,  IWM_RATE_54M_PLCP },
 222 };
 223 #define IWM_RIDX_CCK    0
 224 #define IWM_RIDX_OFDM   4
 225 #define IWM_RIDX_MAX    (nitems(iwm_rates)-1)
 226 #define IWM_RIDX_IS_CCK(_i_) ((_i_) < IWM_RIDX_OFDM)
 227 #define IWM_RIDX_IS_OFDM(_i_) ((_i_) >= IWM_RIDX_OFDM)
 228
 229 struct iwm_nvm_section {
 230         uint16_t length;
 231         uint8_t *data;
 232 };
 233
 234 #define IWM_MVM_UCODE_ALIVE_TIMEOUT     hz
 235 #define IWM_MVM_UCODE_CALIB_TIMEOUT     (2*hz)
 236
 237 struct iwm_mvm_alive_data {
 238         int valid;
 239         uint32_t scd_base_addr;
 240 };
 241
 242 static int      iwm_store_cscheme(struct iwm_softc *, const uint8_t *, size_t);
 243 static int      iwm_firmware_store_section(struct iwm_softc *,
 244                                            enum iwm_ucode_type,
 245                                            const uint8_t *, size_t);
 246 static int      iwm_set_default_calib(struct iwm_softc *, const void *);
 247 static void     iwm_fw_info_free(struct iwm_fw_info *);
 248 static int      iwm_read_firmware(struct iwm_softc *);
 249 static int      iwm_alloc_fwmem(struct iwm_softc *);
 250 static int      iwm_alloc_sched(struct iwm_softc *);
 251 static int      iwm_alloc_kw(struct iwm_softc *);
 252 static int      iwm_alloc_ict(struct iwm_softc *);
 253 static int      iwm_alloc_rx_ring(struct iwm_softc *, struct iwm_rx_ring *);
 254 static void     iwm_reset_rx_ring(struct iwm_softc *, struct iwm_rx_ring *);
 255 static void     iwm_free_rx_ring(struct iwm_softc *, struct iwm_rx_ring *);
 256 static int      iwm_alloc_tx_ring(struct iwm_softc *, struct iwm_tx_ring *,
 257                                   int);
 258 static void     iwm_reset_tx_ring(struct iwm_softc *, struct iwm_tx_ring *);
 259 static void     iwm_free_tx_ring(struct iwm_softc *, struct iwm_tx_ring *);
 260 static void     iwm_enable_interrupts(struct iwm_softc *);
 261 static void     iwm_restore_interrupts(struct iwm_softc *);
 262 static void     iwm_disable_interrupts(struct iwm_softc *);
 263 static void     iwm_ict_reset(struct iwm_softc *);
 264 static int      iwm_allow_mcast(struct ieee80211vap *, struct iwm_softc *);
 265 static void     iwm_stop_device(struct iwm_softc *);
 266 static void     iwm_mvm_nic_config(struct iwm_softc *);
 267 static int      iwm_nic_rx_init(struct iwm_softc *);
 268 static int      iwm_nic_tx_init(struct iwm_softc *);
 269 static int      iwm_nic_init(struct iwm_softc *);
 270 static int      iwm_trans_pcie_fw_alive(struct iwm_softc *, uint32_t);
 271 static int      iwm_nvm_read_chunk(struct iwm_softc *, uint16_t, uint16_t,
 272                                    uint16_t, uint8_t *, uint16_t *);
 273 static int      iwm_nvm_read_section(struct iwm_softc *, uint16_t, uint8_t *,
 274                                      uint16_t *, uint32_t);
 275 static uint32_t iwm_eeprom_channel_flags(uint16_t);
 276 static void     iwm_add_channel_band(struct iwm_softc *,
 277                     struct ieee80211_channel[], int, int *, int, size_t,
 278                     const uint8_t[]);
 279 static void     iwm_init_channel_map(struct ieee80211com *, int, int *,
 280                     struct ieee80211_channel[]);
 281 static struct iwm_nvm_data *
 282         iwm_parse_nvm_data(struct iwm_softc *, const uint16_t *,
 283                            const uint16_t *, const uint16_t *,
 284                            const uint16_t *, const uint16_t *,
 285                            const uint16_t *);
 286 static void     iwm_free_nvm_data(struct iwm_nvm_data *);
 287 static void     iwm_set_hw_address_family_8000(struct iwm_softc *,
 288                                                struct iwm_nvm_data *,
 289                                                const uint16_t *,
 290                                                const uint16_t *);
 291 static int      iwm_get_sku(const struct iwm_softc *, const uint16_t *,
 292                             const uint16_t *);
 293 static int      iwm_get_nvm_version(const struct iwm_softc *, const uint16_t *);
 294 static int      iwm_get_radio_cfg(const struct iwm_softc *, const uint16_t *,
 295                                   const uint16_t *);
 296 static int      iwm_get_n_hw_addrs(const struct iwm_softc *,
 297                                    const uint16_t *);
 298 static void     iwm_set_radio_cfg(const struct iwm_softc *,
 299                                   struct iwm_nvm_data *, uint32_t);
 300 static struct iwm_nvm_data *
 301         iwm_parse_nvm_sections(struct iwm_softc *, struct iwm_nvm_section *);
 302 static int      iwm_nvm_init(struct iwm_softc *);
 303 static int      iwm_pcie_load_section(struct iwm_softc *, uint8_t,
 304                                       const struct iwm_fw_desc *);
 305 static int      iwm_pcie_load_firmware_chunk(struct iwm_softc *, uint32_t,
 306                                              bus_addr_t, uint32_t);
 307 static int      iwm_pcie_load_cpu_sections_8000(struct iwm_softc *sc,
 308                                                 const struct iwm_fw_img *,
 309                                                 int, int *);
 310 static int      iwm_pcie_load_cpu_sections(struct iwm_softc *,
 311                                            const struct iwm_fw_img *,
 312                                            int, int *);
 313 static int      iwm_pcie_load_given_ucode_8000(struct iwm_softc *,
 314                                                const struct iwm_fw_img *);
 315 static int      iwm_pcie_load_given_ucode(struct iwm_softc *,
 316                                           const struct iwm_fw_img *);
 317 static int      iwm_start_fw(struct iwm_softc *, const struct iwm_fw_img *);
 318 static int      iwm_send_tx_ant_cfg(struct iwm_softc *, uint8_t);
 319 static int      iwm_send_phy_cfg_cmd(struct iwm_softc *);
 320 static int      iwm_mvm_load_ucode_wait_alive(struct iwm_softc *,
 321                                               enum iwm_ucode_type);
 322 static int      iwm_run_init_mvm_ucode(struct iwm_softc *, int);
 323 static int      iwm_mvm_config_ltr(struct iwm_softc *sc);
 324 static int      iwm_rx_addbuf(struct iwm_softc *, int, int);
 325 static void     iwm_mvm_rx_rx_phy_cmd(struct iwm_softc *,
 326                                       struct iwm_rx_packet *);
 327 static int      iwm_get_noise(struct iwm_softc *,
 328                     const struct iwm_mvm_statistics_rx_non_phy *);
 329 static void     iwm_mvm_handle_rx_statistics(struct iwm_softc *,
 330                     struct iwm_rx_packet *);
 331 static bool     iwm_mvm_rx_mpdu(struct iwm_softc *, struct mbuf *,
 332                     uint32_t, bool);
 333 static int      iwm_mvm_rx_tx_cmd_single(struct iwm_softc *,
 334                                          struct iwm_rx_packet *,
 335                                          struct iwm_node *);
 336 static void     iwm_mvm_rx_tx_cmd(struct iwm_softc *, struct iwm_rx_packet *);
 337 static void     iwm_cmd_done(struct iwm_softc *, struct iwm_rx_packet *);
 338 #if 0
 339 static void     iwm_update_sched(struct iwm_softc *, int, int, uint8_t,
 340                                  uint16_t);
 341 #endif
 342 static const struct iwm_rate *
 343         iwm_tx_fill_cmd(struct iwm_softc *, struct iwm_node *,
 344                         struct mbuf *, struct iwm_tx_cmd *);
 345 static int      iwm_tx(struct iwm_softc *, struct mbuf *,
 346                        struct ieee80211_node *, int);
 347 static int      iwm_raw_xmit(struct ieee80211_node *, struct mbuf *,
 348                              const struct ieee80211_bpf_params *);
 349 static int      iwm_mvm_update_quotas(struct iwm_softc *, struct iwm_vap *);
 350 static int      iwm_auth(struct ieee80211vap *, struct iwm_softc *);
 351 static struct ieee80211_node *
 352                 iwm_node_alloc(struct ieee80211vap *,
 353                                const uint8_t[IEEE80211_ADDR_LEN]);
 354 static uint8_t  iwm_rate_from_ucode_rate(uint32_t);
 355 static int      iwm_rate2ridx(struct iwm_softc *, uint8_t);
 356 static void     iwm_setrates(struct iwm_softc *, struct iwm_node *, int);
 357 static int      iwm_media_change(struct ifnet *);
 358 static int      iwm_newstate(struct ieee80211vap *, enum ieee80211_state, int);
 359 static void     iwm_endscan_cb(void *, int);
 360 static int      iwm_send_bt_init_conf(struct iwm_softc *);
 361 static boolean_t iwm_mvm_is_lar_supported(struct iwm_softc *);
 362 static boolean_t iwm_mvm_is_wifi_mcc_supported(struct iwm_softc *);
 363 static int      iwm_send_update_mcc_cmd(struct iwm_softc *, const char *);
 364 static void     iwm_mvm_tt_tx_backoff(struct iwm_softc *, uint32_t);
 365 static int      iwm_init_hw(struct iwm_softc *);
 366 static void     iwm_init(struct iwm_softc *);
 367 static void     iwm_start(struct iwm_softc *);
 368 static void     iwm_stop(struct iwm_softc *);
 369 static void     iwm_watchdog(void *);
 370 static void     iwm_parent(struct ieee80211com *);
 371 #ifdef IWM_DEBUG
 372 static const char *
 373                 iwm_desc_lookup(uint32_t);
 374 static void     iwm_nic_error(struct iwm_softc *);
 375 static void     iwm_nic_umac_error(struct iwm_softc *);
 376 #endif
 377 static void     iwm_handle_rxb(struct iwm_softc *, struct mbuf *);
 378 static void     iwm_notif_intr(struct iwm_softc *);
 379 static void     iwm_intr(void *);
 380 static int      iwm_attach(device_t);
 381 static int      iwm_is_valid_ether_addr(uint8_t *);
 382 static void     iwm_preinit(void *);
 383 static int      iwm_detach_local(struct iwm_softc *sc, int);
 384 static void     iwm_init_task(void *);
 385 static void     iwm_radiotap_attach(struct iwm_softc *);
 386 static struct ieee80211vap *
 387                 iwm_vap_create(struct ieee80211com *,
 388                                const char [IFNAMSIZ], int,
 389                                enum ieee80211_opmode, int,
 390                                const uint8_t [IEEE80211_ADDR_LEN],
 391                                const uint8_t [IEEE80211_ADDR_LEN]);
 392 static void     iwm_vap_delete(struct ieee80211vap *);
 393 static void     iwm_xmit_queue_drain(struct iwm_softc *);
 394 static void     iwm_scan_start(struct ieee80211com *);
 395 static void     iwm_scan_end(struct ieee80211com *);
 396 static void     iwm_update_mcast(struct ieee80211com *);
 397 static void     iwm_set_channel(struct ieee80211com *);
 398 static void     iwm_scan_curchan(struct ieee80211_scan_state *, unsigned long);
 399 static void     iwm_scan_mindwell(struct ieee80211_scan_state *);
 400 static int      iwm_detach(device_t);
 401
 402 static int      iwm_lar_disable = 0;
 403 TUNABLE_INT("hw.iwm.lar.disable", &iwm_lar_disable);
 404
 405 /*
 406  * Firmware parser.
 407  */
 408
 409 static int
 410 iwm_store_cscheme(struct iwm_softc *sc, const uint8_t *data, size_t dlen)
 411 {
 412         const struct iwm_fw_cscheme_list *l = (const void *)data;
 413
 414         if (dlen < sizeof(*l) ||
 415             dlen < sizeof(l->size) + l->size * sizeof(*l->cs))
 416                 return EINVAL;
 417
 418         /* we don't actually store anything for now, always use s/w crypto */
 419
 420         return 0;
 421 }
 422
 423 static int
 424 iwm_firmware_store_section(struct iwm_softc *sc,
 425     enum iwm_ucode_type type, const uint8_t *data, size_t dlen)
 426 {
 427         struct iwm_fw_img *fws;
 428         struct iwm_fw_desc *fwone;
 429
 430         if (type >= IWM_UCODE_TYPE_MAX)
 431                 return EINVAL;
 432         if (dlen < sizeof(uint32_t))
 433                 return EINVAL;
 434
 435         fws = &sc->sc_fw.img[type];
 436         if (fws->fw_count >= IWM_UCODE_SECTION_MAX)
 437                 return EINVAL;
 438
 439         fwone = &fws->sec[fws->fw_count];
 440
 441         /* first 32bit are device load offset */
 442         memcpy(&fwone->offset, data, sizeof(uint32_t));
 443
 444         /* rest is data */
 445         fwone->data = data + sizeof(uint32_t);
 446         fwone->len = dlen - sizeof(uint32_t);
 447
 448         fws->fw_count++;
 449
 450         return 0;
 451 }
 452
 453 #define IWM_DEFAULT_SCAN_CHANNELS 40
 454
 455 /* iwlwifi: iwl-drv.c */
 456 struct iwm_tlv_calib_data {
 457         uint32_t ucode_type;
 458         struct iwm_tlv_calib_ctrl calib;
 459 } __packed;
 460
 461 static int
 462 iwm_set_default_calib(struct iwm_softc *sc, const void *data)
 463 {
 464         const struct iwm_tlv_calib_data *def_calib = data;
 465         uint32_t ucode_type = le32toh(def_calib->ucode_type);
 466
 467         if (ucode_type >= IWM_UCODE_TYPE_MAX) {
 468                 device_printf(sc->sc_dev,
 469                     "Wrong ucode_type %u for default "
 470                     "calibration.\n", ucode_type);
 471                 return EINVAL;
 472         }
 473
 474         sc->sc_default_calib[ucode_type].flow_trigger =
 475             def_calib->calib.flow_trigger;
 476         sc->sc_default_calib[ucode_type].event_trigger =
 477             def_calib->calib.event_trigger;
 478
 479         return 0;
 480 }
 481
 482 static int
 483 iwm_set_ucode_api_flags(struct iwm_softc *sc, const uint8_t *data,
 484                         struct iwm_ucode_capabilities *capa)
 485 {
 486         const struct iwm_ucode_api *ucode_api = (const void *)data;
 487         uint32_t api_index = le32toh(ucode_api->api_index);
 488         uint32_t api_flags = le32toh(ucode_api->api_flags);
 489         int i;
 490
 491         if (api_index >= howmany(IWM_NUM_UCODE_TLV_API, 32)) {
 492                 device_printf(sc->sc_dev,
 493                     "api flags index %d larger than supported by driver\n",
 494                     api_index);
 495                 /* don't return an error so we can load FW that has more bits */
 496                 return 0;
 497         }
 498
 499         for (i = 0; i < 32; i++) {
 500                 if (api_flags & (1U << i))
 501                         setbit(capa->enabled_api, i + 32 * api_index);
 502         }
 503
 504         return 0;
 505 }
 506
 507 static int
 508 iwm_set_ucode_capabilities(struct iwm_softc *sc, const uint8_t *data,
 509                            struct iwm_ucode_capabilities *capa)
 510 {
 511         const struct iwm_ucode_capa *ucode_capa = (const void *)data;
 512         uint32_t api_index = le32toh(ucode_capa->api_index);
 513         uint32_t api_flags = le32toh(ucode_capa->api_capa);
 514         int i;
 515
 516         if (api_index >= howmany(IWM_NUM_UCODE_TLV_CAPA, 32)) {
 517                 device_printf(sc->sc_dev,
 518                     "capa flags index %d larger than supported by driver\n",
 519                     api_index);
 520                 /* don't return an error so we can load FW that has more bits */
 521                 return 0;
 522         }
 523
 524         for (i = 0; i < 32; i++) {
 525                 if (api_flags & (1U << i))
 526                         setbit(capa->enabled_capa, i + 32 * api_index);
 527         }
 528
 529         return 0;
 530 }
 531
 532 static void
 533 iwm_fw_info_free(struct iwm_fw_info *fw)
 534 {
 535         firmware_put(fw->fw_fp, FIRMWARE_UNLOAD);
 536         fw->fw_fp = NULL;
 537         memset(fw->img, 0, sizeof(fw->img));
 538 }
 539
 540 static int
 541 iwm_read_firmware(struct iwm_softc *sc)
 542 {
 543         struct iwm_fw_info *fw = &sc->sc_fw;
 544         const struct iwm_tlv_ucode_header *uhdr;
 545         const struct iwm_ucode_tlv *tlv;
 546         struct iwm_ucode_capabilities *capa = &sc->sc_fw.ucode_capa;
 547         enum iwm_ucode_tlv_type tlv_type;
 548         const struct firmware *fwp;
 549         const uint8_t *data;
 550         uint32_t tlv_len;
 551         uint32_t usniffer_img;
 552         const uint8_t *tlv_data;
 553         uint32_t paging_mem_size;
 554         int num_of_cpus;
 555         int error = 0;
 556         size_t len;
 557
 558         /*
 559          * Load firmware into driver memory.
 560          * fw_fp will be set.
 561          */
 562         fwp = firmware_get(sc->cfg->fw_name);
 563         if (fwp == NULL) {
 564                 device_printf(sc->sc_dev,
 565                     "could not read firmware %s (error %d)\n",
 566                     sc->cfg->fw_name, error);
 567                 goto out;
 568         }
 569         fw->fw_fp = fwp;
 570
 571         /* (Re-)Initialize default values. */
 572         capa->flags = 0;
 573         capa->max_probe_length = IWM_DEFAULT_MAX_PROBE_LENGTH;
 574         capa->n_scan_channels = IWM_DEFAULT_SCAN_CHANNELS;
 575         memset(capa->enabled_capa, 0, sizeof(capa->enabled_capa));
 576         memset(capa->enabled_api, 0, sizeof(capa->enabled_api));
 577         memset(sc->sc_fw_mcc, 0, sizeof(sc->sc_fw_mcc));
 578
 579         /*
 580          * Parse firmware contents
 581          */
 582
 583         uhdr = (const void *)fw->fw_fp->data;
 584         if (*(const uint32_t *)fw->fw_fp->data != 0
 585             || le32toh(uhdr->magic) != IWM_TLV_UCODE_MAGIC) {
 586                 device_printf(sc->sc_dev, "invalid firmware %s\n",
 587                     sc->cfg->fw_name);
 588                 error = EINVAL;
 589                 goto out;
 590         }
 591
 592         snprintf(sc->sc_fwver, sizeof(sc->sc_fwver), "%u.%u (API ver %u)",
 593             IWM_UCODE_MAJOR(le32toh(uhdr->ver)),
 594             IWM_UCODE_MINOR(le32toh(uhdr->ver)),
 595             IWM_UCODE_API(le32toh(uhdr->ver)));
 596         data = uhdr->data;
 597         len = fw->fw_fp->datasize - sizeof(*uhdr);
 598
 599         while (len >= sizeof(*tlv)) {
 600                 len -= sizeof(*tlv);
 601                 tlv = (const void *)data;
 602
 603                 tlv_len = le32toh(tlv->length);
 604                 tlv_type = le32toh(tlv->type);
 605                 tlv_data = tlv->data;
 606
 607                 if (len < tlv_len) {
 608                         device_printf(sc->sc_dev,
 609                             "firmware too short: %zu bytes\n",
 610                             len);
 611                         error = EINVAL;
 612                         goto parse_out;
 613                 }
 614                 len -= roundup2(tlv_len, 4);
 615                 data += sizeof(*tlv) + roundup2(tlv_len, 4);
 616
 617                 switch ((int)tlv_type) {
 618                 case IWM_UCODE_TLV_PROBE_MAX_LEN:
 619                         if (tlv_len != sizeof(uint32_t)) {
 620                                 device_printf(sc->sc_dev,
 621                                     "%s: PROBE_MAX_LEN (%u) != sizeof(uint32_t)\n",
 622                                     __func__, tlv_len);
 623                                 error = EINVAL;
 624                                 goto parse_out;
 625                         }
 626                         capa->max_probe_length =
 627                             le32_to_cpup((const uint32_t *)tlv_data);
 628                         /* limit it to something sensible */
 629                         if (capa->max_probe_length >
 630                             IWM_SCAN_OFFLOAD_PROBE_REQ_SIZE) {
 631                                 IWM_DPRINTF(sc, IWM_DEBUG_FIRMWARE_TLV,
 632                                     "%s: IWM_UCODE_TLV_PROBE_MAX_LEN "
 633                                     "ridiculous\n", __func__);
 634                                 error = EINVAL;
 635                                 goto parse_out;
 636                         }
 637                         break;
 638                 case IWM_UCODE_TLV_PAN:
 639                         if (tlv_len) {
 640                                 device_printf(sc->sc_dev,
 641                                     "%s: IWM_UCODE_TLV_PAN: tlv_len (%u) > 0\n",
 642                                     __func__, tlv_len);
 643                                 error = EINVAL;
 644                                 goto parse_out;
 645                         }
 646                         capa->flags |= IWM_UCODE_TLV_FLAGS_PAN;
 647                         break;
 648                 case IWM_UCODE_TLV_FLAGS:
 649                         if (tlv_len < sizeof(uint32_t)) {
 650                                 device_printf(sc->sc_dev,
 651                                     "%s: IWM_UCODE_TLV_FLAGS: tlv_len (%u) < sizeof(uint32_t)\n",
 652                                     __func__, tlv_len);
 653                                 error = EINVAL;
 654                                 goto parse_out;
 655                         }
 656                         if (tlv_len % sizeof(uint32_t)) {
 657                                 device_printf(sc->sc_dev,
 658                                     "%s: IWM_UCODE_TLV_FLAGS: tlv_len (%u) %% sizeof(uint32_t)\n",
 659                                     __func__, tlv_len);
 660                                 error = EINVAL;
 661                                 goto parse_out;
 662                         }
 663                         /*
 664                          * Apparently there can be many flags, but Linux driver
 665                          * parses only the first one, and so do we.
 666                          *
 667                          * XXX: why does this override IWM_UCODE_TLV_PAN?
 668                          * Intentional or a bug?  Observations from
 669                          * current firmware file:
 670                          *  1) TLV_PAN is parsed first
 671                          *  2) TLV_FLAGS contains TLV_FLAGS_PAN
 672                          * ==> this resets TLV_PAN to itself... hnnnk
 673                          */
 674                         capa->flags = le32_to_cpup((const uint32_t *)tlv_data);
 675                         break;
 676                 case IWM_UCODE_TLV_CSCHEME:
 677                         if ((error = iwm_store_cscheme(sc,
 678                             tlv_data, tlv_len)) != 0) {
 679                                 device_printf(sc->sc_dev,
 680                                     "%s: iwm_store_cscheme(): returned %d\n",
 681                                     __func__, error);
 682                                 goto parse_out;
 683                         }
 684                         break;
 685                 case IWM_UCODE_TLV_NUM_OF_CPU:
 686                         if (tlv_len != sizeof(uint32_t)) {
 687                                 device_printf(sc->sc_dev,
 688                                     "%s: IWM_UCODE_TLV_NUM_OF_CPU: tlv_len (%u) != sizeof(uint32_t)\n",
 689                                     __func__, tlv_len);
 690                                 error = EINVAL;
 691                                 goto parse_out;
 692                         }
 693                         num_of_cpus = le32_to_cpup((const uint32_t *)tlv_data);
 694                         if (num_of_cpus == 2) {
 695                                 fw->img[IWM_UCODE_REGULAR].is_dual_cpus =
 696                                         TRUE;
 697                                 fw->img[IWM_UCODE_INIT].is_dual_cpus =
 698                                         TRUE;
 699                                 fw->img[IWM_UCODE_WOWLAN].is_dual_cpus =
 700                                         TRUE;
 701                         } else if ((num_of_cpus > 2) || (num_of_cpus < 1)) {
 702                                 device_printf(sc->sc_dev,
 703                                     "%s: Driver supports only 1 or 2 CPUs\n",
 704                                     __func__);
 705                                 error = EINVAL;
 706                                 goto parse_out;
 707                         }
 708                         break;
 709                 case IWM_UCODE_TLV_SEC_RT:
 710                         if ((error = iwm_firmware_store_section(sc,
 711                             IWM_UCODE_REGULAR, tlv_data, tlv_len)) != 0) {
 712                                 device_printf(sc->sc_dev,
 713                                     "%s: IWM_UCODE_REGULAR: iwm_firmware_store_section() failed; %d\n",
 714                                     __func__, error);
 715                                 goto parse_out;
 716                         }
 717                         break;
 718                 case IWM_UCODE_TLV_SEC_INIT:
 719                         if ((error = iwm_firmware_store_section(sc,
 720                             IWM_UCODE_INIT, tlv_data, tlv_len)) != 0) {
 721                                 device_printf(sc->sc_dev,
 722                                     "%s: IWM_UCODE_INIT: iwm_firmware_store_section() failed; %d\n",
 723                                     __func__, error);
 724                                 goto parse_out;
 725                         }
 726                         break;
 727                 case IWM_UCODE_TLV_SEC_WOWLAN:
 728                         if ((error = iwm_firmware_store_section(sc,
 729                             IWM_UCODE_WOWLAN, tlv_data, tlv_len)) != 0) {
 730                                 device_printf(sc->sc_dev,
 731                                     "%s: IWM_UCODE_WOWLAN: iwm_firmware_store_section() failed; %d\n",
 732                                     __func__, error);
 733                                 goto parse_out;
 734                         }
 735                         break;
 736                 case IWM_UCODE_TLV_DEF_CALIB:
 737                         if (tlv_len != sizeof(struct iwm_tlv_calib_data)) {
 738                                 device_printf(sc->sc_dev,
 739                                     "%s: IWM_UCODE_TLV_DEV_CALIB: tlv_len (%u) < sizeof(iwm_tlv_calib_data) (%zu)\n",
 740                                     __func__, tlv_len,
 741                                     sizeof(struct iwm_tlv_calib_data));
 742                                 error = EINVAL;
 743                                 goto parse_out;
 744                         }
 745                         if ((error = iwm_set_default_calib(sc, tlv_data)) != 0) {
 746                                 device_printf(sc->sc_dev,
 747                                     "%s: iwm_set_default_calib() failed: %d\n",
 748                                     __func__, error);
 749                                 goto parse_out;
 750                         }
 751                         break;
 752                 case IWM_UCODE_TLV_PHY_SKU:
 753                         if (tlv_len != sizeof(uint32_t)) {
 754                                 error = EINVAL;
 755                                 device_printf(sc->sc_dev,
 756                                     "%s: IWM_UCODE_TLV_PHY_SKU: tlv_len (%u) < sizeof(uint32_t)\n",
 757                                     __func__, tlv_len);
 758                                 goto parse_out;
 759                         }
 760                         sc->sc_fw.phy_config =
 761                             le32_to_cpup((const uint32_t *)tlv_data);
 762                         sc->sc_fw.valid_tx_ant = (sc->sc_fw.phy_config &
 763                                                   IWM_FW_PHY_CFG_TX_CHAIN) >>
 764                                                   IWM_FW_PHY_CFG_TX_CHAIN_POS;
 765                         sc->sc_fw.valid_rx_ant = (sc->sc_fw.phy_config &
 766                                                   IWM_FW_PHY_CFG_RX_CHAIN) >>
 767                                                   IWM_FW_PHY_CFG_RX_CHAIN_POS;
 768                         break;
 769
 770                 case IWM_UCODE_TLV_API_CHANGES_SET: {
 771                         if (tlv_len != sizeof(struct iwm_ucode_api)) {
 772                                 error = EINVAL;
 773                                 goto parse_out;
 774                         }
 775                         if (iwm_set_ucode_api_flags(sc, tlv_data, capa)) {
 776                                 error = EINVAL;
 777                                 goto parse_out;
 778                         }
 779                         break;
 780                 }
 781
 782                 case IWM_UCODE_TLV_ENABLED_CAPABILITIES: {
 783                         if (tlv_len != sizeof(struct iwm_ucode_capa)) {
 784                                 error = EINVAL;
 785                                 goto parse_out;
 786                         }
 787                         if (iwm_set_ucode_capabilities(sc, tlv_data, capa)) {
 788                                 error = EINVAL;
 789                                 goto parse_out;
 790                         }
 791                         break;
 792                 }
 793
 794                 case IWM_UCODE_TLV_CMD_VERSIONS:
 795                 case IWM_UCODE_TLV_SDIO_ADMA_ADDR:
 796                 case IWM_UCODE_TLV_FW_GSCAN_CAPA:
 797                         /* ignore, not used by current driver */
 798                         break;
 799
 800                 case IWM_UCODE_TLV_SEC_RT_USNIFFER:
 801                         if ((error = iwm_firmware_store_section(sc,
 802                             IWM_UCODE_REGULAR_USNIFFER, tlv_data,
 803                             tlv_len)) != 0)
 804                                 goto parse_out;
 805                         break;
 806
 807                 case IWM_UCODE_TLV_PAGING:
 808                         if (tlv_len != sizeof(uint32_t)) {
 809                                 error = EINVAL;
 810                                 goto parse_out;
 811                         }
 812                         paging_mem_size = le32_to_cpup((const uint32_t *)tlv_data);
 813
 814                         IWM_DPRINTF(sc, IWM_DEBUG_FIRMWARE_TLV,
 815                             "%s: Paging: paging enabled (size = %u bytes)\n",
 816                             __func__, paging_mem_size);
 817                         if (paging_mem_size > IWM_MAX_PAGING_IMAGE_SIZE) {
 818                                 device_printf(sc->sc_dev,
 819                                         "%s: Paging: driver supports up to %u bytes for paging image\n",
 820                                         __func__, IWM_MAX_PAGING_IMAGE_SIZE);
 821                                 error = EINVAL;
 822                                 goto out;
 823                         }
 824                         if (paging_mem_size & (IWM_FW_PAGING_SIZE - 1)) {
 825                                 device_printf(sc->sc_dev,
 826                                     "%s: Paging: image isn't multiple %u\n",
 827                                     __func__, IWM_FW_PAGING_SIZE);
 828                                 error = EINVAL;
 829                                 goto out;
 830                         }
 831
 832                         sc->sc_fw.img[IWM_UCODE_REGULAR].paging_mem_size =
 833                             paging_mem_size;
 834                         usniffer_img = IWM_UCODE_REGULAR_USNIFFER;
 835                         sc->sc_fw.img[usniffer_img].paging_mem_size =
 836                             paging_mem_size;
 837                         break;
 838
 839                 case IWM_UCODE_TLV_N_SCAN_CHANNELS:
 840                         if (tlv_len != sizeof(uint32_t)) {
 841                                 error = EINVAL;
 842                                 goto parse_out;
 843                         }
 844                         capa->n_scan_channels =
 845                             le32_to_cpup((const uint32_t *)tlv_data);
 846                         break;
 847
 848                 case IWM_UCODE_TLV_FW_VERSION:
 849                         if (tlv_len != sizeof(uint32_t) * 3) {
 850                                 error = EINVAL;
 851                                 goto parse_out;
 852                         }
 853                         snprintf(sc->sc_fwver, sizeof(sc->sc_fwver),
 854                             "%d.%d.%d",
 855                             le32toh(((const uint32_t *)tlv_data)[0]),
 856                             le32toh(((const uint32_t *)tlv_data)[1]),
 857                             le32toh(((const uint32_t *)tlv_data)[2]));
 858                         break;
 859
 860                 case IWM_UCODE_TLV_FW_MEM_SEG:
 861                         break;
 862
 863                 default:
 864                         device_printf(sc->sc_dev,
 865                             "%s: unknown firmware section %d, abort\n",
 866                             __func__, tlv_type);
 867                         error = EINVAL;
 868                         goto parse_out;
 869                 }
 870         }
 871
 872         KASSERT(error == 0, ("unhandled error"));
 873
 874  parse_out:
 875         if (error) {
 876                 device_printf(sc->sc_dev, "firmware parse error %d, "
 877                     "section type %d\n", error, tlv_type);
 878         }
 879
 880  out:
 881         if (error) {
 882                 if (fw->fw_fp != NULL)
 883                         iwm_fw_info_free(fw);
 884         }
 885
 886         return error;
 887 }
 888
 889 /*
 890  * DMA resource routines
 891  */
 892
 893 /* fwmem is used to load firmware onto the card */
 894 static int
 895 iwm_alloc_fwmem(struct iwm_softc *sc)
 896 {
 897         /* Must be aligned on a 16-byte boundary. */
 898         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->fw_dma,
 899             IWM_FH_MEM_TB_MAX_LENGTH, 16);
 900 }
 901
 902 /* tx scheduler rings.  not used? */
 903 static int
 904 iwm_alloc_sched(struct iwm_softc *sc)
 905 {
 906         /* TX scheduler rings must be aligned on a 1KB boundary. */
 907         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->sched_dma,
 908             nitems(sc->txq) * sizeof(struct iwm_agn_scd_bc_tbl), 1024);
 909 }
 910
 911 /* keep-warm page is used internally by the card.  see iwl-fh.h for more info */
 912 static int
 913 iwm_alloc_kw(struct iwm_softc *sc)
 914 {
 915         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->kw_dma, 4096, 4096);
 916 }
 917
 918 /* interrupt cause table */
 919 static int
 920 iwm_alloc_ict(struct iwm_softc *sc)
 921 {
 922         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->ict_dma,
 923             IWM_ICT_SIZE, 1<<IWM_ICT_PADDR_SHIFT);
 924 }
 925
 926 static int
 927 iwm_alloc_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring)
 928 {
 929         bus_size_t size;
 930         size_t descsz;
 931         int count, i, error;
 932
 933         ring->cur = 0;
 934         if (sc->cfg->mqrx_supported) {
 935                 count = IWM_RX_MQ_RING_COUNT;
 936                 descsz = sizeof(uint64_t);
 937         } else {
 938                 count = IWM_RX_LEGACY_RING_COUNT;
 939                 descsz = sizeof(uint32_t);
 940         }
 941
 942         /* Allocate RX descriptors (256-byte aligned). */
 943         size = count * descsz;
 944         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->free_desc_dma, size,
 945             256);
 946         if (error != 0) {
 947                 device_printf(sc->sc_dev,
 948                     "could not allocate RX ring DMA memory\n");
 949                 goto fail;
 950         }
 951         ring->desc = ring->free_desc_dma.vaddr;
 952
 953         /* Allocate RX status area (16-byte aligned). */
 954         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->stat_dma,
 955             sizeof(*ring->stat), 16);
 956         if (error != 0) {
 957                 device_printf(sc->sc_dev,
 958                     "could not allocate RX status DMA memory\n");
 959                 goto fail;
 960         }
 961         ring->stat = ring->stat_dma.vaddr;
 962
 963         if (sc->cfg->mqrx_supported) {
 964                 size = count * sizeof(uint32_t);
 965                 error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->used_desc_dma,
 966                     size, 256);
 967                 if (error != 0) {
 968                         device_printf(sc->sc_dev,
 969                             "could not allocate RX ring DMA memory\n");
 970                         goto fail;
 971                 }
 972         }
 973
 974         /* Create RX buffer DMA tag. */
 975         error = bus_dma_tag_create(sc->sc_dmat, 1, 0,
 976             BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
 977             IWM_RBUF_SIZE, 1, IWM_RBUF_SIZE, 0, NULL, NULL, &ring->data_dmat);
 978         if (error != 0) {
 979                 device_printf(sc->sc_dev,
 980                     "%s: could not create RX buf DMA tag, error %d\n",
 981                     __func__, error);
 982                 goto fail;
 983         }
 984
 985         /* Allocate spare bus_dmamap_t for iwm_rx_addbuf() */
 986         error = bus_dmamap_create(ring->data_dmat, 0, &ring->spare_map);
 987         if (error != 0) {
 988                 device_printf(sc->sc_dev,
 989                     "%s: could not create RX buf DMA map, error %d\n",
 990                     __func__, error);
 991                 goto fail;
 992         }
 993
 994         /*
 995          * Allocate and map RX buffers.
 996          */
 997         for (i = 0; i < count; i++) {
 998                 struct iwm_rx_data *data = &ring->data[i];
 999                 error = bus_dmamap_create(ring->data_dmat, 0, &data->map);
1000                 if (error != 0) {
1001                         device_printf(sc->sc_dev,
1002                             "%s: could not create RX buf DMA map, error %d\n",
1003                             __func__, error);
1004                         goto fail;
1005                 }
1006                 data->m = NULL;
1007
1008                 if ((error = iwm_rx_addbuf(sc, IWM_RBUF_SIZE, i)) != 0) {
1009                         goto fail;
1010                 }
1011         }
1012         return 0;
1013
1014 fail:   iwm_free_rx_ring(sc, ring);
1015         return error;
1016 }
1017
1018 static void
1019 iwm_reset_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring)
1020 {
1021         /* Reset the ring state */
1022         ring->cur = 0;
1023
1024         /*
1025          * The hw rx ring index in shared memory must also be cleared,
1026          * otherwise the discrepancy can cause reprocessing chaos.
1027          */
1028         if (sc->rxq.stat)
1029                 memset(sc->rxq.stat, 0, sizeof(*sc->rxq.stat));
1030 }
1031
1032 static void
1033 iwm_free_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring)
1034 {
1035         int count, i;
1036
1037         iwm_dma_contig_free(&ring->free_desc_dma);
1038         iwm_dma_contig_free(&ring->stat_dma);
1039         iwm_dma_contig_free(&ring->used_desc_dma);
1040
1041         count = sc->cfg->mqrx_supported ? IWM_RX_MQ_RING_COUNT :
1042             IWM_RX_LEGACY_RING_COUNT;
1043
1044         for (i = 0; i < count; i++) {
1045                 struct iwm_rx_data *data = &ring->data[i];
1046
1047                 if (data->m != NULL) {
1048                         bus_dmamap_sync(ring->data_dmat, data->map,
1049                             BUS_DMASYNC_POSTREAD);
1050                         bus_dmamap_unload(ring->data_dmat, data->map);
1051                         m_freem(data->m);
1052                         data->m = NULL;
1053                 }
1054                 if (data->map != NULL) {
1055                         bus_dmamap_destroy(ring->data_dmat, data->map);
1056                         data->map = NULL;
1057                 }
1058         }
1059         if (ring->spare_map != NULL) {
1060                 bus_dmamap_destroy(ring->data_dmat, ring->spare_map);
1061                 ring->spare_map = NULL;
1062         }
1063         if (ring->data_dmat != NULL) {
1064                 bus_dma_tag_destroy(ring->data_dmat);
1065                 ring->data_dmat = NULL;
1066         }
1067 }
1068
1069 static int
1070 iwm_alloc_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring, int qid)
1071 {
1072         bus_addr_t paddr;
1073         bus_size_t size;
1074         size_t maxsize;
1075         int nsegments;
1076         int i, error;
1077
1078         ring->qid = qid;
1079         ring->queued = 0;
1080         ring->cur = 0;
1081
1082         /* Allocate TX descriptors (256-byte aligned). */
1083         size = IWM_TX_RING_COUNT * sizeof (struct iwm_tfd);
1084         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->desc_dma, size, 256);
1085         if (error != 0) {
1086                 device_printf(sc->sc_dev,
1087                     "could not allocate TX ring DMA memory\n");
1088                 goto fail;
1089         }
1090         ring->desc = ring->desc_dma.vaddr;
1091
1092         /*
1093          * We only use rings 0 through 9 (4 EDCA + cmd) so there is no need
1094          * to allocate commands space for other rings.
1095          */
1096         if (qid > IWM_MVM_CMD_QUEUE)
1097                 return 0;
1098
1099         size = IWM_TX_RING_COUNT * sizeof(struct iwm_device_cmd);
1100         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->cmd_dma, size, 4);
1101         if (error != 0) {
1102                 device_printf(sc->sc_dev,
1103                     "could not allocate TX cmd DMA memory\n");
1104                 goto fail;
1105         }
1106         ring->cmd = ring->cmd_dma.vaddr;
1107
1108         /* FW commands may require more mapped space than packets. */
1109         if (qid == IWM_MVM_CMD_QUEUE) {
1110                 maxsize = IWM_RBUF_SIZE;
1111                 nsegments = 1;
1112         } else {
1113                 maxsize = MCLBYTES;
1114                 nsegments = IWM_MAX_SCATTER - 2;
1115         }
1116
1117         error = bus_dma_tag_create(sc->sc_dmat, 1, 0,
1118             BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, maxsize,
1119             nsegments, maxsize, 0, NULL, NULL, &ring->data_dmat);
1120         if (error != 0) {
1121                 device_printf(sc->sc_dev, "could not create TX buf DMA tag\n");
1122                 goto fail;
1123         }
1124
1125         paddr = ring->cmd_dma.paddr;
1126         for (i = 0; i < IWM_TX_RING_COUNT; i++) {
1127                 struct iwm_tx_data *data = &ring->data[i];
1128
1129                 data->cmd_paddr = paddr;
1130                 data->scratch_paddr = paddr + sizeof(struct iwm_cmd_header)
1131                     + offsetof(struct iwm_tx_cmd, scratch);
1132                 paddr += sizeof(struct iwm_device_cmd);
1133
1134                 error = bus_dmamap_create(ring->data_dmat, 0, &data->map);
1135                 if (error != 0) {
1136                         device_printf(sc->sc_dev,
1137                             "could not create TX buf DMA map\n");
1138                         goto fail;
1139                 }
1140         }
1141         KASSERT(paddr == ring->cmd_dma.paddr + size,
1142             ("invalid physical address"));
1143         return 0;
1144
1145 fail:   iwm_free_tx_ring(sc, ring);
1146         return error;
1147 }
1148
1149 static void
1150 iwm_reset_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring)
1151 {
1152         int i;
1153
1154         for (i = 0; i < IWM_TX_RING_COUNT; i++) {
1155                 struct iwm_tx_data *data = &ring->data[i];
1156
1157                 if (data->m != NULL) {
1158                         bus_dmamap_sync(ring->data_dmat, data->map,
1159                             BUS_DMASYNC_POSTWRITE);
1160                         bus_dmamap_unload(ring->data_dmat, data->map);
1161                         m_freem(data->m);
1162                         data->m = NULL;
1163                 }
1164         }
1165         /* Clear TX descriptors. */
1166         memset(ring->desc, 0, ring->desc_dma.size);
1167         bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map,
1168             BUS_DMASYNC_PREWRITE);
1169         sc->qfullmsk &= ~(1 << ring->qid);
1170         ring->queued = 0;
1171         ring->cur = 0;
1172
1173         if (ring->qid == IWM_MVM_CMD_QUEUE && sc->cmd_hold_nic_awake)
1174                 iwm_pcie_clear_cmd_in_flight(sc);
1175 }
1176
1177 static void
1178 iwm_free_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring)
1179 {
1180         int i;
1181
1182         iwm_dma_contig_free(&ring->desc_dma);
1183         iwm_dma_contig_free(&ring->cmd_dma);
1184
1185         for (i = 0; i < IWM_TX_RING_COUNT; i++) {
1186                 struct iwm_tx_data *data = &ring->data[i];
1187
1188                 if (data->m != NULL) {
1189                         bus_dmamap_sync(ring->data_dmat, data->map,
1190                             BUS_DMASYNC_POSTWRITE);
1191                         bus_dmamap_unload(ring->data_dmat, data->map);
1192                         m_freem(data->m);
1193                         data->m = NULL;
1194                 }
1195                 if (data->map != NULL) {
1196                         bus_dmamap_destroy(ring->data_dmat, data->map);
1197                         data->map = NULL;
1198                 }
1199         }
1200         if (ring->data_dmat != NULL) {
1201                 bus_dma_tag_destroy(ring->data_dmat);
1202                 ring->data_dmat = NULL;
1203         }
1204 }
1205
1206 /*
1207  * High-level hardware frobbing routines
1208  */
1209
1210 static void
1211 iwm_enable_interrupts(struct iwm_softc *sc)
1212 {
1213         sc->sc_intmask = IWM_CSR_INI_SET_MASK;
1214         IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask);
1215 }
1216
1217 static void
1218 iwm_restore_interrupts(struct iwm_softc *sc)
1219 {
1220         IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask);
1221 }
1222
1223 static void
1224 iwm_disable_interrupts(struct iwm_softc *sc)
1225 {
1226         /* disable interrupts */
1227         IWM_WRITE(sc, IWM_CSR_INT_MASK, 0);
1228
1229         /* acknowledge all interrupts */
1230         IWM_WRITE(sc, IWM_CSR_INT, ~0);
1231         IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, ~0);
1232 }
1233
1234 static void
1235 iwm_ict_reset(struct iwm_softc *sc)
1236 {
1237         iwm_disable_interrupts(sc);
1238
1239         /* Reset ICT table. */
1240         memset(sc->ict_dma.vaddr, 0, IWM_ICT_SIZE);
1241         sc->ict_cur = 0;
1242
1243         /* Set physical address of ICT table (4KB aligned). */
1244         IWM_WRITE(sc, IWM_CSR_DRAM_INT_TBL_REG,
1245             IWM_CSR_DRAM_INT_TBL_ENABLE
1246             | IWM_CSR_DRAM_INIT_TBL_WRITE_POINTER
1247             | IWM_CSR_DRAM_INIT_TBL_WRAP_CHECK
1248             | sc->ict_dma.paddr >> IWM_ICT_PADDR_SHIFT);
1249
1250         /* Switch to ICT interrupt mode in driver. */
1251         sc->sc_flags |= IWM_FLAG_USE_ICT;
1252
1253         /* Re-enable interrupts. */
1254         IWM_WRITE(sc, IWM_CSR_INT, ~0);
1255         iwm_enable_interrupts(sc);
1256 }
1257
1258 /* iwlwifi pcie/trans.c */
1259
1260 /*
1261  * Since this .. hard-resets things, it's time to actually
1262  * mark the first vap (if any) as having no mac context.
1263  * It's annoying, but since the driver is potentially being
1264  * stop/start'ed whilst active (thanks openbsd port!) we
1265  * have to correctly track this.
1266  */
1267 static void
1268 iwm_stop_device(struct iwm_softc *sc)
1269 {
1270         struct ieee80211com *ic = &sc->sc_ic;
1271         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
1272         int chnl, qid;
1273         uint32_t mask = 0;
1274
1275         /* tell the device to stop sending interrupts */
1276         iwm_disable_interrupts(sc);
1277
1278         /*
1279          * FreeBSD-local: mark the first vap as not-uploaded,
1280          * so the next transition through auth/assoc
1281          * will correctly populate the MAC context.
1282          */
1283         if (vap) {
1284                 struct iwm_vap *iv = IWM_VAP(vap);
1285                 iv->phy_ctxt = NULL;
1286                 iv->is_uploaded = 0;
1287         }
1288         sc->sc_firmware_state = 0;
1289         sc->sc_flags &= ~IWM_FLAG_TE_ACTIVE;
1290
1291         /* device going down, Stop using ICT table */
1292         sc->sc_flags &= ~IWM_FLAG_USE_ICT;
1293
1294         /* stop tx and rx.  tx and rx bits, as usual, are from if_iwn */
1295
1296         if (iwm_nic_lock(sc)) {
1297                 iwm_write_prph(sc, IWM_SCD_TXFACT, 0);
1298
1299                 /* Stop each Tx DMA channel */
1300                 for (chnl = 0; chnl < IWM_FH_TCSR_CHNL_NUM; chnl++) {
1301                         IWM_WRITE(sc,
1302                             IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl), 0);
1303                         mask |= IWM_FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(chnl);
1304                 }
1305
1306                 /* Wait for DMA channels to be idle */
1307                 if (!iwm_poll_bit(sc, IWM_FH_TSSR_TX_STATUS_REG, mask, mask,
1308                     5000)) {
1309                         device_printf(sc->sc_dev,
1310                             "Failing on timeout while stopping DMA channel: [0x%08x]\n",
1311                             IWM_READ(sc, IWM_FH_TSSR_TX_STATUS_REG));
1312                 }
1313                 iwm_nic_unlock(sc);
1314         }
1315         iwm_pcie_rx_stop(sc);
1316
1317         /* Stop RX ring. */
1318         iwm_reset_rx_ring(sc, &sc->rxq);
1319
1320         /* Reset all TX rings. */
1321         for (qid = 0; qid < nitems(sc->txq); qid++)
1322                 iwm_reset_tx_ring(sc, &sc->txq[qid]);
1323
1324         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
1325                 /* Power-down device's busmaster DMA clocks */
1326                 if (iwm_nic_lock(sc)) {
1327                         iwm_write_prph(sc, IWM_APMG_CLK_DIS_REG,
1328                             IWM_APMG_CLK_VAL_DMA_CLK_RQT);
1329                         iwm_nic_unlock(sc);
1330                 }
1331                 DELAY(5);
1332         }
1333
1334         /* Make sure (redundant) we've released our request to stay awake */
1335         IWM_CLRBITS(sc, IWM_CSR_GP_CNTRL,
1336             IWM_CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1337
1338         /* Stop the device, and put it in low power state */
1339         iwm_apm_stop(sc);
1340
1341         /* stop and reset the on-board processor */
1342         IWM_SETBITS(sc, IWM_CSR_RESET, IWM_CSR_RESET_REG_FLAG_SW_RESET);
1343         DELAY(5000);
1344
1345         /*
1346          * Upon stop, the APM issues an interrupt if HW RF kill is set.
1347          */
1348         iwm_disable_interrupts(sc);
1349
1350         /*
1351          * Even if we stop the HW, we still want the RF kill
1352          * interrupt
1353          */
1354         iwm_enable_rfkill_int(sc);
1355         iwm_check_rfkill(sc);
1356 }
1357
1358 /* iwlwifi: mvm/ops.c */
1359 static void
1360 iwm_mvm_nic_config(struct iwm_softc *sc)
1361 {
1362         uint8_t radio_cfg_type, radio_cfg_step, radio_cfg_dash;
1363         uint32_t reg_val = 0;
1364         uint32_t phy_config = iwm_mvm_get_phy_config(sc);
1365
1366         radio_cfg_type = (phy_config & IWM_FW_PHY_CFG_RADIO_TYPE) >>
1367             IWM_FW_PHY_CFG_RADIO_TYPE_POS;
1368         radio_cfg_step = (phy_config & IWM_FW_PHY_CFG_RADIO_STEP) >>
1369             IWM_FW_PHY_CFG_RADIO_STEP_POS;
1370         radio_cfg_dash = (phy_config & IWM_FW_PHY_CFG_RADIO_DASH) >>
1371             IWM_FW_PHY_CFG_RADIO_DASH_POS;
1372
1373         /* SKU control */
1374         reg_val |= IWM_CSR_HW_REV_STEP(sc->sc_hw_rev) <<
1375             IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_STEP;
1376         reg_val |= IWM_CSR_HW_REV_DASH(sc->sc_hw_rev) <<
1377             IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_DASH;
1378
1379         /* radio configuration */
1380         reg_val |= radio_cfg_type << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_TYPE;
1381         reg_val |= radio_cfg_step << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_STEP;
1382         reg_val |= radio_cfg_dash << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_DASH;
1383
1384         IWM_WRITE(sc, IWM_CSR_HW_IF_CONFIG_REG, reg_val);
1385
1386         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
1387             "Radio type=0x%x-0x%x-0x%x\n", radio_cfg_type,
1388             radio_cfg_step, radio_cfg_dash);
1389
1390         /*
1391          * W/A : NIC is stuck in a reset state after Early PCIe power off
1392          * (PCIe power is lost before PERST# is asserted), causing ME FW
1393          * to lose ownership and not being able to obtain it back.
1394          */
1395         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
1396                 iwm_set_bits_mask_prph(sc, IWM_APMG_PS_CTRL_REG,
1397                     IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS,
1398                     ~IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS);
1399         }
1400 }
1401
1402 static int
1403 iwm_nic_rx_mq_init(struct iwm_softc *sc)
1404 {
1405         int enabled;
1406
1407         if (!iwm_nic_lock(sc))
1408                 return EBUSY;
1409
1410         /* Stop RX DMA. */
1411         iwm_write_prph(sc, IWM_RFH_RXF_DMA_CFG, 0);
1412         /* Disable RX used and free queue operation. */
1413         iwm_write_prph(sc, IWM_RFH_RXF_RXQ_ACTIVE, 0);
1414
1415         iwm_write_prph64(sc, IWM_RFH_Q0_FRBDCB_BA_LSB,
1416             sc->rxq.free_desc_dma.paddr);
1417         iwm_write_prph64(sc, IWM_RFH_Q0_URBDCB_BA_LSB,
1418             sc->rxq.used_desc_dma.paddr);
1419         iwm_write_prph64(sc, IWM_RFH_Q0_URBD_STTS_WPTR_LSB,
1420             sc->rxq.stat_dma.paddr);
1421         iwm_write_prph(sc, IWM_RFH_Q0_FRBDCB_WIDX, 0);
1422         iwm_write_prph(sc, IWM_RFH_Q0_FRBDCB_RIDX, 0);
1423         iwm_write_prph(sc, IWM_RFH_Q0_URBDCB_WIDX, 0);
1424
1425         /* We configure only queue 0 for now. */
1426         enabled = ((1 << 0) << 16) | (1 << 0);
1427
1428         /* Enable RX DMA, 4KB buffer size. */
1429         iwm_write_prph(sc, IWM_RFH_RXF_DMA_CFG,
1430             IWM_RFH_DMA_EN_ENABLE_VAL |
1431             IWM_RFH_RXF_DMA_RB_SIZE_4K |
1432             IWM_RFH_RXF_DMA_MIN_RB_4_8 |
1433             IWM_RFH_RXF_DMA_DROP_TOO_LARGE_MASK |
1434             IWM_RFH_RXF_DMA_RBDCB_SIZE_512);
1435
1436         /* Enable RX DMA snooping. */
1437         iwm_write_prph(sc, IWM_RFH_GEN_CFG,
1438             IWM_RFH_GEN_CFG_RFH_DMA_SNOOP |
1439             IWM_RFH_GEN_CFG_SERVICE_DMA_SNOOP |
1440             (sc->cfg->integrated ? IWM_RFH_GEN_CFG_RB_CHUNK_SIZE_64 :
1441             IWM_RFH_GEN_CFG_RB_CHUNK_SIZE_128));
1442
1443         /* Enable the configured queue(s). */
1444         iwm_write_prph(sc, IWM_RFH_RXF_RXQ_ACTIVE, enabled);
1445
1446         iwm_nic_unlock(sc);
1447
1448         IWM_WRITE_1(sc, IWM_CSR_INT_COALESCING, IWM_HOST_INT_TIMEOUT_DEF);
1449
1450         IWM_WRITE(sc, IWM_RFH_Q0_FRBDCB_WIDX_TRG, 8);
1451
1452         return (0);
1453 }
1454
1455 static int
1456 iwm_nic_rx_legacy_init(struct iwm_softc *sc)
1457 {
1458
1459         /* Stop Rx DMA */
1460         iwm_pcie_rx_stop(sc);
1461
1462         if (!iwm_nic_lock(sc))
1463                 return EBUSY;
1464
1465         /* reset and flush pointers */
1466         IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_RBDCB_WPTR, 0);
1467         IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_FLUSH_RB_REQ, 0);
1468         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_RDPTR, 0);
1469         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_RBDCB_WPTR_REG, 0);
1470
1471         /* Set physical address of RX ring (256-byte aligned). */
1472         IWM_WRITE(sc,
1473             IWM_FH_RSCSR_CHNL0_RBDCB_BASE_REG,
1474             sc->rxq.free_desc_dma.paddr >> 8);
1475
1476         /* Set physical address of RX status (16-byte aligned). */
1477         IWM_WRITE(sc,
1478             IWM_FH_RSCSR_CHNL0_STTS_WPTR_REG, sc->rxq.stat_dma.paddr >> 4);
1479
1480         /* Enable Rx DMA
1481          * XXX 5000 HW isn't supported by the iwm(4) driver.
1482          * IWM_FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY is set because of HW bug in
1483          *      the credit mechanism in 5000 HW RX FIFO
1484          * Direct rx interrupts to hosts
1485          * Rx buffer size 4 or 8k or 12k
1486          * RB timeout 0x10
1487          * 256 RBDs
1488          */
1489         IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_CONFIG_REG,
1490             IWM_FH_RCSR_RX_CONFIG_CHNL_EN_ENABLE_VAL            |
1491             IWM_FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY               |  /* HW bug */
1492             IWM_FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_INT_HOST_VAL   |
1493             IWM_FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K            |
1494             (IWM_RX_RB_TIMEOUT << IWM_FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS) |
1495             IWM_RX_QUEUE_SIZE_LOG << IWM_FH_RCSR_RX_CONFIG_RBDCB_SIZE_POS);
1496
1497         IWM_WRITE_1(sc, IWM_CSR_INT_COALESCING, IWM_HOST_INT_TIMEOUT_DEF);
1498
1499         /* W/A for interrupt coalescing bug in 7260 and 3160 */
1500         if (sc->cfg->host_interrupt_operation_mode)
1501                 IWM_SETBITS(sc, IWM_CSR_INT_COALESCING, IWM_HOST_INT_OPER_MODE);
1502
1503         iwm_nic_unlock(sc);
1504
1505         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_WPTR, 8);
1506
1507         return 0;
1508 }
1509
1510 static int
1511 iwm_nic_rx_init(struct iwm_softc *sc)
1512 {
1513         if (sc->cfg->mqrx_supported)
1514                 return iwm_nic_rx_mq_init(sc);
1515         else
1516                 return iwm_nic_rx_legacy_init(sc);
1517 }
1518
1519 static int
1520 iwm_nic_tx_init(struct iwm_softc *sc)
1521 {
1522         int qid;
1523
1524         if (!iwm_nic_lock(sc))
1525                 return EBUSY;
1526
1527         /* Deactivate TX scheduler. */
1528         iwm_write_prph(sc, IWM_SCD_TXFACT, 0);
1529
1530         /* Set physical address of "keep warm" page (16-byte aligned). */
1531         IWM_WRITE(sc, IWM_FH_KW_MEM_ADDR_REG, sc->kw_dma.paddr >> 4);
1532
1533         /* Initialize TX rings. */
1534         for (qid = 0; qid < nitems(sc->txq); qid++) {
1535                 struct iwm_tx_ring *txq = &sc->txq[qid];
1536
1537                 /* Set physical address of TX ring (256-byte aligned). */
1538                 IWM_WRITE(sc, IWM_FH_MEM_CBBC_QUEUE(qid),
1539                     txq->desc_dma.paddr >> 8);
1540                 IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
1541                     "%s: loading ring %d descriptors (%p) at %lx\n",
1542                     __func__,
1543                     qid, txq->desc,
1544                     (unsigned long) (txq->desc_dma.paddr >> 8));
1545         }
1546
1547         iwm_set_bits_prph(sc, IWM_SCD_GP_CTRL,
1548             IWM_SCD_GP_CTRL_AUTO_ACTIVE_MODE |
1549             IWM_SCD_GP_CTRL_ENABLE_31_QUEUES);
1550
1551         iwm_nic_unlock(sc);
1552
1553         return 0;
1554 }
1555
1556 static int
1557 iwm_nic_init(struct iwm_softc *sc)
1558 {
1559         int error;
1560
1561         iwm_apm_init(sc);
1562         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
1563                 iwm_set_pwr(sc);
1564
1565         iwm_mvm_nic_config(sc);
1566
1567         if ((error = iwm_nic_rx_init(sc)) != 0)
1568                 return error;
1569
1570         /*
1571          * Ditto for TX, from iwn
1572          */
1573         if ((error = iwm_nic_tx_init(sc)) != 0)
1574                 return error;
1575
1576         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
1577             "%s: shadow registers enabled\n", __func__);
1578         IWM_SETBITS(sc, IWM_CSR_MAC_SHADOW_REG_CTRL, 0x800fffff);
1579
1580         return 0;
1581 }
1582
1583 int
1584 iwm_enable_txq(struct iwm_softc *sc, int sta_id, int qid, int fifo)
1585 {
1586         int qmsk;
1587
1588         qmsk = 1 << qid;
1589
1590         if (!iwm_nic_lock(sc)) {
1591                 device_printf(sc->sc_dev, "%s: cannot enable txq %d\n",
1592                     __func__, qid);
1593                 return EBUSY;
1594         }
1595
1596         IWM_WRITE(sc, IWM_HBUS_TARG_WRPTR, qid << 8 | 0);
1597
1598         if (qid == IWM_MVM_CMD_QUEUE) {
1599                 /* Disable the scheduler. */
1600                 iwm_write_prph(sc, IWM_SCD_EN_CTRL, 0);
1601
1602                 /* Stop the TX queue prior to configuration. */
1603                 iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid),
1604                     (0 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE) |
1605                     (1 << IWM_SCD_QUEUE_STTS_REG_POS_SCD_ACT_EN));
1606
1607                 iwm_nic_unlock(sc);
1608
1609                 /* Disable aggregations for this queue. */
1610                 iwm_clear_bits_prph(sc, IWM_SCD_AGGR_SEL, qmsk);
1611
1612                 if (!iwm_nic_lock(sc)) {
1613                         device_printf(sc->sc_dev,
1614                             "%s: cannot enable txq %d\n", __func__, qid);
1615                         return EBUSY;
1616                 }
1617                 iwm_write_prph(sc, IWM_SCD_QUEUE_RDPTR(qid), 0);
1618                 iwm_nic_unlock(sc);
1619
1620                 iwm_write_mem32(sc,
1621                     sc->scd_base_addr + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid), 0);
1622                 /* Set scheduler window size and frame limit. */
1623                 iwm_write_mem32(sc,
1624                     sc->scd_base_addr + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid) +
1625                     sizeof(uint32_t),
1626                     ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_POS) &
1627                     IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_MSK) |
1628                     ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_POS) &
1629                     IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_MSK));
1630
1631                 if (!iwm_nic_lock(sc)) {
1632                         device_printf(sc->sc_dev,
1633                             "%s: cannot enable txq %d\n", __func__, qid);
1634                         return EBUSY;
1635                 }
1636                 iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid),
1637                     (1 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE) |
1638                     (fifo << IWM_SCD_QUEUE_STTS_REG_POS_TXF) |
1639                     (1 << IWM_SCD_QUEUE_STTS_REG_POS_WSL) |
1640                     IWM_SCD_QUEUE_STTS_REG_MSK);
1641
1642                 /* Enable the scheduler for this queue. */
1643                 iwm_write_prph(sc, IWM_SCD_EN_CTRL, qmsk);
1644         } else {
1645                 struct iwm_scd_txq_cfg_cmd cmd;
1646                 int error;
1647
1648                 iwm_nic_unlock(sc);
1649
1650                 memset(&cmd, 0, sizeof(cmd));
1651                 cmd.scd_queue = qid;
1652                 cmd.enable = 1;
1653                 cmd.sta_id = sta_id;
1654                 cmd.tx_fifo = fifo;
1655                 cmd.aggregate = 0;
1656                 cmd.window = IWM_FRAME_LIMIT;
1657
1658                 error = iwm_mvm_send_cmd_pdu(sc, IWM_SCD_QUEUE_CFG, IWM_CMD_SYNC,
1659                     sizeof(cmd), &cmd);
1660                 if (error) {
1661                         device_printf(sc->sc_dev,
1662                             "cannot enable txq %d\n", qid);
1663                         return error;
1664                 }
1665
1666                 if (!iwm_nic_lock(sc))
1667                         return EBUSY;
1668         }
1669
1670         iwm_nic_unlock(sc);
1671
1672         IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "%s: enabled txq %d FIFO %d\n",
1673             __func__, qid, fifo);
1674
1675         return 0;
1676 }
1677
1678 static int
1679 iwm_trans_pcie_fw_alive(struct iwm_softc *sc, uint32_t scd_base_addr)
1680 {
1681         int error, chnl;
1682
1683         int clear_dwords = (IWM_SCD_TRANS_TBL_MEM_UPPER_BOUND -
1684             IWM_SCD_CONTEXT_MEM_LOWER_BOUND) / sizeof(uint32_t);
1685
1686         if (!iwm_nic_lock(sc))
1687                 return EBUSY;
1688
1689         iwm_ict_reset(sc);
1690
1691         sc->scd_base_addr = iwm_read_prph(sc, IWM_SCD_SRAM_BASE_ADDR);
1692         if (scd_base_addr != 0 &&
1693             scd_base_addr != sc->scd_base_addr) {
1694                 device_printf(sc->sc_dev,
1695                     "%s: sched addr mismatch: alive: 0x%x prph: 0x%x\n",
1696                     __func__, sc->scd_base_addr, scd_base_addr);
1697         }
1698
1699         iwm_nic_unlock(sc);
1700
1701         /* reset context data, TX status and translation data */
1702         error = iwm_write_mem(sc,
1703             sc->scd_base_addr + IWM_SCD_CONTEXT_MEM_LOWER_BOUND,
1704             NULL, clear_dwords);
1705         if (error)
1706                 return EBUSY;
1707
1708         if (!iwm_nic_lock(sc))
1709                 return EBUSY;
1710
1711         /* Set physical address of TX scheduler rings (1KB aligned). */
1712         iwm_write_prph(sc, IWM_SCD_DRAM_BASE_ADDR, sc->sched_dma.paddr >> 10);
1713
1714         iwm_write_prph(sc, IWM_SCD_CHAINEXT_EN, 0);
1715
1716         iwm_nic_unlock(sc);
1717
1718         /* enable command channel */
1719         error = iwm_enable_txq(sc, 0 /* unused */, IWM_MVM_CMD_QUEUE, 7);
1720         if (error)
1721                 return error;
1722
1723         if (!iwm_nic_lock(sc))
1724                 return EBUSY;
1725
1726         iwm_write_prph(sc, IWM_SCD_TXFACT, 0xff);
1727
1728         /* Enable DMA channels. */
1729         for (chnl = 0; chnl < IWM_FH_TCSR_CHNL_NUM; chnl++) {
1730                 IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl),
1731                     IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE |
1732                     IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE);
1733         }
1734
1735         IWM_SETBITS(sc, IWM_FH_TX_CHICKEN_BITS_REG,
1736             IWM_FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN);
1737
1738         iwm_nic_unlock(sc);
1739
1740         /* Enable L1-Active */
1741         if (sc->cfg->device_family < IWM_DEVICE_FAMILY_8000) {
1742                 iwm_clear_bits_prph(sc, IWM_APMG_PCIDEV_STT_REG,
1743                     IWM_APMG_PCIDEV_STT_VAL_L1_ACT_DIS);
1744         }
1745
1746         return error;
1747 }
1748
1749 /*
1750  * NVM read access and content parsing.  We do not support
1751  * external NVM or writing NVM.
1752  * iwlwifi/mvm/nvm.c
1753  */
1754
1755 /* Default NVM size to read */
1756 #define IWM_NVM_DEFAULT_CHUNK_SIZE      (2*1024)
1757
1758 #define IWM_NVM_WRITE_OPCODE 1
1759 #define IWM_NVM_READ_OPCODE 0
1760
1761 /* load nvm chunk response */
1762 enum {
1763         IWM_READ_NVM_CHUNK_SUCCEED = 0,
1764         IWM_READ_NVM_CHUNK_NOT_VALID_ADDRESS = 1
1765 };
1766
1767 static int
1768 iwm_nvm_read_chunk(struct iwm_softc *sc, uint16_t section,
1769         uint16_t offset, uint16_t length, uint8_t *data, uint16_t *len)
1770 {
1771         struct iwm_nvm_access_cmd nvm_access_cmd = {
1772                 .offset = htole16(offset),
1773                 .length = htole16(length),
1774                 .type = htole16(section),
1775                 .op_code = IWM_NVM_READ_OPCODE,
1776         };
1777         struct iwm_nvm_access_resp *nvm_resp;
1778         struct iwm_rx_packet *pkt;
1779         struct iwm_host_cmd cmd = {
1780                 .id = IWM_NVM_ACCESS_CMD,
1781                 .flags = IWM_CMD_WANT_SKB | IWM_CMD_SEND_IN_RFKILL,
1782                 .data = { &nvm_access_cmd, },
1783         };
1784         int ret, bytes_read, offset_read;
1785         uint8_t *resp_data;
1786
1787         cmd.len[0] = sizeof(struct iwm_nvm_access_cmd);
1788
1789         ret = iwm_send_cmd(sc, &cmd);
1790         if (ret) {
1791                 device_printf(sc->sc_dev,
1792                     "Could not send NVM_ACCESS command (error=%d)\n", ret);
1793                 return ret;
1794         }
1795
1796         pkt = cmd.resp_pkt;
1797
1798         /* Extract NVM response */
1799         nvm_resp = (void *)pkt->data;
1800         ret = le16toh(nvm_resp->status);
1801         bytes_read = le16toh(nvm_resp->length);
1802         offset_read = le16toh(nvm_resp->offset);
1803         resp_data = nvm_resp->data;
1804         if (ret) {
1805                 if ((offset != 0) &&
1806                     (ret == IWM_READ_NVM_CHUNK_NOT_VALID_ADDRESS)) {
1807                         /*
1808                          * meaning of NOT_VALID_ADDRESS:
1809                          * driver try to read chunk from address that is
1810                          * multiple of 2K and got an error since addr is empty.
1811                          * meaning of (offset != 0): driver already
1812                          * read valid data from another chunk so this case
1813                          * is not an error.
1814                          */
1815                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1816                                     "NVM access command failed on offset 0x%x since that section size is multiple 2K\n",
1817                                     offset);
1818                         *len = 0;
1819                         ret = 0;
1820                 } else {
1821                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1822                                     "NVM access command failed with status %d\n", ret);
1823                         ret = EIO;
1824                 }
1825                 goto exit;
1826         }
1827
1828         if (offset_read != offset) {
1829                 device_printf(sc->sc_dev,
1830                     "NVM ACCESS response with invalid offset %d\n",
1831                     offset_read);
1832                 ret = EINVAL;
1833                 goto exit;
1834         }
1835
1836         if (bytes_read > length) {
1837                 device_printf(sc->sc_dev,
1838                     "NVM ACCESS response with too much data "
1839                     "(%d bytes requested, %d bytes received)\n",
1840                     length, bytes_read);
1841                 ret = EINVAL;
1842                 goto exit;
1843         }
1844
1845         /* Write data to NVM */
1846         memcpy(data + offset, resp_data, bytes_read);
1847         *len = bytes_read;
1848
1849  exit:
1850         iwm_free_resp(sc, &cmd);
1851         return ret;
1852 }
1853
1854 /*
1855  * Reads an NVM section completely.
1856  * NICs prior to 7000 family don't have a real NVM, but just read
1857  * section 0 which is the EEPROM. Because the EEPROM reading is unlimited
1858  * by uCode, we need to manually check in this case that we don't
1859  * overflow and try to read more than the EEPROM size.
1860  * For 7000 family NICs, we supply the maximal size we can read, and
1861  * the uCode fills the response with as much data as we can,
1862  * without overflowing, so no check is needed.
1863  */
1864 static int
1865 iwm_nvm_read_section(struct iwm_softc *sc,
1866         uint16_t section, uint8_t *data, uint16_t *len, uint32_t size_read)
1867 {
1868         uint16_t seglen, length, offset = 0;
1869         int ret;
1870
1871         /* Set nvm section read length */
1872         length = IWM_NVM_DEFAULT_CHUNK_SIZE;
1873
1874         seglen = length;
1875
1876         /* Read the NVM until exhausted (reading less than requested) */
1877         while (seglen == length) {
1878                 /* Check no memory assumptions fail and cause an overflow */
1879                 if ((size_read + offset + length) >
1880                     sc->cfg->eeprom_size) {
1881                         device_printf(sc->sc_dev,
1882                             "EEPROM size is too small for NVM\n");
1883                         return ENOBUFS;
1884                 }
1885
1886                 ret = iwm_nvm_read_chunk(sc, section, offset, length, data, &seglen);
1887                 if (ret) {
1888                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1889                                     "Cannot read NVM from section %d offset %d, length %d\n",
1890                                     section, offset, length);
1891                         return ret;
1892                 }
1893                 offset += seglen;
1894         }
1895
1896         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1897                     "NVM section %d read completed\n", section);
1898         *len = offset;
1899         return 0;
1900 }
1901
1902 /*
1903  * BEGIN IWM_NVM_PARSE
1904  */
1905
1906 /* iwlwifi/iwl-nvm-parse.c */
1907
1908 /* NVM offsets (in words) definitions */
1909 enum iwm_nvm_offsets {
1910         /* NVM HW-Section offset (in words) definitions */
1911         IWM_HW_ADDR = 0x15,
1912
1913 /* NVM SW-Section offset (in words) definitions */
1914         IWM_NVM_SW_SECTION = 0x1C0,
1915         IWM_NVM_VERSION = 0,
1916         IWM_RADIO_CFG = 1,
1917         IWM_SKU = 2,
1918         IWM_N_HW_ADDRS = 3,
1919         IWM_NVM_CHANNELS = 0x1E0 - IWM_NVM_SW_SECTION,
1920
1921 /* NVM calibration section offset (in words) definitions */
1922         IWM_NVM_CALIB_SECTION = 0x2B8,
1923         IWM_XTAL_CALIB = 0x316 - IWM_NVM_CALIB_SECTION
1924 };
1925
1926 enum iwm_8000_nvm_offsets {
1927         /* NVM HW-Section offset (in words) definitions */
1928         IWM_HW_ADDR0_WFPM_8000 = 0x12,
1929         IWM_HW_ADDR1_WFPM_8000 = 0x16,
1930         IWM_HW_ADDR0_PCIE_8000 = 0x8A,
1931         IWM_HW_ADDR1_PCIE_8000 = 0x8E,
1932         IWM_MAC_ADDRESS_OVERRIDE_8000 = 1,
1933
1934         /* NVM SW-Section offset (in words) definitions */
1935         IWM_NVM_SW_SECTION_8000 = 0x1C0,
1936         IWM_NVM_VERSION_8000 = 0,
1937         IWM_RADIO_CFG_8000 = 0,
1938         IWM_SKU_8000 = 2,
1939         IWM_N_HW_ADDRS_8000 = 3,
1940
1941         /* NVM REGULATORY -Section offset (in words) definitions */
1942         IWM_NVM_CHANNELS_8000 = 0,
1943         IWM_NVM_LAR_OFFSET_8000_OLD = 0x4C7,
1944         IWM_NVM_LAR_OFFSET_8000 = 0x507,
1945         IWM_NVM_LAR_ENABLED_8000 = 0x7,
1946
1947         /* NVM calibration section offset (in words) definitions */
1948         IWM_NVM_CALIB_SECTION_8000 = 0x2B8,
1949         IWM_XTAL_CALIB_8000 = 0x316 - IWM_NVM_CALIB_SECTION_8000
1950 };
1951
1952 /* SKU Capabilities (actual values from NVM definition) */
1953 enum nvm_sku_bits {
1954         IWM_NVM_SKU_CAP_BAND_24GHZ      = (1 << 0),
1955         IWM_NVM_SKU_CAP_BAND_52GHZ      = (1 << 1),
1956         IWM_NVM_SKU_CAP_11N_ENABLE      = (1 << 2),
1957         IWM_NVM_SKU_CAP_11AC_ENABLE     = (1 << 3),
1958 };
1959
1960 /* radio config bits (actual values from NVM definition) */
1961 #define IWM_NVM_RF_CFG_DASH_MSK(x)   (x & 0x3)         /* bits 0-1   */
1962 #define IWM_NVM_RF_CFG_STEP_MSK(x)   ((x >> 2)  & 0x3) /* bits 2-3   */
1963 #define IWM_NVM_RF_CFG_TYPE_MSK(x)   ((x >> 4)  & 0x3) /* bits 4-5   */
1964 #define IWM_NVM_RF_CFG_PNUM_MSK(x)   ((x >> 6)  & 0x3) /* bits 6-7   */
1965 #define IWM_NVM_RF_CFG_TX_ANT_MSK(x) ((x >> 8)  & 0xF) /* bits 8-11  */
1966 #define IWM_NVM_RF_CFG_RX_ANT_MSK(x) ((x >> 12) & 0xF) /* bits 12-15 */
1967
1968 #define IWM_NVM_RF_CFG_FLAVOR_MSK_8000(x)       (x & 0xF)
1969 #define IWM_NVM_RF_CFG_DASH_MSK_8000(x)         ((x >> 4) & 0xF)
1970 #define IWM_NVM_RF_CFG_STEP_MSK_8000(x)         ((x >> 8) & 0xF)
1971 #define IWM_NVM_RF_CFG_TYPE_MSK_8000(x)         ((x >> 12) & 0xFFF)
1972 #define IWM_NVM_RF_CFG_TX_ANT_MSK_8000(x)       ((x >> 24) & 0xF)
1973 #define IWM_NVM_RF_CFG_RX_ANT_MSK_8000(x)       ((x >> 28) & 0xF)
1974
1975 /**
1976  * enum iwm_nvm_channel_flags - channel flags in NVM
1977  * @IWM_NVM_CHANNEL_VALID: channel is usable for this SKU/geo
1978  * @IWM_NVM_CHANNEL_IBSS: usable as an IBSS channel
1979  * @IWM_NVM_CHANNEL_ACTIVE: active scanning allowed
1980  * @IWM_NVM_CHANNEL_RADAR: radar detection required
1981  * XXX cannot find this (DFS) flag in iwm-nvm-parse.c
1982  * @IWM_NVM_CHANNEL_DFS: dynamic freq selection candidate
1983  * @IWM_NVM_CHANNEL_WIDE: 20 MHz channel okay (?)
1984  * @IWM_NVM_CHANNEL_40MHZ: 40 MHz channel okay (?)
1985  * @IWM_NVM_CHANNEL_80MHZ: 80 MHz channel okay (?)
1986  * @IWM_NVM_CHANNEL_160MHZ: 160 MHz channel okay (?)
1987  */
1988 enum iwm_nvm_channel_flags {
1989         IWM_NVM_CHANNEL_VALID = (1 << 0),
1990         IWM_NVM_CHANNEL_IBSS = (1 << 1),
1991         IWM_NVM_CHANNEL_ACTIVE = (1 << 3),
1992         IWM_NVM_CHANNEL_RADAR = (1 << 4),
1993         IWM_NVM_CHANNEL_DFS = (1 << 7),
1994         IWM_NVM_CHANNEL_WIDE = (1 << 8),
1995         IWM_NVM_CHANNEL_40MHZ = (1 << 9),
1996         IWM_NVM_CHANNEL_80MHZ = (1 << 10),
1997         IWM_NVM_CHANNEL_160MHZ = (1 << 11),
1998 };
1999
2000 /*
2001  * Translate EEPROM flags to net80211.
2002  */
2003 static uint32_t
2004 iwm_eeprom_channel_flags(uint16_t ch_flags)
2005 {
2006         uint32_t nflags;
2007
2008         nflags = 0;
2009         if ((ch_flags & IWM_NVM_CHANNEL_ACTIVE) == 0)
2010                 nflags |= IEEE80211_CHAN_PASSIVE;
2011         if ((ch_flags & IWM_NVM_CHANNEL_IBSS) == 0)
2012                 nflags |= IEEE80211_CHAN_NOADHOC;
2013         if (ch_flags & IWM_NVM_CHANNEL_RADAR) {
2014                 nflags |= IEEE80211_CHAN_DFS;
2015                 /* Just in case. */
2016                 nflags |= IEEE80211_CHAN_NOADHOC;
2017         }
2018
2019         return (nflags);
2020 }
2021
2022 static void
2023 iwm_add_channel_band(struct iwm_softc *sc, struct ieee80211_channel chans[],
2024     int maxchans, int *nchans, int ch_idx, size_t ch_num,
2025     const uint8_t bands[])
2026 {
2027         const uint16_t * const nvm_ch_flags = sc->nvm_data->nvm_ch_flags;
2028         uint32_t nflags;
2029         uint16_t ch_flags;
2030         uint8_t ieee;
2031         int error;
2032
2033         for (; ch_idx < ch_num; ch_idx++) {
2034                 ch_flags = le16_to_cpup(nvm_ch_flags + ch_idx);
2035                 if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
2036                         ieee = iwm_nvm_channels[ch_idx];
2037                 else
2038                         ieee = iwm_nvm_channels_8000[ch_idx];
2039
2040                 if (!(ch_flags & IWM_NVM_CHANNEL_VALID)) {
2041                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM,
2042                             "Ch. %d Flags %x [%sGHz] - No traffic\n",
2043                             ieee, ch_flags,
2044                             (ch_idx >= IWM_NUM_2GHZ_CHANNELS) ?
2045                             "5.2" : "2.4");
2046                         continue;
2047                 }
2048
2049                 nflags = iwm_eeprom_channel_flags(ch_flags);
2050                 error = ieee80211_add_channel(chans, maxchans, nchans,
2051                     ieee, 0, 0, nflags, bands);
2052                 if (error != 0)
2053                         break;
2054
2055                 IWM_DPRINTF(sc, IWM_DEBUG_EEPROM,
2056                     "Ch. %d Flags %x [%sGHz] - Added\n",
2057                     ieee, ch_flags,
2058                     (ch_idx >= IWM_NUM_2GHZ_CHANNELS) ?
2059                     "5.2" : "2.4");
2060         }
2061 }
2062
2063 static void
2064 iwm_init_channel_map(struct ieee80211com *ic, int maxchans, int *nchans,
2065     struct ieee80211_channel chans[])
2066 {
2067         struct iwm_softc *sc = ic->ic_softc;
2068         struct iwm_nvm_data *data = sc->nvm_data;
2069         uint8_t bands[IEEE80211_MODE_BYTES];
2070         size_t ch_num;
2071
2072         memset(bands, 0, sizeof(bands));
2073         /* 1-13: 11b/g channels. */
2074         setbit(bands, IEEE80211_MODE_11B);
2075         setbit(bands, IEEE80211_MODE_11G);
2076         iwm_add_channel_band(sc, chans, maxchans, nchans, 0,
2077             IWM_NUM_2GHZ_CHANNELS - 1, bands);
2078
2079         /* 14: 11b channel only. */
2080         clrbit(bands, IEEE80211_MODE_11G);
2081         iwm_add_channel_band(sc, chans, maxchans, nchans,
2082             IWM_NUM_2GHZ_CHANNELS - 1, IWM_NUM_2GHZ_CHANNELS, bands);
2083
2084         if (data->sku_cap_band_52GHz_enable) {
2085                 if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
2086                         ch_num = nitems(iwm_nvm_channels);
2087                 else
2088                         ch_num = nitems(iwm_nvm_channels_8000);
2089                 memset(bands, 0, sizeof(bands));
2090                 setbit(bands, IEEE80211_MODE_11A);
2091                 iwm_add_channel_band(sc, chans, maxchans, nchans,
2092                     IWM_NUM_2GHZ_CHANNELS, ch_num, bands);
2093         }
2094 }
2095
2096 static void
2097 iwm_set_hw_address_family_8000(struct iwm_softc *sc, struct iwm_nvm_data *data,
2098         const uint16_t *mac_override, const uint16_t *nvm_hw)
2099 {
2100         const uint8_t *hw_addr;
2101
2102         if (mac_override) {
2103                 static const uint8_t reserved_mac[] = {
2104                         0x02, 0xcc, 0xaa, 0xff, 0xee, 0x00
2105                 };
2106
2107                 hw_addr = (const uint8_t *)(mac_override +
2108                                  IWM_MAC_ADDRESS_OVERRIDE_8000);
2109
2110                 /*
2111                  * Store the MAC address from MAO section.
2112                  * No byte swapping is required in MAO section
2113                  */
2114                 IEEE80211_ADDR_COPY(data->hw_addr, hw_addr);
2115
2116                 /*
2117                  * Force the use of the OTP MAC address in case of reserved MAC
2118                  * address in the NVM, or if address is given but invalid.
2119                  */
2120                 if (!IEEE80211_ADDR_EQ(reserved_mac, hw_addr) &&
2121                     !IEEE80211_ADDR_EQ(ieee80211broadcastaddr, data->hw_addr) &&
2122                     iwm_is_valid_ether_addr(data->hw_addr) &&
2123                     !IEEE80211_IS_MULTICAST(data->hw_addr))
2124                         return;
2125
2126                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2127                     "%s: mac address from nvm override section invalid\n",
2128                     __func__);
2129         }
2130
2131         if (nvm_hw) {
2132                 /* read the mac address from WFMP registers */
2133                 uint32_t mac_addr0 =
2134                     htole32(iwm_read_prph(sc, IWM_WFMP_MAC_ADDR_0));
2135                 uint32_t mac_addr1 =
2136                     htole32(iwm_read_prph(sc, IWM_WFMP_MAC_ADDR_1));
2137
2138                 hw_addr = (const uint8_t *)&mac_addr0;
2139                 data->hw_addr[0] = hw_addr[3];
2140                 data->hw_addr[1] = hw_addr[2];
2141                 data->hw_addr[2] = hw_addr[1];
2142                 data->hw_addr[3] = hw_addr[0];
2143
2144                 hw_addr = (const uint8_t *)&mac_addr1;
2145                 data->hw_addr[4] = hw_addr[1];
2146                 data->hw_addr[5] = hw_addr[0];
2147
2148                 return;
2149         }
2150
2151         device_printf(sc->sc_dev, "%s: mac address not found\n", __func__);
2152         memset(data->hw_addr, 0, sizeof(data->hw_addr));
2153 }
2154
2155 static int
2156 iwm_get_sku(const struct iwm_softc *sc, const uint16_t *nvm_sw,
2157             const uint16_t *phy_sku)
2158 {
2159         if (sc->cfg->device_family < IWM_DEVICE_FAMILY_8000)
2160                 return le16_to_cpup(nvm_sw + IWM_SKU);
2161
2162         return le32_to_cpup((const uint32_t *)(phy_sku + IWM_SKU_8000));
2163 }
2164
2165 static int
2166 iwm_get_nvm_version(const struct iwm_softc *sc, const uint16_t *nvm_sw)
2167 {
2168         if (sc->cfg->device_family < IWM_DEVICE_FAMILY_8000)
2169                 return le16_to_cpup(nvm_sw + IWM_NVM_VERSION);
2170         else
2171                 return le32_to_cpup((const uint32_t *)(nvm_sw +
2172                                                 IWM_NVM_VERSION_8000));
2173 }
2174
2175 static int
2176 iwm_get_radio_cfg(const struct iwm_softc *sc, const uint16_t *nvm_sw,
2177                   const uint16_t *phy_sku)
2178 {
2179         if (sc->cfg->device_family < IWM_DEVICE_FAMILY_8000)
2180                 return le16_to_cpup(nvm_sw + IWM_RADIO_CFG);
2181
2182         return le32_to_cpup((const uint32_t *)(phy_sku + IWM_RADIO_CFG_8000));
2183 }
2184
2185 static int
2186 iwm_get_n_hw_addrs(const struct iwm_softc *sc, const uint16_t *nvm_sw)
2187 {
2188         int n_hw_addr;
2189
2190         if (sc->cfg->device_family < IWM_DEVICE_FAMILY_8000)
2191                 return le16_to_cpup(nvm_sw + IWM_N_HW_ADDRS);
2192
2193         n_hw_addr = le32_to_cpup((const uint32_t *)(nvm_sw + IWM_N_HW_ADDRS_8000));
2194
2195         return n_hw_addr & IWM_N_HW_ADDR_MASK;
2196 }
2197
2198 static void
2199 iwm_set_radio_cfg(const struct iwm_softc *sc, struct iwm_nvm_data *data,
2200                   uint32_t radio_cfg)
2201 {
2202         if (sc->cfg->device_family < IWM_DEVICE_FAMILY_8000) {
2203                 data->radio_cfg_type = IWM_NVM_RF_CFG_TYPE_MSK(radio_cfg);
2204                 data->radio_cfg_step = IWM_NVM_RF_CFG_STEP_MSK(radio_cfg);
2205                 data->radio_cfg_dash = IWM_NVM_RF_CFG_DASH_MSK(radio_cfg);
2206                 data->radio_cfg_pnum = IWM_NVM_RF_CFG_PNUM_MSK(radio_cfg);
2207                 return;
2208         }
2209
2210         /* set the radio configuration for family 8000 */
2211         data->radio_cfg_type = IWM_NVM_RF_CFG_TYPE_MSK_8000(radio_cfg);
2212         data->radio_cfg_step = IWM_NVM_RF_CFG_STEP_MSK_8000(radio_cfg);
2213         data->radio_cfg_dash = IWM_NVM_RF_CFG_DASH_MSK_8000(radio_cfg);
2214         data->radio_cfg_pnum = IWM_NVM_RF_CFG_FLAVOR_MSK_8000(radio_cfg);
2215         data->valid_tx_ant = IWM_NVM_RF_CFG_TX_ANT_MSK_8000(radio_cfg);
2216         data->valid_rx_ant = IWM_NVM_RF_CFG_RX_ANT_MSK_8000(radio_cfg);
2217 }
2218
2219 static int
2220 iwm_set_hw_address(struct iwm_softc *sc, struct iwm_nvm_data *data,
2221                    const uint16_t *nvm_hw, const uint16_t *mac_override)
2222 {
2223 #ifdef notyet /* for FAMILY 9000 */
2224         if (cfg->mac_addr_from_csr) {
2225                 iwm_set_hw_address_from_csr(sc, data);
2226         } else
2227 #endif
2228         if (sc->cfg->device_family < IWM_DEVICE_FAMILY_8000) {
2229                 const uint8_t *hw_addr = (const uint8_t *)(nvm_hw + IWM_HW_ADDR);
2230
2231                 /* The byte order is little endian 16 bit, meaning 214365 */
2232                 data->hw_addr[0] = hw_addr[1];
2233                 data->hw_addr[1] = hw_addr[0];
2234                 data->hw_addr[2] = hw_addr[3];
2235                 data->hw_addr[3] = hw_addr[2];
2236                 data->hw_addr[4] = hw_addr[5];
2237                 data->hw_addr[5] = hw_addr[4];
2238         } else {
2239                 iwm_set_hw_address_family_8000(sc, data, mac_override, nvm_hw);
2240         }
2241
2242         if (!iwm_is_valid_ether_addr(data->hw_addr)) {
2243                 device_printf(sc->sc_dev, "no valid mac address was found\n");
2244                 return EINVAL;
2245         }
2246
2247         return 0;
2248 }
2249
2250 static struct iwm_nvm_data *
2251 iwm_parse_nvm_data(struct iwm_softc *sc,
2252                    const uint16_t *nvm_hw, const uint16_t *nvm_sw,
2253                    const uint16_t *nvm_calib, const uint16_t *mac_override,
2254                    const uint16_t *phy_sku, const uint16_t *regulatory)
2255 {
2256         struct iwm_nvm_data *data;
2257         uint32_t sku, radio_cfg;
2258         uint16_t lar_config;
2259
2260         if (sc->cfg->device_family < IWM_DEVICE_FAMILY_8000) {
2261                 data = malloc(sizeof(*data) +
2262                     IWM_NUM_CHANNELS * sizeof(uint16_t),
2263                     M_DEVBUF, M_NOWAIT | M_ZERO);
2264         } else {
2265                 data = malloc(sizeof(*data) +
2266                     IWM_NUM_CHANNELS_8000 * sizeof(uint16_t),
2267                     M_DEVBUF, M_NOWAIT | M_ZERO);
2268         }
2269         if (!data)
2270                 return NULL;
2271
2272         data->nvm_version = iwm_get_nvm_version(sc, nvm_sw);
2273
2274         radio_cfg = iwm_get_radio_cfg(sc, nvm_sw, phy_sku);
2275         iwm_set_radio_cfg(sc, data, radio_cfg);
2276
2277         sku = iwm_get_sku(sc, nvm_sw, phy_sku);
2278         data->sku_cap_band_24GHz_enable = sku & IWM_NVM_SKU_CAP_BAND_24GHZ;
2279         data->sku_cap_band_52GHz_enable = sku & IWM_NVM_SKU_CAP_BAND_52GHZ;
2280         data->sku_cap_11n_enable = 0;
2281
2282         data->n_hw_addrs = iwm_get_n_hw_addrs(sc, nvm_sw);
2283
2284         if (sc->cfg->device_family >= IWM_DEVICE_FAMILY_8000) {
2285                 /* TODO: use IWL_NVM_EXT */
2286                 uint16_t lar_offset = data->nvm_version < 0xE39 ?
2287                                        IWM_NVM_LAR_OFFSET_8000_OLD :
2288                                        IWM_NVM_LAR_OFFSET_8000;
2289
2290                 lar_config = le16_to_cpup(regulatory + lar_offset);
2291                 data->lar_enabled = !!(lar_config &
2292                                        IWM_NVM_LAR_ENABLED_8000);
2293         }
2294
2295         /* If no valid mac address was found - bail out */
2296         if (iwm_set_hw_address(sc, data, nvm_hw, mac_override)) {
2297                 free(data, M_DEVBUF);
2298                 return NULL;
2299         }
2300
2301         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
2302                 memcpy(data->nvm_ch_flags, sc->cfg->nvm_type == IWM_NVM_SDP ?
2303                     &regulatory[0] : &nvm_sw[IWM_NVM_CHANNELS],
2304                     IWM_NUM_CHANNELS * sizeof(uint16_t));
2305         } else {
2306                 memcpy(data->nvm_ch_flags, &regulatory[IWM_NVM_CHANNELS_8000],
2307                     IWM_NUM_CHANNELS_8000 * sizeof(uint16_t));
2308         }
2309
2310         return data;
2311 }
2312
2313 static void
2314 iwm_free_nvm_data(struct iwm_nvm_data *data)
2315 {
2316         if (data != NULL)
2317                 free(data, M_DEVBUF);
2318 }
2319
2320 static struct iwm_nvm_data *
2321 iwm_parse_nvm_sections(struct iwm_softc *sc, struct iwm_nvm_section *sections)
2322 {
2323         const uint16_t *hw, *sw, *calib, *regulatory, *mac_override, *phy_sku;
2324
2325         /* Checking for required sections */
2326         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
2327                 if (!sections[IWM_NVM_SECTION_TYPE_SW].data ||
2328                     !sections[sc->cfg->nvm_hw_section_num].data) {
2329                         device_printf(sc->sc_dev,
2330                             "Can't parse empty OTP/NVM sections\n");
2331                         return NULL;
2332                 }
2333         } else if (sc->cfg->device_family >= IWM_DEVICE_FAMILY_8000) {
2334                 /* SW and REGULATORY sections are mandatory */
2335                 if (!sections[IWM_NVM_SECTION_TYPE_SW].data ||
2336                     !sections[IWM_NVM_SECTION_TYPE_REGULATORY].data) {
2337                         device_printf(sc->sc_dev,
2338                             "Can't parse empty OTP/NVM sections\n");
2339                         return NULL;
2340                 }
2341                 /* MAC_OVERRIDE or at least HW section must exist */
2342                 if (!sections[sc->cfg->nvm_hw_section_num].data &&
2343                     !sections[IWM_NVM_SECTION_TYPE_MAC_OVERRIDE].data) {
2344                         device_printf(sc->sc_dev,
2345                             "Can't parse mac_address, empty sections\n");
2346                         return NULL;
2347                 }
2348
2349                 /* PHY_SKU section is mandatory in B0 */
2350                 if (!sections[IWM_NVM_SECTION_TYPE_PHY_SKU].data) {
2351                         device_printf(sc->sc_dev,
2352                             "Can't parse phy_sku in B0, empty sections\n");
2353                         return NULL;
2354                 }
2355         } else {
2356                 panic("unknown device family %d\n", sc->cfg->device_family);
2357         }
2358
2359         hw = (const uint16_t *) sections[sc->cfg->nvm_hw_section_num].data;
2360         sw = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_SW].data;
2361         calib = (const uint16_t *)
2362             sections[IWM_NVM_SECTION_TYPE_CALIBRATION].data;
2363         regulatory = sc->cfg->nvm_type == IWM_NVM_SDP ?
2364             (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_REGULATORY_SDP].data :
2365             (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_REGULATORY].data;
2366         mac_override = (const uint16_t *)
2367             sections[IWM_NVM_SECTION_TYPE_MAC_OVERRIDE].data;
2368         phy_sku = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_PHY_SKU].data;
2369
2370         return iwm_parse_nvm_data(sc, hw, sw, calib, mac_override,
2371             phy_sku, regulatory);
2372 }
2373
2374 static int
2375 iwm_nvm_init(struct iwm_softc *sc)
2376 {
2377         struct iwm_nvm_section nvm_sections[IWM_NVM_MAX_NUM_SECTIONS];
2378         int i, ret, section;
2379         uint32_t size_read = 0;
2380         uint8_t *nvm_buffer, *temp;
2381         uint16_t len;
2382
2383         memset(nvm_sections, 0, sizeof(nvm_sections));
2384
2385         if (sc->cfg->nvm_hw_section_num >= IWM_NVM_MAX_NUM_SECTIONS)
2386                 return EINVAL;
2387
2388         /* load NVM values from nic */
2389         /* Read From FW NVM */
2390         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM, "Read from NVM\n");
2391
2392         nvm_buffer = malloc(sc->cfg->eeprom_size, M_DEVBUF, M_NOWAIT | M_ZERO);
2393         if (!nvm_buffer)
2394                 return ENOMEM;
2395         for (section = 0; section < IWM_NVM_MAX_NUM_SECTIONS; section++) {
2396                 /* we override the constness for initial read */
2397                 ret = iwm_nvm_read_section(sc, section, nvm_buffer,
2398                                            &len, size_read);
2399                 if (ret)
2400                         continue;
2401                 size_read += len;
2402                 temp = malloc(len, M_DEVBUF, M_NOWAIT);
2403                 if (!temp) {
2404                         ret = ENOMEM;
2405                         break;
2406                 }
2407                 memcpy(temp, nvm_buffer, len);
2408
2409                 nvm_sections[section].data = temp;
2410                 nvm_sections[section].length = len;
2411         }
2412         if (!size_read)
2413                 device_printf(sc->sc_dev, "OTP is blank\n");
2414         free(nvm_buffer, M_DEVBUF);
2415
2416         sc->nvm_data = iwm_parse_nvm_sections(sc, nvm_sections);
2417         if (!sc->nvm_data)
2418                 return EINVAL;
2419         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
2420                     "nvm version = %x\n", sc->nvm_data->nvm_version);
2421
2422         for (i = 0; i < IWM_NVM_MAX_NUM_SECTIONS; i++) {
2423                 if (nvm_sections[i].data != NULL)
2424                         free(nvm_sections[i].data, M_DEVBUF);
2425         }
2426
2427         return 0;
2428 }
2429
2430 static int
2431 iwm_pcie_load_section(struct iwm_softc *sc, uint8_t section_num,
2432         const struct iwm_fw_desc *section)
2433 {
2434         struct iwm_dma_info *dma = &sc->fw_dma;
2435         uint8_t *v_addr;
2436         bus_addr_t p_addr;
2437         uint32_t offset, chunk_sz = MIN(IWM_FH_MEM_TB_MAX_LENGTH, section->len);
2438         int ret = 0;
2439
2440         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2441                     "%s: [%d] uCode section being loaded...\n",
2442                     __func__, section_num);
2443
2444         v_addr = dma->vaddr;
2445         p_addr = dma->paddr;
2446
2447         for (offset = 0; offset < section->len; offset += chunk_sz) {
2448                 uint32_t copy_size, dst_addr;
2449                 int extended_addr = FALSE;
2450
2451                 copy_size = MIN(chunk_sz, section->len - offset);
2452                 dst_addr = section->offset + offset;
2453
2454                 if (dst_addr >= IWM_FW_MEM_EXTENDED_START &&
2455                     dst_addr <= IWM_FW_MEM_EXTENDED_END)
2456                         extended_addr = TRUE;
2457
2458                 if (extended_addr)
2459                         iwm_set_bits_prph(sc, IWM_LMPM_CHICK,
2460                                           IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE);
2461
2462                 memcpy(v_addr, (const uint8_t *)section->data + offset,
2463                     copy_size);
2464                 bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREWRITE);
2465                 ret = iwm_pcie_load_firmware_chunk(sc, dst_addr, p_addr,
2466                                                    copy_size);
2467
2468                 if (extended_addr)
2469                         iwm_clear_bits_prph(sc, IWM_LMPM_CHICK,
2470                                             IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE);
2471
2472                 if (ret) {
2473                         device_printf(sc->sc_dev,
2474                             "%s: Could not load the [%d] uCode section\n",
2475                             __func__, section_num);
2476                         break;
2477                 }
2478         }
2479
2480         return ret;
2481 }
2482
2483 /*
2484  * ucode
2485  */
2486 static int
2487 iwm_pcie_load_firmware_chunk(struct iwm_softc *sc, uint32_t dst_addr,
2488                              bus_addr_t phy_addr, uint32_t byte_cnt)
2489 {
2490         sc->sc_fw_chunk_done = 0;
2491
2492         if (!iwm_nic_lock(sc))
2493                 return EBUSY;
2494
2495         IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(IWM_FH_SRVC_CHNL),
2496             IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE);
2497
2498         IWM_WRITE(sc, IWM_FH_SRVC_CHNL_SRAM_ADDR_REG(IWM_FH_SRVC_CHNL),
2499             dst_addr);
2500
2501         IWM_WRITE(sc, IWM_FH_TFDIB_CTRL0_REG(IWM_FH_SRVC_CHNL),
2502             phy_addr & IWM_FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK);
2503
2504         IWM_WRITE(sc, IWM_FH_TFDIB_CTRL1_REG(IWM_FH_SRVC_CHNL),
2505             (iwm_get_dma_hi_addr(phy_addr)
2506              << IWM_FH_MEM_TFDIB_REG1_ADDR_BITSHIFT) | byte_cnt);
2507
2508         IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_BUF_STS_REG(IWM_FH_SRVC_CHNL),
2509             1 << IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM |
2510             1 << IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX |
2511             IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID);
2512
2513         IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(IWM_FH_SRVC_CHNL),
2514             IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE    |
2515             IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE |
2516             IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD);
2517
2518         iwm_nic_unlock(sc);
2519
2520         /* wait up to 5s for this segment to load */
2521         msleep(&sc->sc_fw, &sc->sc_mtx, 0, "iwmfw", hz * 5);
2522
2523         if (!sc->sc_fw_chunk_done) {
2524                 device_printf(sc->sc_dev,
2525                     "fw chunk addr 0x%x len %d failed to load\n",
2526                     dst_addr, byte_cnt);
2527                 return ETIMEDOUT;
2528         }
2529
2530         return 0;
2531 }
2532
2533 static int
2534 iwm_pcie_load_cpu_sections_8000(struct iwm_softc *sc,
2535         const struct iwm_fw_img *image, int cpu, int *first_ucode_section)
2536 {
2537         int shift_param;
2538         int i, ret = 0, sec_num = 0x1;
2539         uint32_t val, last_read_idx = 0;
2540
2541         if (cpu == 1) {
2542                 shift_param = 0;
2543                 *first_ucode_section = 0;
2544         } else {
2545                 shift_param = 16;
2546                 (*first_ucode_section)++;
2547         }
2548
2549         for (i = *first_ucode_section; i < IWM_UCODE_SECTION_MAX; i++) {
2550                 last_read_idx = i;
2551
2552                 /*
2553                  * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
2554                  * CPU1 to CPU2.
2555                  * PAGING_SEPARATOR_SECTION delimiter - separate between
2556                  * CPU2 non paged to CPU2 paging sec.
2557                  */
2558                 if (!image->sec[i].data ||
2559                     image->sec[i].offset == IWM_CPU1_CPU2_SEPARATOR_SECTION ||
2560                     image->sec[i].offset == IWM_PAGING_SEPARATOR_SECTION) {
2561                         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2562                                     "Break since Data not valid or Empty section, sec = %d\n",
2563                                     i);
2564                         break;
2565                 }
2566                 ret = iwm_pcie_load_section(sc, i, &image->sec[i]);
2567                 if (ret)
2568                         return ret;
2569
2570                 /* Notify the ucode of the loaded section number and status */
2571                 if (iwm_nic_lock(sc)) {
2572                         val = IWM_READ(sc, IWM_FH_UCODE_LOAD_STATUS);
2573                         val = val | (sec_num << shift_param);
2574                         IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, val);
2575                         sec_num = (sec_num << 1) | 0x1;
2576                         iwm_nic_unlock(sc);
2577                 }
2578         }
2579
2580         *first_ucode_section = last_read_idx;
2581
2582         iwm_enable_interrupts(sc);
2583
2584         if (iwm_nic_lock(sc)) {
2585                 if (cpu == 1)
2586                         IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, 0xFFFF);
2587                 else
2588                         IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, 0xFFFFFFFF);
2589                 iwm_nic_unlock(sc);
2590         }
2591
2592         return 0;
2593 }
2594
2595 static int
2596 iwm_pcie_load_cpu_sections(struct iwm_softc *sc,
2597         const struct iwm_fw_img *image, int cpu, int *first_ucode_section)
2598 {
2599         int shift_param;
2600         int i, ret = 0;
2601         uint32_t last_read_idx = 0;
2602
2603         if (cpu == 1) {
2604                 shift_param = 0;
2605                 *first_ucode_section = 0;
2606         } else {
2607                 shift_param = 16;
2608                 (*first_ucode_section)++;
2609         }
2610
2611         for (i = *first_ucode_section; i < IWM_UCODE_SECTION_MAX; i++) {
2612                 last_read_idx = i;
2613
2614                 /*
2615                  * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
2616                  * CPU1 to CPU2.
2617                  * PAGING_SEPARATOR_SECTION delimiter - separate between
2618                  * CPU2 non paged to CPU2 paging sec.
2619                  */
2620                 if (!image->sec[i].data ||
2621                     image->sec[i].offset == IWM_CPU1_CPU2_SEPARATOR_SECTION ||
2622                     image->sec[i].offset == IWM_PAGING_SEPARATOR_SECTION) {
2623                         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2624                                     "Break since Data not valid or Empty section, sec = %d\n",
2625                                      i);
2626                         break;
2627                 }
2628
2629                 ret = iwm_pcie_load_section(sc, i, &image->sec[i]);
2630                 if (ret)
2631                         return ret;
2632         }
2633
2634         *first_ucode_section = last_read_idx;
2635
2636         return 0;
2637
2638 }
2639
2640 static int
2641 iwm_pcie_load_given_ucode(struct iwm_softc *sc, const struct iwm_fw_img *image)
2642 {
2643         int ret = 0;
2644         int first_ucode_section;
2645
2646         IWM_DPRINTF(sc, IWM_DEBUG_RESET, "working with %s CPU\n",
2647                      image->is_dual_cpus ? "Dual" : "Single");
2648
2649         /* load to FW the binary non secured sections of CPU1 */
2650         ret = iwm_pcie_load_cpu_sections(sc, image, 1, &first_ucode_section);
2651         if (ret)
2652                 return ret;
2653
2654         if (image->is_dual_cpus) {
2655                 /* set CPU2 header address */
2656                 if (iwm_nic_lock(sc)) {
2657                         iwm_write_prph(sc,
2658                                        IWM_LMPM_SECURE_UCODE_LOAD_CPU2_HDR_ADDR,
2659                                        IWM_LMPM_SECURE_CPU2_HDR_MEM_SPACE);
2660                         iwm_nic_unlock(sc);
2661                 }
2662
2663                 /* load to FW the binary sections of CPU2 */
2664                 ret = iwm_pcie_load_cpu_sections(sc, image, 2,
2665                                                  &first_ucode_section);
2666                 if (ret)
2667                         return ret;
2668         }
2669
2670         iwm_enable_interrupts(sc);
2671
2672         /* release CPU reset */
2673         IWM_WRITE(sc, IWM_CSR_RESET, 0);
2674
2675         return 0;
2676 }
2677
2678 int
2679 iwm_pcie_load_given_ucode_8000(struct iwm_softc *sc,
2680         const struct iwm_fw_img *image)
2681 {
2682         int ret = 0;
2683         int first_ucode_section;
2684
2685         IWM_DPRINTF(sc, IWM_DEBUG_RESET, "working with %s CPU\n",
2686                     image->is_dual_cpus ? "Dual" : "Single");
2687
2688         /* configure the ucode to be ready to get the secured image */
2689         /* release CPU reset */
2690         if (iwm_nic_lock(sc)) {
2691                 iwm_write_prph(sc, IWM_RELEASE_CPU_RESET,
2692                     IWM_RELEASE_CPU_RESET_BIT);
2693                 iwm_nic_unlock(sc);
2694         }
2695
2696         /* load to FW the binary Secured sections of CPU1 */
2697         ret = iwm_pcie_load_cpu_sections_8000(sc, image, 1,
2698             &first_ucode_section);
2699         if (ret)
2700                 return ret;
2701
2702         /* load to FW the binary sections of CPU2 */
2703         return iwm_pcie_load_cpu_sections_8000(sc, image, 2,
2704             &first_ucode_section);
2705 }
2706
2707 /* XXX Get rid of this definition */
2708 static inline void
2709 iwm_enable_fw_load_int(struct iwm_softc *sc)
2710 {
2711         IWM_DPRINTF(sc, IWM_DEBUG_INTR, "Enabling FW load interrupt\n");
2712         sc->sc_intmask = IWM_CSR_INT_BIT_FH_TX;
2713         IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask);
2714 }
2715
2716 /* XXX Add proper rfkill support code */
2717 static int
2718 iwm_start_fw(struct iwm_softc *sc, const struct iwm_fw_img *fw)
2719 {
2720         int ret;
2721
2722         /* This may fail if AMT took ownership of the device */
2723         if (iwm_prepare_card_hw(sc)) {
2724                 device_printf(sc->sc_dev,
2725                     "%s: Exit HW not ready\n", __func__);
2726                 ret = EIO;
2727                 goto out;
2728         }
2729
2730         IWM_WRITE(sc, IWM_CSR_INT, 0xFFFFFFFF);
2731
2732         iwm_disable_interrupts(sc);
2733
2734         /* make sure rfkill handshake bits are cleared */
2735         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL);
2736         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR,
2737             IWM_CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED);
2738
2739         /* clear (again), then enable host interrupts */
2740         IWM_WRITE(sc, IWM_CSR_INT, 0xFFFFFFFF);
2741
2742         ret = iwm_nic_init(sc);
2743         if (ret) {
2744                 device_printf(sc->sc_dev, "%s: Unable to init nic\n", __func__);
2745                 goto out;
2746         }
2747
2748         /*
2749          * Now, we load the firmware and don't want to be interrupted, even
2750          * by the RF-Kill interrupt (hence mask all the interrupt besides the
2751          * FH_TX interrupt which is needed to load the firmware). If the
2752          * RF-Kill switch is toggled, we will find out after having loaded
2753          * the firmware and return the proper value to the caller.
2754          */
2755         iwm_enable_fw_load_int(sc);
2756
2757         /* really make sure rfkill handshake bits are cleared */
2758         /* maybe we should write a few times more?  just to make sure */
2759         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL);
2760         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL);
2761
2762         /* Load the given image to the HW */
2763         if (sc->cfg->device_family >= IWM_DEVICE_FAMILY_8000)
2764                 ret = iwm_pcie_load_given_ucode_8000(sc, fw);
2765         else
2766                 ret = iwm_pcie_load_given_ucode(sc, fw);
2767
2768         /* XXX re-check RF-Kill state */
2769
2770 out:
2771         return ret;
2772 }
2773
2774 static int
2775 iwm_send_tx_ant_cfg(struct iwm_softc *sc, uint8_t valid_tx_ant)
2776 {
2777         struct iwm_tx_ant_cfg_cmd tx_ant_cmd = {
2778                 .valid = htole32(valid_tx_ant),
2779         };
2780
2781         return iwm_mvm_send_cmd_pdu(sc, IWM_TX_ANT_CONFIGURATION_CMD,
2782             IWM_CMD_SYNC, sizeof(tx_ant_cmd), &tx_ant_cmd);
2783 }
2784
2785 /* iwlwifi: mvm/fw.c */
2786 static int
2787 iwm_send_phy_cfg_cmd(struct iwm_softc *sc)
2788 {
2789         struct iwm_phy_cfg_cmd phy_cfg_cmd;
2790         enum iwm_ucode_type ucode_type = sc->cur_ucode;
2791
2792         /* Set parameters */
2793         phy_cfg_cmd.phy_cfg = htole32(iwm_mvm_get_phy_config(sc));
2794         phy_cfg_cmd.calib_control.event_trigger =
2795             sc->sc_default_calib[ucode_type].event_trigger;
2796         phy_cfg_cmd.calib_control.flow_trigger =
2797             sc->sc_default_calib[ucode_type].flow_trigger;
2798
2799         IWM_DPRINTF(sc, IWM_DEBUG_CMD | IWM_DEBUG_RESET,
2800             "Sending Phy CFG command: 0x%x\n", phy_cfg_cmd.phy_cfg);
2801         return iwm_mvm_send_cmd_pdu(sc, IWM_PHY_CONFIGURATION_CMD, IWM_CMD_SYNC,
2802             sizeof(phy_cfg_cmd), &phy_cfg_cmd);
2803 }
2804
2805 static int
2806 iwm_alive_fn(struct iwm_softc *sc, struct iwm_rx_packet *pkt, void *data)
2807 {
2808         struct iwm_mvm_alive_data *alive_data = data;
2809         struct iwm_mvm_alive_resp_v3 *palive3;
2810         struct iwm_mvm_alive_resp *palive;
2811         struct iwm_umac_alive *umac;
2812         struct iwm_lmac_alive *lmac1;
2813         struct iwm_lmac_alive *lmac2 = NULL;
2814         uint16_t status;
2815
2816         if (iwm_rx_packet_payload_len(pkt) == sizeof(*palive)) {
2817                 palive = (void *)pkt->data;
2818                 umac = &palive->umac_data;
2819                 lmac1 = &palive->lmac_data[0];
2820                 lmac2 = &palive->lmac_data[1];
2821                 status = le16toh(palive->status);
2822         } else {
2823                 palive3 = (void *)pkt->data;
2824                 umac = &palive3->umac_data;
2825                 lmac1 = &palive3->lmac_data;
2826                 status = le16toh(palive3->status);
2827         }
2828
2829         sc->error_event_table[0] = le32toh(lmac1->error_event_table_ptr);
2830         if (lmac2)
2831                 sc->error_event_table[1] =
2832                         le32toh(lmac2->error_event_table_ptr);
2833         sc->log_event_table = le32toh(lmac1->log_event_table_ptr);
2834         sc->umac_error_event_table = le32toh(umac->error_info_addr);
2835         alive_data->scd_base_addr = le32toh(lmac1->scd_base_ptr);
2836         alive_data->valid = status == IWM_ALIVE_STATUS_OK;
2837         if (sc->umac_error_event_table)
2838                 sc->support_umac_log = TRUE;
2839
2840         IWM_DPRINTF(sc, IWM_DEBUG_FW,
2841                     "Alive ucode status 0x%04x revision 0x%01X 0x%01X\n",
2842                     status, lmac1->ver_type, lmac1->ver_subtype);
2843
2844         if (lmac2)
2845                 IWM_DPRINTF(sc, IWM_DEBUG_FW, "Alive ucode CDB\n");
2846
2847         IWM_DPRINTF(sc, IWM_DEBUG_FW,
2848                     "UMAC version: Major - 0x%x, Minor - 0x%x\n",
2849                     le32toh(umac->umac_major),
2850                     le32toh(umac->umac_minor));
2851
2852         return TRUE;
2853 }
2854
2855 static int
2856 iwm_wait_phy_db_entry(struct iwm_softc *sc,
2857         struct iwm_rx_packet *pkt, void *data)
2858 {
2859         struct iwm_phy_db *phy_db = data;
2860
2861         if (pkt->hdr.code != IWM_CALIB_RES_NOTIF_PHY_DB) {
2862                 if(pkt->hdr.code != IWM_INIT_COMPLETE_NOTIF) {
2863                         device_printf(sc->sc_dev, "%s: Unexpected cmd: %d\n",
2864                             __func__, pkt->hdr.code);
2865                 }
2866                 return TRUE;
2867         }
2868
2869         if (iwm_phy_db_set_section(phy_db, pkt)) {
2870                 device_printf(sc->sc_dev,
2871                     "%s: iwm_phy_db_set_section failed\n", __func__);
2872         }
2873
2874         return FALSE;
2875 }
2876
2877 static int
2878 iwm_mvm_load_ucode_wait_alive(struct iwm_softc *sc,
2879         enum iwm_ucode_type ucode_type)
2880 {
2881         struct iwm_notification_wait alive_wait;
2882         struct iwm_mvm_alive_data alive_data;
2883         const struct iwm_fw_img *fw;
2884         enum iwm_ucode_type old_type = sc->cur_ucode;
2885         int error;
2886         static const uint16_t alive_cmd[] = { IWM_MVM_ALIVE };
2887
2888         fw = &sc->sc_fw.img[ucode_type];
2889         sc->cur_ucode = ucode_type;
2890         sc->ucode_loaded = FALSE;
2891
2892         memset(&alive_data, 0, sizeof(alive_data));
2893         iwm_init_notification_wait(sc->sc_notif_wait, &alive_wait,
2894                                    alive_cmd, nitems(alive_cmd),
2895                                    iwm_alive_fn, &alive_data);
2896
2897         error = iwm_start_fw(sc, fw);
2898         if (error) {
2899                 device_printf(sc->sc_dev, "iwm_start_fw: failed %d\n", error);
2900                 sc->cur_ucode = old_type;
2901                 iwm_remove_notification(sc->sc_notif_wait, &alive_wait);
2902                 return error;
2903         }
2904
2905         /*
2906          * Some things may run in the background now, but we
2907          * just wait for the ALIVE notification here.
2908          */
2909         IWM_UNLOCK(sc);
2910         error = iwm_wait_notification(sc->sc_notif_wait, &alive_wait,
2911                                       IWM_MVM_UCODE_ALIVE_TIMEOUT);
2912         IWM_LOCK(sc);
2913         if (error) {
2914                 if (sc->cfg->device_family >= IWM_DEVICE_FAMILY_8000) {
2915                         uint32_t a = 0x5a5a5a5a, b = 0x5a5a5a5a;
2916                         if (iwm_nic_lock(sc)) {
2917                                 a = iwm_read_prph(sc, IWM_SB_CPU_1_STATUS);
2918                                 b = iwm_read_prph(sc, IWM_SB_CPU_2_STATUS);
2919                                 iwm_nic_unlock(sc);
2920                         }
2921                         device_printf(sc->sc_dev,
2922                             "SecBoot CPU1 Status: 0x%x, CPU2 Status: 0x%x\n",
2923                             a, b);
2924                 }
2925                 sc->cur_ucode = old_type;
2926                 return error;
2927         }
2928
2929         if (!alive_data.valid) {
2930                 device_printf(sc->sc_dev, "%s: Loaded ucode is not valid\n",
2931                     __func__);
2932                 sc->cur_ucode = old_type;
2933                 return EIO;
2934         }
2935
2936         iwm_trans_pcie_fw_alive(sc, alive_data.scd_base_addr);
2937
2938         /*
2939          * configure and operate fw paging mechanism.
2940          * driver configures the paging flow only once, CPU2 paging image
2941          * included in the IWM_UCODE_INIT image.
2942          */
2943         if (fw->paging_mem_size) {
2944                 error = iwm_save_fw_paging(sc, fw);
2945                 if (error) {
2946                         device_printf(sc->sc_dev,
2947                             "%s: failed to save the FW paging image\n",
2948                             __func__);
2949                         return error;
2950                 }
2951
2952                 error = iwm_send_paging_cmd(sc, fw);
2953                 if (error) {
2954                         device_printf(sc->sc_dev,
2955                             "%s: failed to send the paging cmd\n", __func__);
2956                         iwm_free_fw_paging(sc);
2957                         return error;
2958                 }
2959         }
2960
2961         if (!error)
2962                 sc->ucode_loaded = TRUE;
2963         return error;
2964 }
2965
2966 /*
2967  * mvm misc bits
2968  */
2969
2970 /*
2971  * follows iwlwifi/fw.c
2972  */
2973 static int
2974 iwm_run_init_mvm_ucode(struct iwm_softc *sc, int justnvm)
2975 {
2976         struct iwm_notification_wait calib_wait;
2977         static const uint16_t init_complete[] = {
2978                 IWM_INIT_COMPLETE_NOTIF,
2979                 IWM_CALIB_RES_NOTIF_PHY_DB
2980         };
2981         int ret;
2982
2983         /* do not operate with rfkill switch turned on */
2984         if ((sc->sc_flags & IWM_FLAG_RFKILL) && !justnvm) {
2985                 device_printf(sc->sc_dev,
2986                     "radio is disabled by hardware switch\n");
2987                 return EPERM;
2988         }
2989
2990         iwm_init_notification_wait(sc->sc_notif_wait,
2991                                    &calib_wait,
2992                                    init_complete,
2993                                    nitems(init_complete),
2994                                    iwm_wait_phy_db_entry,
2995                                    sc->sc_phy_db);
2996
2997         /* Will also start the device */
2998         ret = iwm_mvm_load_ucode_wait_alive(sc, IWM_UCODE_INIT);
2999         if (ret) {
3000                 device_printf(sc->sc_dev, "Failed to start INIT ucode: %d\n",
3001                     ret);
3002                 goto error;
3003         }
3004
3005         if (sc->cfg->device_family < IWM_DEVICE_FAMILY_8000) {
3006                 ret = iwm_send_bt_init_conf(sc);
3007                 if (ret) {
3008                         device_printf(sc->sc_dev,
3009                             "failed to send bt coex configuration: %d\n", ret);
3010                         goto error;
3011                 }
3012         }
3013
3014         if (justnvm) {
3015                 /* Read nvm */
3016                 ret = iwm_nvm_init(sc);
3017                 if (ret) {
3018                         device_printf(sc->sc_dev, "failed to read nvm\n");
3019                         goto error;
3020                 }
3021                 IEEE80211_ADDR_COPY(sc->sc_ic.ic_macaddr, sc->nvm_data->hw_addr);
3022                 goto error;
3023         }
3024
3025         /* Send TX valid antennas before triggering calibrations */
3026         ret = iwm_send_tx_ant_cfg(sc, iwm_mvm_get_valid_tx_ant(sc));
3027         if (ret) {
3028                 device_printf(sc->sc_dev,
3029                     "failed to send antennas before calibration: %d\n", ret);
3030                 goto error;
3031         }
3032
3033         /*
3034          * Send phy configurations command to init uCode
3035          * to start the 16.0 uCode init image internal calibrations.
3036          */
3037         ret = iwm_send_phy_cfg_cmd(sc);
3038         if (ret) {
3039                 device_printf(sc->sc_dev,
3040                     "%s: Failed to run INIT calibrations: %d\n",
3041                     __func__, ret);
3042                 goto error;
3043         }
3044
3045         /*
3046          * Nothing to do but wait for the init complete notification
3047          * from the firmware.
3048          */
3049         IWM_UNLOCK(sc);
3050         ret = iwm_wait_notification(sc->sc_notif_wait, &calib_wait,
3051             IWM_MVM_UCODE_CALIB_TIMEOUT);
3052         IWM_LOCK(sc);
3053
3054
3055         goto out;
3056
3057 error:
3058         iwm_remove_notification(sc->sc_notif_wait, &calib_wait);
3059 out:
3060         return ret;
3061 }
3062
3063 static int
3064 iwm_mvm_config_ltr(struct iwm_softc *sc)
3065 {
3066         struct iwm_ltr_config_cmd cmd = {
3067                 .flags = htole32(IWM_LTR_CFG_FLAG_FEATURE_ENABLE),
3068         };
3069
3070         if (!sc->sc_ltr_enabled)
3071                 return 0;
3072
3073         return iwm_mvm_send_cmd_pdu(sc, IWM_LTR_CONFIG, 0, sizeof(cmd), &cmd);
3074 }
3075
3076 /*
3077  * receive side
3078  */
3079
3080 /* (re)stock rx ring, called at init-time and at runtime */
3081 static int
3082 iwm_rx_addbuf(struct iwm_softc *sc, int size, int idx)
3083 {
3084         struct iwm_rx_ring *ring = &sc->rxq;
3085         struct iwm_rx_data *data = &ring->data[idx];
3086         struct mbuf *m;
3087         bus_dmamap_t dmamap;
3088         bus_dma_segment_t seg;
3089         int nsegs, error;
3090
3091         m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, IWM_RBUF_SIZE);
3092         if (m == NULL)
3093                 return ENOBUFS;
3094
3095         m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
3096         error = bus_dmamap_load_mbuf_sg(ring->data_dmat, ring->spare_map, m,
3097             &seg, &nsegs, BUS_DMA_NOWAIT);
3098         if (error != 0) {
3099                 device_printf(sc->sc_dev,
3100                     "%s: can't map mbuf, error %d\n", __func__, error);
3101                 m_freem(m);
3102                 return error;
3103         }
3104
3105         if (data->m != NULL)
3106                 bus_dmamap_unload(ring->data_dmat, data->map);
3107
3108         /* Swap ring->spare_map with data->map */
3109         dmamap = data->map;
3110         data->map = ring->spare_map;
3111         ring->spare_map = dmamap;
3112
3113         bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_PREREAD);
3114         data->m = m;
3115
3116         /* Update RX descriptor. */
3117         KASSERT((seg.ds_addr & 255) == 0, ("seg.ds_addr not aligned"));
3118         if (sc->cfg->mqrx_supported)
3119                 ((uint64_t *)ring->desc)[idx] = htole64(seg.ds_addr);
3120         else
3121                 ((uint32_t *)ring->desc)[idx] = htole32(seg.ds_addr >> 8);
3122         bus_dmamap_sync(ring->free_desc_dma.tag, ring->free_desc_dma.map,
3123             BUS_DMASYNC_PREWRITE);
3124
3125         return 0;
3126 }
3127
3128 static void
3129 iwm_mvm_rx_rx_phy_cmd(struct iwm_softc *sc, struct iwm_rx_packet *pkt)
3130 {
3131         struct iwm_rx_phy_info *phy_info = (void *)pkt->data;
3132
3133         IWM_DPRINTF(sc, IWM_DEBUG_RECV, "received PHY stats\n");
3134
3135         memcpy(&sc->sc_last_phy_info, phy_info, sizeof(sc->sc_last_phy_info));
3136 }
3137
3138 /*
3139  * Retrieve the average noise (in dBm) among receivers.
3140  */
3141 static int
3142 iwm_get_noise(struct iwm_softc *sc,
3143     const struct iwm_mvm_statistics_rx_non_phy *stats)
3144 {
3145         int i, total, nbant, noise;
3146
3147         total = nbant = noise = 0;
3148         for (i = 0; i < 3; i++) {
3149                 noise = le32toh(stats->beacon_silence_rssi[i]) & 0xff;
3150                 IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: i=%d, noise=%d\n",
3151                     __func__,
3152                     i,
3153                     noise);
3154
3155                 if (noise) {
3156                         total += noise;
3157                         nbant++;
3158                 }
3159         }
3160
3161         IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: nbant=%d, total=%d\n",
3162             __func__, nbant, total);
3163 #if 0
3164         /* There should be at least one antenna but check anyway. */
3165         return (nbant == 0) ? -127 : (total / nbant) - 107;
3166 #else
3167         /* For now, just hard-code it to -96 to be safe */
3168         return (-96);
3169 #endif
3170 }
3171
3172 static void
3173 iwm_mvm_handle_rx_statistics(struct iwm_softc *sc, struct iwm_rx_packet *pkt)
3174 {
3175         struct iwm_notif_statistics_v10 *stats = (void *)&pkt->data;
3176
3177         memcpy(&sc->sc_stats, stats, sizeof(sc->sc_stats));
3178         sc->sc_noise = iwm_get_noise(sc, &stats->rx.general);
3179 }
3180
3181 /* iwlwifi: mvm/rx.c */
3182 /*
3183  * iwm_mvm_get_signal_strength - use new rx PHY INFO API
3184  * values are reported by the fw as positive values - need to negate
3185  * to obtain their dBM.  Account for missing antennas by replacing 0
3186  * values by -256dBm: practically 0 power and a non-feasible 8 bit value.
3187  */
3188 static int
3189 iwm_mvm_rx_get_signal_strength(struct iwm_softc *sc,
3190     struct iwm_rx_phy_info *phy_info)
3191 {
3192         int energy_a, energy_b, energy_c, max_energy;
3193         uint32_t val;
3194
3195         val = le32toh(phy_info->non_cfg_phy[IWM_RX_INFO_ENERGY_ANT_ABC_IDX]);
3196         energy_a = (val & IWM_RX_INFO_ENERGY_ANT_A_MSK) >>
3197             IWM_RX_INFO_ENERGY_ANT_A_POS;
3198         energy_a = energy_a ? -energy_a : -256;
3199         energy_b = (val & IWM_RX_INFO_ENERGY_ANT_B_MSK) >>
3200             IWM_RX_INFO_ENERGY_ANT_B_POS;
3201         energy_b = energy_b ? -energy_b : -256;
3202         energy_c = (val & IWM_RX_INFO_ENERGY_ANT_C_MSK) >>
3203             IWM_RX_INFO_ENERGY_ANT_C_POS;
3204         energy_c = energy_c ? -energy_c : -256;
3205         max_energy = MAX(energy_a, energy_b);
3206         max_energy = MAX(max_energy, energy_c);
3207
3208         IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3209             "energy In A %d B %d C %d , and max %d\n",
3210             energy_a, energy_b, energy_c, max_energy);
3211
3212         return max_energy;
3213 }
3214
3215 static int
3216 iwm_mvm_rxmq_get_signal_strength(struct iwm_softc *sc,
3217     struct iwm_rx_mpdu_desc *desc)
3218 {
3219         int energy_a, energy_b;
3220
3221         energy_a = desc->v1.energy_a;
3222         energy_b = desc->v1.energy_b;
3223         energy_a = energy_a ? -energy_a : -256;
3224         energy_b = energy_b ? -energy_b : -256;
3225         return MAX(energy_a, energy_b);
3226 }
3227
3228 /*
3229  * iwm_mvm_rx_rx_mpdu - IWM_REPLY_RX_MPDU_CMD handler
3230  *
3231  * Handles the actual data of the Rx packet from the fw
3232  */
3233 static bool
3234 iwm_mvm_rx_rx_mpdu(struct iwm_softc *sc, struct mbuf *m, uint32_t offset,
3235     bool stolen)
3236 {
3237         struct ieee80211com *ic = &sc->sc_ic;
3238         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
3239         struct ieee80211_frame *wh;
3240         struct ieee80211_rx_stats rxs;
3241         struct iwm_rx_phy_info *phy_info;
3242         struct iwm_rx_mpdu_res_start *rx_res;
3243         struct iwm_rx_packet *pkt = mtodoff(m, struct iwm_rx_packet *, offset);
3244         uint32_t len;
3245         uint32_t rx_pkt_status;
3246         int rssi;
3247
3248         phy_info = &sc->sc_last_phy_info;
3249         rx_res = (struct iwm_rx_mpdu_res_start *)pkt->data;
3250         wh = (struct ieee80211_frame *)(pkt->data + sizeof(*rx_res));
3251         len = le16toh(rx_res->byte_count);
3252         rx_pkt_status = le32toh(*(uint32_t *)(pkt->data + sizeof(*rx_res) + len));
3253
3254         if (__predict_false(phy_info->cfg_phy_cnt > 20)) {
3255                 device_printf(sc->sc_dev,
3256                     "dsp size out of range [0,20]: %d\n",
3257                     phy_info->cfg_phy_cnt);
3258                 return false;
3259         }
3260
3261         if (!(rx_pkt_status & IWM_RX_MPDU_RES_STATUS_CRC_OK) ||
3262             !(rx_pkt_status & IWM_RX_MPDU_RES_STATUS_OVERRUN_OK)) {
3263                 IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3264                     "Bad CRC or FIFO: 0x%08X.\n", rx_pkt_status);
3265                 return false;
3266         }
3267
3268         rssi = iwm_mvm_rx_get_signal_strength(sc, phy_info);
3269
3270         /* Map it to relative value */
3271         rssi = rssi - sc->sc_noise;
3272
3273         /* replenish ring for the buffer we're going to feed to the sharks */
3274         if (!stolen && iwm_rx_addbuf(sc, IWM_RBUF_SIZE, sc->rxq.cur) != 0) {
3275                 device_printf(sc->sc_dev, "%s: unable to add more buffers\n",
3276                     __func__);
3277                 return false;
3278         }
3279
3280         m->m_data = pkt->data + sizeof(*rx_res);
3281         m->m_pkthdr.len = m->m_len = len;
3282
3283         IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3284             "%s: rssi=%d, noise=%d\n", __func__, rssi, sc->sc_noise);
3285
3286         IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3287             "%s: phy_info: channel=%d, flags=0x%08x\n",
3288             __func__,
3289             le16toh(phy_info->channel),
3290             le16toh(phy_info->phy_flags));
3291
3292         /*
3293          * Populate an RX state struct with the provided information.
3294          */
3295         bzero(&rxs, sizeof(rxs));
3296         rxs.r_flags |= IEEE80211_R_IEEE | IEEE80211_R_FREQ;
3297         rxs.r_flags |= IEEE80211_R_NF | IEEE80211_R_RSSI;
3298         rxs.c_ieee = le16toh(phy_info->channel);
3299         if (le16toh(phy_info->phy_flags & IWM_RX_RES_PHY_FLAGS_BAND_24)) {
3300                 rxs.c_freq = ieee80211_ieee2mhz(rxs.c_ieee, IEEE80211_CHAN_2GHZ);
3301         } else {
3302                 rxs.c_freq = ieee80211_ieee2mhz(rxs.c_ieee, IEEE80211_CHAN_5GHZ);
3303         }
3304
3305         /* rssi is in 1/2db units */
3306         rxs.c_rssi = rssi * 2;
3307         rxs.c_nf = sc->sc_noise;
3308         if (ieee80211_add_rx_params(m, &rxs) == 0)
3309                 return false;
3310
3311         if (ieee80211_radiotap_active_vap(vap)) {
3312                 struct iwm_rx_radiotap_header *tap = &sc->sc_rxtap;
3313
3314                 tap->wr_flags = 0;
3315                 if (phy_info->phy_flags & htole16(IWM_PHY_INFO_FLAG_SHPREAMBLE))
3316                         tap->wr_flags |= IEEE80211_RADIOTAP_F_SHORTPRE;
3317                 tap->wr_chan_freq = htole16(rxs.c_freq);
3318                 /* XXX only if ic->ic_curchan->ic_ieee == rxs.c_ieee */
3319                 tap->wr_chan_flags = htole16(ic->ic_curchan->ic_flags);
3320                 tap->wr_dbm_antsignal = (int8_t)rssi;
3321                 tap->wr_dbm_antnoise = (int8_t)sc->sc_noise;
3322                 tap->wr_tsft = phy_info->system_timestamp;
3323                 switch (phy_info->rate) {
3324                 /* CCK rates. */
3325                 case  10: tap->wr_rate =   2; break;
3326                 case  20: tap->wr_rate =   4; break;
3327                 case  55: tap->wr_rate =  11; break;
3328                 case 110: tap->wr_rate =  22; break;
3329                 /* OFDM rates. */
3330                 case 0xd: tap->wr_rate =  12; break;
3331                 case 0xf: tap->wr_rate =  18; break;
3332                 case 0x5: tap->wr_rate =  24; break;
3333                 case 0x7: tap->wr_rate =  36; break;
3334                 case 0x9: tap->wr_rate =  48; break;
3335                 case 0xb: tap->wr_rate =  72; break;
3336                 case 0x1: tap->wr_rate =  96; break;
3337                 case 0x3: tap->wr_rate = 108; break;
3338                 /* Unknown rate: should not happen. */
3339                 default:  tap->wr_rate =   0;
3340                 }
3341         }
3342
3343         return true;
3344 }
3345
3346 static bool
3347 iwm_mvm_rx_mpdu_mq(struct iwm_softc *sc, struct mbuf *m, uint32_t offset,
3348     bool stolen)
3349 {
3350         struct ieee80211com *ic = &sc->sc_ic;
3351         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
3352         struct ieee80211_frame *wh;
3353         struct ieee80211_rx_stats rxs;
3354         struct iwm_rx_mpdu_desc *desc;
3355         struct iwm_rx_packet *pkt;
3356         int rssi;
3357         uint32_t hdrlen, len, rate_n_flags;
3358         uint16_t phy_info;
3359         uint8_t channel;
3360
3361         pkt = mtodo(m, offset);
3362         desc = (void *)pkt->data;
3363
3364         if (!(desc->status & htole16(IWM_RX_MPDU_RES_STATUS_CRC_OK)) ||
3365             !(desc->status & htole16(IWM_RX_MPDU_RES_STATUS_OVERRUN_OK))) {
3366                 IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3367                     "Bad CRC or FIFO: 0x%08X.\n", desc->status);
3368                 return false;
3369         }
3370
3371         channel = desc->v1.channel;
3372         len = le16toh(desc->mpdu_len);
3373         phy_info = le16toh(desc->phy_info);
3374         rate_n_flags = desc->v1.rate_n_flags;
3375
3376         wh = mtodo(m, sizeof(*desc));
3377         m->m_data = pkt->data + sizeof(*desc);
3378         m->m_pkthdr.len = m->m_len = len;
3379         m->m_len = len;
3380
3381         /* Account for padding following the frame header. */
3382         if ((desc->mac_flags2 & IWM_RX_MPDU_MFLG2_PAD)) {
3383                 hdrlen = ieee80211_anyhdrsize(wh);
3384                 memmove(mtodo(m, 2), mtodo(m, 0), hdrlen);
3385                 m->m_data = mtodo(m, 2);
3386                 wh = mtod(m, struct ieee80211_frame *);
3387         }
3388
3389         /* Map it to relative value */
3390         rssi = iwm_mvm_rxmq_get_signal_strength(sc, desc);
3391         rssi = rssi - sc->sc_noise;
3392
3393         /* replenish ring for the buffer we're going to feed to the sharks */
3394         if (!stolen && iwm_rx_addbuf(sc, IWM_RBUF_SIZE, sc->rxq.cur) != 0) {
3395                 device_printf(sc->sc_dev, "%s: unable to add more buffers\n",
3396                     __func__);
3397                 return false;
3398         }
3399
3400         IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3401             "%s: rssi=%d, noise=%d\n", __func__, rssi, sc->sc_noise);
3402
3403         /*
3404          * Populate an RX state struct with the provided information.
3405          */
3406         bzero(&rxs, sizeof(rxs));
3407         rxs.r_flags |= IEEE80211_R_IEEE | IEEE80211_R_FREQ;
3408         rxs.r_flags |= IEEE80211_R_NF | IEEE80211_R_RSSI;
3409         rxs.c_ieee = channel;
3410         rxs.c_freq = ieee80211_ieee2mhz(rxs.c_ieee,
3411             channel <= 14 ? IEEE80211_CHAN_2GHZ : IEEE80211_CHAN_5GHZ);
3412
3413         /* rssi is in 1/2db units */
3414         rxs.c_rssi = rssi * 2;
3415         rxs.c_nf = sc->sc_noise;
3416         if (ieee80211_add_rx_params(m, &rxs) == 0)
3417                 return false;
3418
3419         if (ieee80211_radiotap_active_vap(vap)) {
3420                 struct iwm_rx_radiotap_header *tap = &sc->sc_rxtap;
3421
3422                 tap->wr_flags = 0;
3423                 if ((phy_info & IWM_RX_MPDU_PHY_SHORT_PREAMBLE) != 0)
3424                         tap->wr_flags |= IEEE80211_RADIOTAP_F_SHORTPRE;
3425                 tap->wr_chan_freq = htole16(rxs.c_freq);
3426                 /* XXX only if ic->ic_curchan->ic_ieee == rxs.c_ieee */
3427                 tap->wr_chan_flags = htole16(ic->ic_curchan->ic_flags);
3428                 tap->wr_dbm_antsignal = (int8_t)rssi;
3429                 tap->wr_dbm_antnoise = (int8_t)sc->sc_noise;
3430                 tap->wr_tsft = desc->v1.gp2_on_air_rise;
3431                 switch ((rate_n_flags & 0xff)) {
3432                 /* CCK rates. */
3433                 case  10: tap->wr_rate =   2; break;
3434                 case  20: tap->wr_rate =   4; break;
3435                 case  55: tap->wr_rate =  11; break;
3436                 case 110: tap->wr_rate =  22; break;
3437                 /* OFDM rates. */
3438                 case 0xd: tap->wr_rate =  12; break;
3439                 case 0xf: tap->wr_rate =  18; break;
3440                 case 0x5: tap->wr_rate =  24; break;
3441                 case 0x7: tap->wr_rate =  36; break;
3442                 case 0x9: tap->wr_rate =  48; break;
3443                 case 0xb: tap->wr_rate =  72; break;
3444                 case 0x1: tap->wr_rate =  96; break;
3445                 case 0x3: tap->wr_rate = 108; break;
3446                 /* Unknown rate: should not happen. */
3447                 default:  tap->wr_rate =   0;
3448                 }
3449         }
3450
3451         return true;
3452 }
3453
3454 static bool
3455 iwm_mvm_rx_mpdu(struct iwm_softc *sc, struct mbuf *m, uint32_t offset,
3456     bool stolen)
3457 {
3458         struct ieee80211com *ic;
3459         struct ieee80211_frame *wh;
3460         struct ieee80211_node *ni;
3461         bool ret;
3462
3463         ic = &sc->sc_ic;
3464
3465         ret = sc->cfg->mqrx_supported ?
3466             iwm_mvm_rx_mpdu_mq(sc, m, offset, stolen) :
3467             iwm_mvm_rx_rx_mpdu(sc, m, offset, stolen);
3468         if (!ret) {
3469                 counter_u64_add(ic->ic_ierrors, 1);
3470                 return (ret);
3471         }
3472
3473         wh = mtod(m, struct ieee80211_frame *);
3474         ni = ieee80211_find_rxnode(ic, (struct ieee80211_frame_min *)wh);
3475
3476         IWM_UNLOCK(sc);
3477         if (ni != NULL) {
3478                 IWM_DPRINTF(sc, IWM_DEBUG_RECV, "input m %p\n", m);
3479                 ieee80211_input_mimo(ni, m);
3480                 ieee80211_free_node(ni);
3481         } else {
3482                 IWM_DPRINTF(sc, IWM_DEBUG_RECV, "inputall m %p\n", m);
3483                 ieee80211_input_mimo_all(ic, m);
3484         }
3485         IWM_LOCK(sc);
3486
3487         return true;
3488 }
3489
3490 static int
3491 iwm_mvm_rx_tx_cmd_single(struct iwm_softc *sc, struct iwm_rx_packet *pkt,
3492         struct iwm_node *in)
3493 {
3494         struct iwm_mvm_tx_resp *tx_resp = (void *)pkt->data;
3495         struct ieee80211_ratectl_tx_status *txs = &sc->sc_txs;
3496         struct ieee80211_node *ni = &in->in_ni;
3497         struct ieee80211vap *vap = ni->ni_vap;
3498         int status = le16toh(tx_resp->status.status) & IWM_TX_STATUS_MSK;
3499         int new_rate, cur_rate = vap->iv_bss->ni_txrate;
3500         boolean_t rate_matched;
3501         uint8_t tx_resp_rate;
3502
3503         KASSERT(tx_resp->frame_count == 1, ("too many frames"));
3504
3505         /* Update rate control statistics. */
3506         IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "%s: status=0x%04x, seq=%d, fc=%d, btc=%d, frts=%d, ff=%d, irate=%08x, wmt=%d\n",
3507             __func__,
3508             (int) le16toh(tx_resp->status.status),
3509             (int) le16toh(tx_resp->status.sequence),
3510             tx_resp->frame_count,
3511             tx_resp->bt_kill_count,
3512             tx_resp->failure_rts,
3513             tx_resp->failure_frame,
3514             le32toh(tx_resp->initial_rate),
3515             (int) le16toh(tx_resp->wireless_media_time));
3516
3517         tx_resp_rate = iwm_rate_from_ucode_rate(le32toh(tx_resp->initial_rate));
3518
3519         /* For rate control, ignore frames sent at different initial rate */
3520         rate_matched = (tx_resp_rate != 0 && tx_resp_rate == cur_rate);
3521
3522         if (tx_resp_rate != 0 && cur_rate != 0 && !rate_matched) {
3523                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3524                     "tx_resp_rate doesn't match ni_txrate (tx_resp_rate=%u "
3525                     "ni_txrate=%d)\n", tx_resp_rate, cur_rate);
3526         }
3527
3528         txs->flags = IEEE80211_RATECTL_STATUS_SHORT_RETRY |
3529                      IEEE80211_RATECTL_STATUS_LONG_RETRY;
3530         txs->short_retries = tx_resp->failure_rts;
3531         txs->long_retries = tx_resp->failure_frame;
3532         if (status != IWM_TX_STATUS_SUCCESS &&
3533             status != IWM_TX_STATUS_DIRECT_DONE) {
3534                 switch (status) {
3535                 case IWM_TX_STATUS_FAIL_SHORT_LIMIT:
3536                         txs->status = IEEE80211_RATECTL_TX_FAIL_SHORT;
3537                         break;
3538                 case IWM_TX_STATUS_FAIL_LONG_LIMIT:
3539                         txs->status = IEEE80211_RATECTL_TX_FAIL_LONG;
3540                         break;
3541                 case IWM_TX_STATUS_FAIL_LIFE_EXPIRE:
3542                         txs->status = IEEE80211_RATECTL_TX_FAIL_EXPIRED;
3543                         break;
3544                 default:
3545                         txs->status = IEEE80211_RATECTL_TX_FAIL_UNSPECIFIED;
3546                         break;
3547                 }
3548         } else {
3549                 txs->status = IEEE80211_RATECTL_TX_SUCCESS;
3550         }
3551
3552         if (rate_matched) {
3553                 ieee80211_ratectl_tx_complete(ni, txs);
3554
3555                 int rix = ieee80211_ratectl_rate(vap->iv_bss, NULL, 0);
3556                 new_rate = vap->iv_bss->ni_txrate;
3557                 if (new_rate != 0 && new_rate != cur_rate) {
3558                         struct iwm_node *in = IWM_NODE(vap->iv_bss);
3559                         iwm_setrates(sc, in, rix);
3560                         iwm_mvm_send_lq_cmd(sc, &in->in_lq, FALSE);
3561                 }
3562         }
3563
3564         return (txs->status != IEEE80211_RATECTL_TX_SUCCESS);
3565 }
3566
3567 static void
3568 iwm_mvm_rx_tx_cmd(struct iwm_softc *sc, struct iwm_rx_packet *pkt)
3569 {
3570         struct iwm_cmd_header *cmd_hdr;
3571         struct iwm_tx_ring *ring;
3572         struct iwm_tx_data *txd;
3573         struct iwm_node *in;
3574         struct mbuf *m;
3575         int idx, qid, qmsk, status;
3576
3577         cmd_hdr = &pkt->hdr;
3578         idx = cmd_hdr->idx;
3579         qid = cmd_hdr->qid;
3580
3581         ring = &sc->txq[qid];
3582         txd = &ring->data[idx];
3583         in = txd->in;
3584         m = txd->m;
3585
3586         KASSERT(txd->done == 0, ("txd not done"));
3587         KASSERT(txd->in != NULL, ("txd without node"));
3588         KASSERT(txd->m != NULL, ("txd without mbuf"));
3589
3590         sc->sc_tx_timer = 0;
3591
3592         status = iwm_mvm_rx_tx_cmd_single(sc, pkt, in);
3593
3594         /* Unmap and free mbuf. */
3595         bus_dmamap_sync(ring->data_dmat, txd->map, BUS_DMASYNC_POSTWRITE);
3596         bus_dmamap_unload(ring->data_dmat, txd->map);
3597
3598         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3599             "free txd %p, in %p\n", txd, txd->in);
3600         txd->done = 1;
3601         txd->m = NULL;
3602         txd->in = NULL;
3603
3604         ieee80211_tx_complete(&in->in_ni, m, status);
3605
3606         qmsk = 1 << qid;
3607         if (--ring->queued < IWM_TX_RING_LOMARK && (sc->qfullmsk & qmsk) != 0) {
3608                 sc->qfullmsk &= ~qmsk;
3609                 if (sc->qfullmsk == 0)
3610                         iwm_start(sc);
3611         }
3612 }
3613
3614 /*
3615  * transmit side
3616  */
3617
3618 /*
3619  * Process a "command done" firmware notification.  This is where we wakeup
3620  * processes waiting for a synchronous command completion.
3621  * from if_iwn
3622  */
3623 static void
3624 iwm_cmd_done(struct iwm_softc *sc, struct iwm_rx_packet *pkt)
3625 {
3626         struct iwm_tx_ring *ring = &sc->txq[IWM_MVM_CMD_QUEUE];
3627         struct iwm_tx_data *data;
3628
3629         if (pkt->hdr.qid != IWM_MVM_CMD_QUEUE) {
3630                 return; /* Not a command ack. */
3631         }
3632
3633         /* XXX wide commands? */
3634         IWM_DPRINTF(sc, IWM_DEBUG_CMD,
3635             "cmd notification type 0x%x qid %d idx %d\n",
3636             pkt->hdr.code, pkt->hdr.qid, pkt->hdr.idx);
3637
3638         data = &ring->data[pkt->hdr.idx];
3639
3640         /* If the command was mapped in an mbuf, free it. */
3641         if (data->m != NULL) {
3642                 bus_dmamap_sync(ring->data_dmat, data->map,
3643                     BUS_DMASYNC_POSTWRITE);
3644                 bus_dmamap_unload(ring->data_dmat, data->map);
3645                 m_freem(data->m);
3646                 data->m = NULL;
3647         }
3648         wakeup(&ring->desc[pkt->hdr.idx]);
3649
3650         if (((pkt->hdr.idx + ring->queued) % IWM_TX_RING_COUNT) != ring->cur) {
3651                 device_printf(sc->sc_dev,
3652                     "%s: Some HCMDs skipped?: idx=%d queued=%d cur=%d\n",
3653                     __func__, pkt->hdr.idx, ring->queued, ring->cur);
3654                 /* XXX call iwm_force_nmi() */
3655         }
3656
3657         KASSERT(ring->queued > 0, ("ring->queued is empty?"));
3658         ring->queued--;
3659         if (ring->queued == 0)
3660                 iwm_pcie_clear_cmd_in_flight(sc);
3661 }
3662
3663 #if 0
3664 /*
3665  * necessary only for block ack mode
3666  */
3667 void
3668 iwm_update_sched(struct iwm_softc *sc, int qid, int idx, uint8_t sta_id,
3669         uint16_t len)
3670 {
3671         struct iwm_agn_scd_bc_tbl *scd_bc_tbl;
3672         uint16_t w_val;
3673
3674         scd_bc_tbl = sc->sched_dma.vaddr;
3675
3676         len += 8; /* magic numbers came naturally from paris */
3677         len = roundup(len, 4) / 4;
3678
3679         w_val = htole16(sta_id << 12 | len);
3680
3681         /* Update TX scheduler. */
3682         scd_bc_tbl[qid].tfd_offset[idx] = w_val;
3683         bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map,
3684             BUS_DMASYNC_PREWRITE);
3685
3686         /* I really wonder what this is ?!? */
3687         if (idx < IWM_TFD_QUEUE_SIZE_BC_DUP) {
3688                 scd_bc_tbl[qid].tfd_offset[IWM_TFD_QUEUE_SIZE_MAX + idx] = w_val;
3689                 bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map,
3690                     BUS_DMASYNC_PREWRITE);
3691         }
3692 }
3693 #endif
3694
3695 static int
3696 iwm_tx_rateidx_global_lookup(struct iwm_softc *sc, uint8_t rate)
3697 {
3698         int i;
3699
3700         for (i = 0; i < nitems(iwm_rates); i++) {
3701                 if (iwm_rates[i].rate == rate)
3702                         return (i);
3703         }
3704         /* XXX error? */
3705         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE,
3706             "%s: couldn't find an entry for rate=%d\n",
3707             __func__,
3708             rate);
3709         return (0);
3710 }
3711
3712 /*
3713  * Fill in the rate related information for a transmit command.
3714  */
3715 static const struct iwm_rate *
3716 iwm_tx_fill_cmd(struct iwm_softc *sc, struct iwm_node *in,
3717         struct mbuf *m, struct iwm_tx_cmd *tx)
3718 {
3719         struct ieee80211_node *ni = &in->in_ni;
3720         struct ieee80211_frame *wh;
3721         const struct ieee80211_txparam *tp = ni->ni_txparms;
3722         const struct iwm_rate *rinfo;
3723         int type;
3724         int ridx, rate_flags;
3725
3726         wh = mtod(m, struct ieee80211_frame *);
3727         type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK;
3728
3729         tx->rts_retry_limit = IWM_RTS_DFAULT_RETRY_LIMIT;
3730         tx->data_retry_limit = IWM_DEFAULT_TX_RETRY;
3731
3732         if (type == IEEE80211_FC0_TYPE_MGT ||
3733             type == IEEE80211_FC0_TYPE_CTL ||
3734             (m->m_flags & M_EAPOL) != 0) {
3735                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->mgmtrate);
3736                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3737                     "%s: MGT (%d)\n", __func__, tp->mgmtrate);
3738         } else if (IEEE80211_IS_MULTICAST(wh->i_addr1)) {
3739                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->mcastrate);
3740                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3741                     "%s: MCAST (%d)\n", __func__, tp->mcastrate);
3742         } else if (tp->ucastrate != IEEE80211_FIXED_RATE_NONE) {
3743                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->ucastrate);
3744                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3745                     "%s: FIXED_RATE (%d)\n", __func__, tp->ucastrate);
3746         } else {
3747                 /* for data frames, use RS table */
3748                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: DATA\n", __func__);
3749                 ridx = iwm_rate2ridx(sc, ni->ni_txrate);
3750                 if (ridx == -1)
3751                         ridx = 0;
3752
3753                 /* This is the index into the programmed table */
3754                 tx->initial_rate_index = 0;
3755                 tx->tx_flags |= htole32(IWM_TX_CMD_FLG_STA_RATE);
3756         }
3757
3758         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE,
3759             "%s: frame type=%d txrate %d\n",
3760                 __func__, type, iwm_rates[ridx].rate);
3761
3762         rinfo = &iwm_rates[ridx];
3763
3764         IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: ridx=%d; rate=%d, CCK=%d\n",
3765             __func__, ridx,
3766             rinfo->rate,
3767             !! (IWM_RIDX_IS_CCK(ridx))
3768             );
3769
3770         /* XXX TODO: hard-coded TX antenna? */
3771         rate_flags = 1 << IWM_RATE_MCS_ANT_POS;
3772         if (IWM_RIDX_IS_CCK(ridx))
3773                 rate_flags |= IWM_RATE_MCS_CCK_MSK;
3774         tx->rate_n_flags = htole32(rate_flags | rinfo->plcp);
3775
3776         return rinfo;
3777 }
3778
3779 #define TB0_SIZE 16
3780 static int
3781 iwm_tx(struct iwm_softc *sc, struct mbuf *m, struct ieee80211_node *ni, int ac)
3782 {
3783         struct ieee80211com *ic = &sc->sc_ic;
3784         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
3785         struct iwm_node *in = IWM_NODE(ni);
3786         struct iwm_tx_ring *ring;
3787         struct iwm_tx_data *data;
3788         struct iwm_tfd *desc;
3789         struct iwm_device_cmd *cmd;
3790         struct iwm_tx_cmd *tx;
3791         struct ieee80211_frame *wh;
3792         struct ieee80211_key *k = NULL;
3793         struct mbuf *m1;
3794         const struct iwm_rate *rinfo;
3795         uint32_t flags;
3796         u_int hdrlen;
3797         bus_dma_segment_t *seg, segs[IWM_MAX_SCATTER];
3798         int nsegs;
3799         uint8_t tid, type;
3800         int i, totlen, error, pad;
3801
3802         wh = mtod(m, struct ieee80211_frame *);
3803         hdrlen = ieee80211_anyhdrsize(wh);
3804         type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK;
3805         tid = 0;
3806         ring = &sc->txq[ac];
3807         desc = &ring->desc[ring->cur];
3808         data = &ring->data[ring->cur];
3809
3810         /* Fill out iwm_tx_cmd to send to the firmware */
3811         cmd = &ring->cmd[ring->cur];
3812         cmd->hdr.code = IWM_TX_CMD;
3813         cmd->hdr.flags = 0;
3814         cmd->hdr.qid = ring->qid;
3815         cmd->hdr.idx = ring->cur;
3816
3817         tx = (void *)cmd->data;
3818         memset(tx, 0, sizeof(*tx));
3819
3820         rinfo = iwm_tx_fill_cmd(sc, in, m, tx);
3821
3822         /* Encrypt the frame if need be. */
3823         if (wh->i_fc[1] & IEEE80211_FC1_PROTECTED) {
3824                 /* Retrieve key for TX && do software encryption. */
3825                 k = ieee80211_crypto_encap(ni, m);
3826                 if (k == NULL) {
3827                         m_freem(m);
3828                         return (ENOBUFS);
3829                 }
3830                 /* 802.11 header may have moved. */
3831                 wh = mtod(m, struct ieee80211_frame *);
3832         }
3833
3834         if (ieee80211_radiotap_active_vap(vap)) {
3835                 struct iwm_tx_radiotap_header *tap = &sc->sc_txtap;
3836
3837                 tap->wt_flags = 0;
3838                 tap->wt_chan_freq = htole16(ni->ni_chan->ic_freq);
3839                 tap->wt_chan_flags = htole16(ni->ni_chan->ic_flags);
3840                 tap->wt_rate = rinfo->rate;
3841                 if (k != NULL)
3842                         tap->wt_flags |= IEEE80211_RADIOTAP_F_WEP;
3843                 ieee80211_radiotap_tx(vap, m);
3844         }
3845
3846         flags = 0;
3847         totlen = m->m_pkthdr.len;
3848         if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) {
3849                 flags |= IWM_TX_CMD_FLG_ACK;
3850         }
3851
3852         if (type == IEEE80211_FC0_TYPE_DATA &&
3853             totlen + IEEE80211_CRC_LEN > vap->iv_rtsthreshold &&
3854             !IEEE80211_IS_MULTICAST(wh->i_addr1)) {
3855                 flags |= IWM_TX_CMD_FLG_PROT_REQUIRE;
3856         }
3857
3858         if (IEEE80211_IS_MULTICAST(wh->i_addr1) ||
3859             type != IEEE80211_FC0_TYPE_DATA)
3860                 tx->sta_id = sc->sc_aux_sta.sta_id;
3861         else
3862                 tx->sta_id = IWM_STATION_ID;
3863
3864         if (type == IEEE80211_FC0_TYPE_MGT) {
3865                 uint8_t subtype = wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK;
3866
3867                 if (subtype == IEEE80211_FC0_SUBTYPE_ASSOC_REQ ||
3868                     subtype == IEEE80211_FC0_SUBTYPE_REASSOC_REQ) {
3869                         tx->pm_frame_timeout = htole16(IWM_PM_FRAME_ASSOC);
3870                 } else if (subtype == IEEE80211_FC0_SUBTYPE_ACTION) {
3871                         tx->pm_frame_timeout = htole16(IWM_PM_FRAME_NONE);
3872                 } else {
3873                         tx->pm_frame_timeout = htole16(IWM_PM_FRAME_MGMT);
3874                 }
3875         } else {
3876                 tx->pm_frame_timeout = htole16(IWM_PM_FRAME_NONE);
3877         }
3878
3879         if (hdrlen & 3) {
3880                 /* First segment length must be a multiple of 4. */
3881                 flags |= IWM_TX_CMD_FLG_MH_PAD;
3882                 pad = 4 - (hdrlen & 3);
3883         } else
3884                 pad = 0;
3885
3886         tx->next_frame_len = 0;
3887
3888         tx->len = htole16(totlen);
3889         tx->tid_tspec = tid;
3890         tx->life_time = htole32(IWM_TX_CMD_LIFE_TIME_INFINITE);
3891
3892         /* Set physical address of "scratch area". */
3893         tx->dram_lsb_ptr = htole32(data->scratch_paddr);
3894         tx->dram_msb_ptr = iwm_get_dma_hi_addr(data->scratch_paddr);
3895
3896         /* Copy 802.11 header in TX command. */
3897         memcpy((uint8_t *)tx + sizeof(*tx), wh, hdrlen);
3898
3899         flags |= IWM_TX_CMD_FLG_BT_DIS | IWM_TX_CMD_FLG_SEQ_CTL;
3900
3901         tx->sec_ctl = 0;
3902         tx->tx_flags |= htole32(flags);
3903
3904         /* Trim 802.11 header. */
3905         m_adj(m, hdrlen);
3906         error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m,
3907             segs, &nsegs, BUS_DMA_NOWAIT);
3908         if (error != 0) {
3909                 if (error != EFBIG) {
3910                         device_printf(sc->sc_dev, "can't map mbuf (error %d)\n",
3911                             error);
3912                         m_freem(m);
3913                         return error;
3914                 }
3915                 /* Too many DMA segments, linearize mbuf. */
3916                 m1 = m_collapse(m, M_NOWAIT, IWM_MAX_SCATTER - 2);
3917                 if (m1 == NULL) {
3918                         device_printf(sc->sc_dev,
3919                             "%s: could not defrag mbuf\n", __func__);
3920                         m_freem(m);
3921                         return (ENOBUFS);
3922                 }
3923                 m = m1;
3924
3925                 error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m,
3926                     segs, &nsegs, BUS_DMA_NOWAIT);
3927                 if (error != 0) {
3928                         device_printf(sc->sc_dev, "can't map mbuf (error %d)\n",
3929                             error);
3930                         m_freem(m);
3931                         return error;
3932                 }
3933         }
3934         data->m = m;
3935         data->in = in;
3936         data->done = 0;
3937
3938         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3939             "sending txd %p, in %p\n", data, data->in);
3940         KASSERT(data->in != NULL, ("node is NULL"));
3941
3942         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3943             "sending data: qid=%d idx=%d len=%d nsegs=%d txflags=0x%08x rate_n_flags=0x%08x rateidx=%u\n",
3944             ring->qid, ring->cur, totlen, nsegs,
3945             le32toh(tx->tx_flags),
3946             le32toh(tx->rate_n_flags),
3947             tx->initial_rate_index
3948             );
3949
3950         /* Fill TX descriptor. */
3951         memset(desc, 0, sizeof(*desc));
3952         desc->num_tbs = 2 + nsegs;
3953
3954         desc->tbs[0].lo = htole32(data->cmd_paddr);
3955         desc->tbs[0].hi_n_len = htole16(iwm_get_dma_hi_addr(data->cmd_paddr) |
3956             (TB0_SIZE << 4));
3957         desc->tbs[1].lo = htole32(data->cmd_paddr + TB0_SIZE);
3958         desc->tbs[1].hi_n_len = htole16(iwm_get_dma_hi_addr(data->cmd_paddr) |
3959             ((sizeof(struct iwm_cmd_header) + sizeof(*tx) +
3960             hdrlen + pad - TB0_SIZE) << 4));
3961
3962         /* Other DMA segments are for data payload. */
3963         for (i = 0; i < nsegs; i++) {
3964                 seg = &segs[i];
3965                 desc->tbs[i + 2].lo = htole32(seg->ds_addr);
3966                 desc->tbs[i + 2].hi_n_len =
3967                     htole16(iwm_get_dma_hi_addr(seg->ds_addr)) |
3968                     (seg->ds_len << 4);
3969         }
3970
3971         bus_dmamap_sync(ring->data_dmat, data->map,
3972             BUS_DMASYNC_PREWRITE);
3973         bus_dmamap_sync(ring->cmd_dma.tag, ring->cmd_dma.map,
3974             BUS_DMASYNC_PREWRITE);
3975         bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map,
3976             BUS_DMASYNC_PREWRITE);
3977
3978 #if 0
3979         iwm_update_sched(sc, ring->qid, ring->cur, tx->sta_id, le16toh(tx->len));
3980 #endif
3981
3982         /* Kick TX ring. */
3983         ring->cur = (ring->cur + 1) % IWM_TX_RING_COUNT;
3984         IWM_WRITE(sc, IWM_HBUS_TARG_WRPTR, ring->qid << 8 | ring->cur);
3985
3986         /* Mark TX ring as full if we reach a certain threshold. */
3987         if (++ring->queued > IWM_TX_RING_HIMARK) {
3988                 sc->qfullmsk |= 1 << ring->qid;
3989         }
3990
3991         return 0;
3992 }
3993
3994 static int
3995 iwm_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
3996     const struct ieee80211_bpf_params *params)
3997 {
3998         struct ieee80211com *ic = ni->ni_ic;
3999         struct iwm_softc *sc = ic->ic_softc;
4000         int error = 0;
4001
4002         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
4003             "->%s begin\n", __func__);
4004
4005         if ((sc->sc_flags & IWM_FLAG_HW_INITED) == 0) {
4006                 m_freem(m);
4007                 IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
4008                     "<-%s not RUNNING\n", __func__);
4009                 return (ENETDOWN);
4010         }
4011
4012         IWM_LOCK(sc);
4013         /* XXX fix this */
4014         if (params == NULL) {
4015                 error = iwm_tx(sc, m, ni, 0);
4016         } else {
4017                 error = iwm_tx(sc, m, ni, 0);
4018         }
4019         if (sc->sc_tx_timer == 0)
4020                 callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog, sc);
4021         sc->sc_tx_timer = 5;
4022         IWM_UNLOCK(sc);
4023
4024         return (error);
4025 }
4026
4027 /*
4028  * mvm/tx.c
4029  */
4030
4031 /*
4032  * Note that there are transports that buffer frames before they reach
4033  * the firmware. This means that after flush_tx_path is called, the
4034  * queue might not be empty. The race-free way to handle this is to:
4035  * 1) set the station as draining
4036  * 2) flush the Tx path
4037  * 3) wait for the transport queues to be empty
4038  */
4039 int
4040 iwm_mvm_flush_tx_path(struct iwm_softc *sc, uint32_t tfd_msk, uint32_t flags)
4041 {
4042         int ret;
4043         struct iwm_tx_path_flush_cmd flush_cmd = {
4044                 .queues_ctl = htole32(tfd_msk),
4045                 .flush_ctl = htole16(IWM_DUMP_TX_FIFO_FLUSH),
4046         };
4047
4048         ret = iwm_mvm_send_cmd_pdu(sc, IWM_TXPATH_FLUSH, flags,
4049             sizeof(flush_cmd), &flush_cmd);
4050         if (ret)
4051                 device_printf(sc->sc_dev,
4052                     "Flushing tx queue failed: %d\n", ret);
4053         return ret;
4054 }
4055
4056 /*
4057  * BEGIN mvm/quota.c
4058  */
4059
4060 static int
4061 iwm_mvm_update_quotas(struct iwm_softc *sc, struct iwm_vap *ivp)
4062 {
4063         struct iwm_time_quota_cmd cmd;
4064         int i, idx, ret, num_active_macs, quota, quota_rem;
4065         int colors[IWM_MAX_BINDINGS] = { -1, -1, -1, -1, };
4066         int n_ifs[IWM_MAX_BINDINGS] = {0, };
4067         uint16_t id;
4068
4069         memset(&cmd, 0, sizeof(cmd));
4070
4071         /* currently, PHY ID == binding ID */
4072         if (ivp) {
4073                 id = ivp->phy_ctxt->id;
4074                 KASSERT(id < IWM_MAX_BINDINGS, ("invalid id"));
4075                 colors[id] = ivp->phy_ctxt->color;
4076
4077                 if (1)
4078                         n_ifs[id] = 1;
4079         }
4080
4081         /*
4082          * The FW's scheduling session consists of
4083          * IWM_MVM_MAX_QUOTA fragments. Divide these fragments
4084          * equally between all the bindings that require quota
4085          */
4086         num_active_macs = 0;
4087         for (i = 0; i < IWM_MAX_BINDINGS; i++) {
4088                 cmd.quotas[i].id_and_color = htole32(IWM_FW_CTXT_INVALID);
4089                 num_active_macs += n_ifs[i];
4090         }
4091
4092         quota = 0;
4093         quota_rem = 0;
4094         if (num_active_macs) {
4095                 quota = IWM_MVM_MAX_QUOTA / num_active_macs;
4096                 quota_rem = IWM_MVM_MAX_QUOTA % num_active_macs;
4097         }
4098
4099         for (idx = 0, i = 0; i < IWM_MAX_BINDINGS; i++) {
4100                 if (colors[i] < 0)
4101                         continue;
4102
4103                 cmd.quotas[idx].id_and_color =
4104                         htole32(IWM_FW_CMD_ID_AND_COLOR(i, colors[i]));
4105
4106                 if (n_ifs[i] <= 0) {
4107                         cmd.quotas[idx].quota = htole32(0);
4108                         cmd.quotas[idx].max_duration = htole32(0);
4109                 } else {
4110                         cmd.quotas[idx].quota = htole32(quota * n_ifs[i]);
4111                         cmd.quotas[idx].max_duration = htole32(0);
4112                 }
4113                 idx++;
4114         }
4115
4116         /* Give the remainder of the session to the first binding */
4117         cmd.quotas[0].quota = htole32(le32toh(cmd.quotas[0].quota) + quota_rem);
4118
4119         ret = iwm_mvm_send_cmd_pdu(sc, IWM_TIME_QUOTA_CMD, IWM_CMD_SYNC,
4120             sizeof(cmd), &cmd);
4121         if (ret)
4122                 device_printf(sc->sc_dev,
4123                     "%s: Failed to send quota: %d\n", __func__, ret);
4124         return ret;
4125 }
4126
4127 /*
4128  * END mvm/quota.c
4129  */
4130
4131 /*
4132  * ieee80211 routines
4133  */
4134
4135 /*
4136  * Change to AUTH state in 80211 state machine.  Roughly matches what
4137  * Linux does in bss_info_changed().
4138  */
4139 static int
4140 iwm_auth(struct ieee80211vap *vap, struct iwm_softc *sc)
4141 {
4142         struct ieee80211_node *ni;
4143         struct iwm_node *in;
4144         struct iwm_vap *iv = IWM_VAP(vap);
4145         uint32_t duration;
4146         int error;
4147
4148         /*
4149          * XXX i have a feeling that the vap node is being
4150          * freed from underneath us. Grr.
4151          */
4152         ni = ieee80211_ref_node(vap->iv_bss);
4153         in = IWM_NODE(ni);
4154         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_STATE,
4155             "%s: called; vap=%p, bss ni=%p\n",
4156             __func__,
4157             vap,
4158             ni);
4159         IWM_DPRINTF(sc, IWM_DEBUG_STATE, "%s: Current node bssid: %s\n",
4160             __func__, ether_sprintf(ni->ni_bssid));
4161
4162         in->in_assoc = 0;
4163         iv->iv_auth = 1;
4164
4165         /*
4166          * Firmware bug - it'll crash if the beacon interval is less
4167          * than 16. We can't avoid connecting at all, so refuse the
4168          * station state change, this will cause net80211 to abandon
4169          * attempts to connect to this AP, and eventually wpa_s will
4170          * blacklist the AP...
4171          */
4172         if (ni->ni_intval < 16) {
4173                 device_printf(sc->sc_dev,
4174                     "AP %s beacon interval is %d, refusing due to firmware bug!\n",
4175                     ether_sprintf(ni->ni_bssid), ni->ni_intval);
4176                 error = EINVAL;
4177                 goto out;
4178         }
4179
4180         error = iwm_allow_mcast(vap, sc);
4181         if (error) {
4182                 device_printf(sc->sc_dev,
4183                     "%s: failed to set multicast\n", __func__);
4184                 goto out;
4185         }
4186
4187         /*
4188          * This is where it deviates from what Linux does.
4189          *
4190          * Linux iwlwifi doesn't reset the nic each time, nor does it
4191          * call ctxt_add() here.  Instead, it adds it during vap creation,
4192          * and always does a mac_ctx_changed().
4193          *
4194          * The openbsd port doesn't attempt to do that - it reset things
4195          * at odd states and does the add here.
4196          *
4197          * So, until the state handling is fixed (ie, we never reset
4198          * the NIC except for a firmware failure, which should drag
4199          * the NIC back to IDLE, re-setup and re-add all the mac/phy
4200          * contexts that are required), let's do a dirty hack here.
4201          */
4202         if (iv->is_uploaded) {
4203                 if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) {
4204                         device_printf(sc->sc_dev,
4205                             "%s: failed to update MAC\n", __func__);
4206                         goto out;
4207                 }
4208         } else {
4209                 if ((error = iwm_mvm_mac_ctxt_add(sc, vap)) != 0) {
4210                         device_printf(sc->sc_dev,
4211                             "%s: failed to add MAC\n", __func__);
4212                         goto out;
4213                 }
4214         }
4215         sc->sc_firmware_state = 1;
4216
4217         if ((error = iwm_mvm_phy_ctxt_changed(sc, &sc->sc_phyctxt[0],
4218             in->in_ni.ni_chan, 1, 1)) != 0) {
4219                 device_printf(sc->sc_dev,
4220                     "%s: failed update phy ctxt\n", __func__);
4221                 goto out;
4222         }
4223         iv->phy_ctxt = &sc->sc_phyctxt[0];
4224
4225         if ((error = iwm_mvm_binding_add_vif(sc, iv)) != 0) {
4226                 device_printf(sc->sc_dev,
4227                     "%s: binding update cmd\n", __func__);
4228                 goto out;
4229         }
4230         sc->sc_firmware_state = 2;
4231         /*
4232          * Authentication becomes unreliable when powersaving is left enabled
4233          * here. Powersaving will be activated again when association has
4234          * finished or is aborted.
4235          */
4236         iv->ps_disabled = TRUE;
4237         error = iwm_mvm_power_update_mac(sc);
4238         iv->ps_disabled = FALSE;
4239         if (error != 0) {
4240                 device_printf(sc->sc_dev,
4241                     "%s: failed to update power management\n",
4242                     __func__);
4243                 goto out;
4244         }
4245         if ((error = iwm_mvm_add_sta(sc, in)) != 0) {
4246                 device_printf(sc->sc_dev,
4247                     "%s: failed to add sta\n", __func__);
4248                 goto out;
4249         }
4250         sc->sc_firmware_state = 3;
4251
4252         /*
4253          * Prevent the FW from wandering off channel during association
4254          * by "protecting" the session with a time event.
4255          */
4256         /* XXX duration is in units of TU, not MS */
4257         duration = IWM_MVM_TE_SESSION_PROTECTION_MAX_TIME_MS;
4258         iwm_mvm_protect_session(sc, iv, duration, 500 /* XXX magic number */, TRUE);
4259
4260         error = 0;
4261 out:
4262         if (error != 0)
4263                 iv->iv_auth = 0;
4264         ieee80211_free_node(ni);
4265         return (error);
4266 }
4267
4268 static struct ieee80211_node *
4269 iwm_node_alloc(struct ieee80211vap *vap, const uint8_t mac[IEEE80211_ADDR_LEN])
4270 {
4271         return malloc(sizeof (struct iwm_node), M_80211_NODE,
4272             M_NOWAIT | M_ZERO);
4273 }
4274
4275 static uint8_t
4276 iwm_rate_from_ucode_rate(uint32_t rate_n_flags)
4277 {
4278         uint8_t plcp = rate_n_flags & 0xff;
4279         int i;
4280
4281         for (i = 0; i <= IWM_RIDX_MAX; i++) {
4282                 if (iwm_rates[i].plcp == plcp)
4283                         return iwm_rates[i].rate;
4284         }
4285         return 0;
4286 }
4287
4288 uint8_t
4289 iwm_ridx2rate(struct ieee80211_rateset *rs, int ridx)
4290 {
4291         int i;
4292         uint8_t rval;
4293
4294         for (i = 0; i < rs->rs_nrates; i++) {
4295                 rval = (rs->rs_rates[i] & IEEE80211_RATE_VAL);
4296                 if (rval == iwm_rates[ridx].rate)
4297                         return rs->rs_rates[i];
4298         }
4299
4300         return 0;
4301 }
4302
4303 static int
4304 iwm_rate2ridx(struct iwm_softc *sc, uint8_t rate)
4305 {
4306         int i;
4307
4308         for (i = 0; i <= IWM_RIDX_MAX; i++) {
4309                 if (iwm_rates[i].rate == rate)
4310                         return i;
4311         }
4312
4313         device_printf(sc->sc_dev,
4314             "%s: WARNING: device rate for %u not found!\n",
4315             __func__, rate);
4316
4317         return -1;
4318 }
4319
4320
4321 static void
4322 iwm_setrates(struct iwm_softc *sc, struct iwm_node *in, int rix)
4323 {
4324         struct ieee80211_node *ni = &in->in_ni;
4325         struct iwm_lq_cmd *lq = &in->in_lq;
4326         struct ieee80211_rateset *rs = &ni->ni_rates;
4327         int nrates = rs->rs_nrates;
4328         int i, ridx, tab = 0;
4329 //      int txant = 0;
4330
4331         KASSERT(rix >= 0 && rix < nrates, ("invalid rix"));
4332
4333         if (nrates > nitems(lq->rs_table)) {
4334                 device_printf(sc->sc_dev,
4335                     "%s: node supports %d rates, driver handles "
4336                     "only %zu\n", __func__, nrates, nitems(lq->rs_table));
4337                 return;
4338         }
4339         if (nrates == 0) {
4340                 device_printf(sc->sc_dev,
4341                     "%s: node supports 0 rates, odd!\n", __func__);
4342                 return;
4343         }
4344         nrates = imin(rix + 1, nrates);
4345
4346         IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
4347             "%s: nrates=%d\n", __func__, nrates);
4348
4349         /* then construct a lq_cmd based on those */
4350         memset(lq, 0, sizeof(*lq));
4351         lq->sta_id = IWM_STATION_ID;
4352
4353         /* For HT, always enable RTS/CTS to avoid excessive retries. */
4354         if (ni->ni_flags & IEEE80211_NODE_HT)
4355                 lq->flags |= IWM_LQ_FLAG_USE_RTS_MSK;
4356
4357         /*
4358          * are these used? (we don't do SISO or MIMO)
4359          * need to set them to non-zero, though, or we get an error.
4360          */
4361         lq->single_stream_ant_msk = 1;
4362         lq->dual_stream_ant_msk = 1;
4363
4364         /*
4365          * Build the actual rate selection table.
4366          * The lowest bits are the rates.  Additionally,
4367          * CCK needs bit 9 to be set.  The rest of the bits
4368          * we add to the table select the tx antenna
4369          * Note that we add the rates in the highest rate first
4370          * (opposite of ni_rates).
4371          */
4372         for (i = 0; i < nrates; i++) {
4373                 int rate = rs->rs_rates[rix - i] & IEEE80211_RATE_VAL;
4374                 int nextant;
4375
4376                 /* Map 802.11 rate to HW rate index. */
4377                 ridx = iwm_rate2ridx(sc, rate);
4378                 if (ridx == -1)
4379                         continue;
4380
4381 #if 0
4382                 if (txant == 0)
4383                         txant = iwm_mvm_get_valid_tx_ant(sc);
4384                 nextant = 1<<(ffs(txant)-1);
4385                 txant &= ~nextant;
4386 #else
4387                 nextant = iwm_mvm_get_valid_tx_ant(sc);
4388 #endif
4389                 tab = iwm_rates[ridx].plcp;
4390                 tab |= nextant << IWM_RATE_MCS_ANT_POS;
4391                 if (IWM_RIDX_IS_CCK(ridx))
4392                         tab |= IWM_RATE_MCS_CCK_MSK;
4393                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
4394                     "station rate i=%d, rate=%d, hw=%x\n",
4395                     i, iwm_rates[ridx].rate, tab);
4396                 lq->rs_table[i] = htole32(tab);
4397         }
4398         /* then fill the rest with the lowest possible rate */
4399         for (i = nrates; i < nitems(lq->rs_table); i++) {
4400                 KASSERT(tab != 0, ("invalid tab"));
4401                 lq->rs_table[i] = htole32(tab);
4402         }
4403 }
4404
4405 static int
4406 iwm_media_change(struct ifnet *ifp)
4407 {
4408         struct ieee80211vap *vap = ifp->if_softc;
4409         struct ieee80211com *ic = vap->iv_ic;
4410         struct iwm_softc *sc = ic->ic_softc;
4411         int error;
4412
4413         error = ieee80211_media_change(ifp);
4414         if (error != ENETRESET)
4415                 return error;
4416
4417         IWM_LOCK(sc);
4418         if (ic->ic_nrunning > 0) {
4419                 iwm_stop(sc);
4420                 iwm_init(sc);
4421         }
4422         IWM_UNLOCK(sc);
4423         return error;
4424 }
4425
4426 static void
4427 iwm_bring_down_firmware(struct iwm_softc *sc, struct ieee80211vap *vap)
4428 {
4429         struct iwm_vap *ivp = IWM_VAP(vap);
4430         int error;
4431
4432         /* Avoid Tx watchdog triggering, when transfers get dropped here. */
4433         sc->sc_tx_timer = 0;
4434
4435         ivp->iv_auth = 0;
4436         if (sc->sc_firmware_state == 3) {
4437                 iwm_xmit_queue_drain(sc);
4438 //              iwm_mvm_flush_tx_path(sc, 0xf, IWM_CMD_SYNC);
4439                 error = iwm_mvm_rm_sta(sc, vap, TRUE);
4440                 if (error) {
4441                         device_printf(sc->sc_dev,
4442                             "%s: Failed to remove station: %d\n",
4443                             __func__, error);
4444                 }
4445         }
4446         if (sc->sc_firmware_state == 3) {
4447                 error = iwm_mvm_mac_ctxt_changed(sc, vap);
4448                 if (error) {
4449                         device_printf(sc->sc_dev,
4450                             "%s: Failed to change mac context: %d\n",
4451                             __func__, error);
4452                 }
4453         }
4454         if (sc->sc_firmware_state == 3) {
4455                 error = iwm_mvm_sf_update(sc, vap, FALSE);
4456                 if (error) {
4457                         device_printf(sc->sc_dev,
4458                             "%s: Failed to update smart FIFO: %d\n",
4459                             __func__, error);
4460                 }
4461         }
4462         if (sc->sc_firmware_state == 3) {
4463                 error = iwm_mvm_rm_sta_id(sc, vap);
4464                 if (error) {
4465                         device_printf(sc->sc_dev,
4466                             "%s: Failed to remove station id: %d\n",
4467                             __func__, error);
4468                 }
4469         }
4470         if (sc->sc_firmware_state == 3) {
4471                 error = iwm_mvm_update_quotas(sc, NULL);
4472                 if (error) {
4473                         device_printf(sc->sc_dev,
4474                             "%s: Failed to update PHY quota: %d\n",
4475                             __func__, error);
4476                 }
4477         }
4478         if (sc->sc_firmware_state == 3) {
4479                 /* XXX Might need to specify bssid correctly. */
4480                 error = iwm_mvm_mac_ctxt_changed(sc, vap);
4481                 if (error) {
4482                         device_printf(sc->sc_dev,
4483                             "%s: Failed to change mac context: %d\n",
4484                             __func__, error);
4485                 }
4486         }
4487         if (sc->sc_firmware_state == 3) {
4488                 sc->sc_firmware_state = 2;
4489         }
4490         if (sc->sc_firmware_state > 1) {
4491                 error = iwm_mvm_binding_remove_vif(sc, ivp);
4492                 if (error) {
4493                         device_printf(sc->sc_dev,
4494                             "%s: Failed to remove channel ctx: %d\n",
4495                             __func__, error);
4496                 }
4497         }
4498         if (sc->sc_firmware_state > 1) {
4499                 sc->sc_firmware_state = 1;
4500         }
4501         ivp->phy_ctxt = NULL;
4502         if (sc->sc_firmware_state > 0) {
4503                 error = iwm_mvm_mac_ctxt_changed(sc, vap);
4504                 if (error) {
4505                         device_printf(sc->sc_dev,
4506                             "%s: Failed to change mac context: %d\n",
4507                             __func__, error);
4508                 }
4509         }
4510         if (sc->sc_firmware_state > 0) {
4511                 error = iwm_mvm_power_update_mac(sc);
4512                 if (error != 0) {
4513                         device_printf(sc->sc_dev,
4514                             "%s: failed to update power management\n",
4515                             __func__);
4516                 }
4517         }
4518         sc->sc_firmware_state = 0;
4519 }
4520
4521 static int
4522 iwm_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
4523 {
4524         struct iwm_vap *ivp = IWM_VAP(vap);
4525         struct ieee80211com *ic = vap->iv_ic;
4526         struct iwm_softc *sc = ic->ic_softc;
4527         struct iwm_node *in;
4528         int error;
4529
4530         IWM_DPRINTF(sc, IWM_DEBUG_STATE,
4531             "switching state %s -> %s arg=0x%x\n",
4532             ieee80211_state_name[vap->iv_state],
4533             ieee80211_state_name[nstate],
4534             arg);
4535
4536         IEEE80211_UNLOCK(ic);
4537         IWM_LOCK(sc);
4538
4539         if ((sc->sc_flags & IWM_FLAG_SCAN_RUNNING) &&
4540             (nstate == IEEE80211_S_AUTH ||
4541              nstate == IEEE80211_S_ASSOC ||
4542              nstate == IEEE80211_S_RUN)) {
4543                 /* Stop blinking for a scan, when authenticating. */
4544                 iwm_led_blink_stop(sc);
4545         }
4546
4547         if (vap->iv_state == IEEE80211_S_RUN && nstate != IEEE80211_S_RUN) {
4548                 iwm_mvm_led_disable(sc);
4549                 /* disable beacon filtering if we're hopping out of RUN */
4550                 iwm_mvm_disable_beacon_filter(sc);
4551                 if (((in = IWM_NODE(vap->iv_bss)) != NULL))
4552                         in->in_assoc = 0;
4553         }
4554
4555         if ((vap->iv_state == IEEE80211_S_AUTH ||
4556              vap->iv_state == IEEE80211_S_ASSOC ||
4557              vap->iv_state == IEEE80211_S_RUN) &&
4558             (nstate == IEEE80211_S_INIT ||
4559              nstate == IEEE80211_S_SCAN ||
4560              nstate == IEEE80211_S_AUTH)) {
4561                 iwm_mvm_stop_session_protection(sc, ivp);
4562         }
4563
4564         if ((vap->iv_state == IEEE80211_S_RUN ||
4565              vap->iv_state == IEEE80211_S_ASSOC) &&
4566             nstate == IEEE80211_S_INIT) {
4567                 /*
4568                  * In this case, iv_newstate() wants to send an 80211 frame on
4569                  * the network that we are leaving. So we need to call it,
4570                  * before tearing down all the firmware state.
4571                  */
4572                 IWM_UNLOCK(sc);
4573                 IEEE80211_LOCK(ic);
4574                 ivp->iv_newstate(vap, nstate, arg);
4575                 IEEE80211_UNLOCK(ic);
4576                 IWM_LOCK(sc);
4577                 iwm_bring_down_firmware(sc, vap);
4578                 IWM_UNLOCK(sc);
4579                 IEEE80211_LOCK(ic);
4580                 return 0;
4581         }
4582
4583         switch (nstate) {
4584         case IEEE80211_S_INIT:
4585         case IEEE80211_S_SCAN:
4586                 break;
4587
4588         case IEEE80211_S_AUTH:
4589                 iwm_bring_down_firmware(sc, vap);
4590                 if ((error = iwm_auth(vap, sc)) != 0) {
4591                         device_printf(sc->sc_dev,
4592                             "%s: could not move to auth state: %d\n",
4593                             __func__, error);
4594                         iwm_bring_down_firmware(sc, vap);
4595                         IWM_UNLOCK(sc);
4596                         IEEE80211_LOCK(ic);
4597                         return 1;
4598                 }
4599                 break;
4600
4601         case IEEE80211_S_ASSOC:
4602                 /*
4603                  * EBS may be disabled due to previous failures reported by FW.
4604                  * Reset EBS status here assuming environment has been changed.
4605                  */
4606                 sc->last_ebs_successful = TRUE;
4607                 break;
4608
4609         case IEEE80211_S_RUN:
4610                 in = IWM_NODE(vap->iv_bss);
4611                 /* Update the association state, now we have it all */
4612                 /* (eg associd comes in at this point */
4613                 error = iwm_mvm_update_sta(sc, in);
4614                 if (error != 0) {
4615                         device_printf(sc->sc_dev,
4616                             "%s: failed to update STA\n", __func__);
4617                         IWM_UNLOCK(sc);
4618                         IEEE80211_LOCK(ic);
4619                         return error;
4620                 }
4621                 in->in_assoc = 1;
4622                 error = iwm_mvm_mac_ctxt_changed(sc, vap);
4623                 if (error != 0) {
4624                         device_printf(sc->sc_dev,
4625                             "%s: failed to update MAC: %d\n", __func__, error);
4626                 }
4627
4628                 iwm_mvm_sf_update(sc, vap, FALSE);
4629                 iwm_mvm_enable_beacon_filter(sc, ivp);
4630                 iwm_mvm_power_update_mac(sc);
4631                 iwm_mvm_update_quotas(sc, ivp);
4632                 int rix = ieee80211_ratectl_rate(&in->in_ni, NULL, 0);
4633                 iwm_setrates(sc, in, rix);
4634
4635                 if ((error = iwm_mvm_send_lq_cmd(sc, &in->in_lq, TRUE)) != 0) {
4636                         device_printf(sc->sc_dev,
4637                             "%s: IWM_LQ_CMD failed: %d\n", __func__, error);
4638                 }
4639
4640                 iwm_mvm_led_enable(sc);
4641                 break;
4642
4643         default:
4644                 break;
4645         }
4646         IWM_UNLOCK(sc);
4647         IEEE80211_LOCK(ic);
4648
4649         return (ivp->iv_newstate(vap, nstate, arg));
4650 }
4651
4652 void
4653 iwm_endscan_cb(void *arg, int pending)
4654 {
4655         struct iwm_softc *sc = arg;
4656         struct ieee80211com *ic = &sc->sc_ic;
4657
4658         IWM_DPRINTF(sc, IWM_DEBUG_SCAN | IWM_DEBUG_TRACE,
4659             "%s: scan ended\n",
4660             __func__);
4661
4662         ieee80211_scan_done(TAILQ_FIRST(&ic->ic_vaps));
4663 }
4664
4665 static int
4666 iwm_send_bt_init_conf(struct iwm_softc *sc)
4667 {
4668         struct iwm_bt_coex_cmd bt_cmd;
4669
4670         bt_cmd.mode = htole32(IWM_BT_COEX_WIFI);
4671         bt_cmd.enabled_modules = htole32(IWM_BT_COEX_HIGH_BAND_RET);
4672
4673         return iwm_mvm_send_cmd_pdu(sc, IWM_BT_CONFIG, 0, sizeof(bt_cmd),
4674             &bt_cmd);
4675 }
4676
4677 static boolean_t
4678 iwm_mvm_is_lar_supported(struct iwm_softc *sc)
4679 {
4680         boolean_t nvm_lar = sc->nvm_data->lar_enabled;
4681         boolean_t tlv_lar = iwm_fw_has_capa(sc, IWM_UCODE_TLV_CAPA_LAR_SUPPORT);
4682
4683         if (iwm_lar_disable)
4684                 return FALSE;
4685
4686         /*
4687          * Enable LAR only if it is supported by the FW (TLV) &&
4688          * enabled in the NVM
4689          */
4690         if (sc->cfg->device_family >= IWM_DEVICE_FAMILY_8000)
4691                 return nvm_lar && tlv_lar;
4692         else
4693                 return tlv_lar;
4694 }
4695
4696 static boolean_t
4697 iwm_mvm_is_wifi_mcc_supported(struct iwm_softc *sc)
4698 {
4699         return iwm_fw_has_api(sc, IWM_UCODE_TLV_API_WIFI_MCC_UPDATE) ||
4700             iwm_fw_has_capa(sc, IWM_UCODE_TLV_CAPA_LAR_MULTI_MCC);
4701 }
4702
4703 static int
4704 iwm_send_update_mcc_cmd(struct iwm_softc *sc, const char *alpha2)
4705 {
4706         struct iwm_mcc_update_cmd mcc_cmd;
4707         struct iwm_host_cmd hcmd = {
4708                 .id = IWM_MCC_UPDATE_CMD,
4709                 .flags = (IWM_CMD_SYNC | IWM_CMD_WANT_SKB),
4710                 .data = { &mcc_cmd },
4711         };
4712         int ret;
4713 #ifdef IWM_DEBUG
4714         struct iwm_rx_packet *pkt;
4715         struct iwm_mcc_update_resp_v1 *mcc_resp_v1 = NULL;
4716         struct iwm_mcc_update_resp *mcc_resp;
4717         int n_channels;
4718         uint16_t mcc;
4719 #endif
4720         int resp_v2 = iwm_fw_has_capa(sc, IWM_UCODE_TLV_CAPA_LAR_SUPPORT_V2);
4721
4722         if (!iwm_mvm_is_lar_supported(sc)) {
4723                 IWM_DPRINTF(sc, IWM_DEBUG_LAR, "%s: no LAR support\n",
4724                     __func__);
4725                 return 0;
4726         }
4727
4728         memset(&mcc_cmd, 0, sizeof(mcc_cmd));
4729         mcc_cmd.mcc = htole16(alpha2[0] << 8 | alpha2[1]);
4730         if (iwm_mvm_is_wifi_mcc_supported(sc))
4731                 mcc_cmd.source_id = IWM_MCC_SOURCE_GET_CURRENT;
4732         else
4733                 mcc_cmd.source_id = IWM_MCC_SOURCE_OLD_FW;
4734
4735         if (resp_v2)
4736                 hcmd.len[0] = sizeof(struct iwm_mcc_update_cmd);
4737         else
4738                 hcmd.len[0] = sizeof(struct iwm_mcc_update_cmd_v1);
4739
4740         IWM_DPRINTF(sc, IWM_DEBUG_LAR,
4741             "send MCC update to FW with '%c%c' src = %d\n",
4742             alpha2[0], alpha2[1], mcc_cmd.source_id);
4743
4744         ret = iwm_send_cmd(sc, &hcmd);
4745         if (ret)
4746                 return ret;
4747
4748 #ifdef IWM_DEBUG
4749         pkt = hcmd.resp_pkt;
4750
4751         /* Extract MCC response */
4752         if (resp_v2) {
4753                 mcc_resp = (void *)pkt->data;
4754                 mcc = mcc_resp->mcc;
4755                 n_channels =  le32toh(mcc_resp->n_channels);
4756         } else {
4757                 mcc_resp_v1 = (void *)pkt->data;
4758                 mcc = mcc_resp_v1->mcc;
4759                 n_channels =  le32toh(mcc_resp_v1->n_channels);
4760         }
4761
4762         /* W/A for a FW/NVM issue - returns 0x00 for the world domain */
4763         if (mcc == 0)
4764                 mcc = 0x3030;  /* "00" - world */
4765
4766         IWM_DPRINTF(sc, IWM_DEBUG_LAR,
4767             "regulatory domain '%c%c' (%d channels available)\n",
4768             mcc >> 8, mcc & 0xff, n_channels);
4769 #endif
4770         iwm_free_resp(sc, &hcmd);
4771
4772         return 0;
4773 }
4774
4775 static void
4776 iwm_mvm_tt_tx_backoff(struct iwm_softc *sc, uint32_t backoff)
4777 {
4778         struct iwm_host_cmd cmd = {
4779                 .id = IWM_REPLY_THERMAL_MNG_BACKOFF,
4780                 .len = { sizeof(uint32_t), },
4781                 .data = { &backoff, },
4782         };
4783
4784         if (iwm_send_cmd(sc, &cmd) != 0) {
4785                 device_printf(sc->sc_dev,
4786                     "failed to change thermal tx backoff\n");
4787         }
4788 }
4789
4790 static int
4791 iwm_init_hw(struct iwm_softc *sc)
4792 {
4793         struct ieee80211com *ic = &sc->sc_ic;
4794         int error, i, ac;
4795
4796         sc->sf_state = IWM_SF_UNINIT;
4797
4798         if ((error = iwm_start_hw(sc)) != 0) {
4799                 printf("iwm_start_hw: failed %d\n", error);
4800                 return error;
4801         }
4802
4803         if ((error = iwm_run_init_mvm_ucode(sc, 0)) != 0) {
4804                 printf("iwm_run_init_mvm_ucode: failed %d\n", error);
4805                 return error;
4806         }
4807
4808         /*
4809          * should stop and start HW since that INIT
4810          * image just loaded
4811          */
4812         iwm_stop_device(sc);
4813         sc->sc_ps_disabled = FALSE;
4814         if ((error = iwm_start_hw(sc)) != 0) {
4815                 device_printf(sc->sc_dev, "could not initialize hardware\n");
4816                 return error;
4817         }
4818
4819         /* omstart, this time with the regular firmware */
4820         error = iwm_mvm_load_ucode_wait_alive(sc, IWM_UCODE_REGULAR);
4821         if (error) {
4822                 device_printf(sc->sc_dev, "could not load firmware\n");
4823                 goto error;
4824         }
4825
4826         error = iwm_mvm_sf_update(sc, NULL, FALSE);
4827         if (error)
4828                 device_printf(sc->sc_dev, "Failed to initialize Smart Fifo\n");
4829
4830         if ((error = iwm_send_bt_init_conf(sc)) != 0) {
4831                 device_printf(sc->sc_dev, "bt init conf failed\n");
4832                 goto error;
4833         }
4834
4835         error = iwm_send_tx_ant_cfg(sc, iwm_mvm_get_valid_tx_ant(sc));
4836         if (error != 0) {
4837                 device_printf(sc->sc_dev, "antenna config failed\n");
4838                 goto error;
4839         }
4840
4841         /* Send phy db control command and then phy db calibration */
4842         if ((error = iwm_send_phy_db_data(sc->sc_phy_db)) != 0)
4843                 goto error;
4844
4845         if ((error = iwm_send_phy_cfg_cmd(sc)) != 0) {
4846                 device_printf(sc->sc_dev, "phy_cfg_cmd failed\n");
4847                 goto error;
4848         }
4849
4850         /* Add auxiliary station for scanning */
4851         if ((error = iwm_mvm_add_aux_sta(sc)) != 0) {
4852                 device_printf(sc->sc_dev, "add_aux_sta failed\n");
4853                 goto error;
4854         }
4855
4856         for (i = 0; i < IWM_NUM_PHY_CTX; i++) {
4857                 /*
4858                  * The channel used here isn't relevant as it's
4859                  * going to be overwritten in the other flows.
4860                  * For now use the first channel we have.
4861                  */
4862                 if ((error = iwm_mvm_phy_ctxt_add(sc,
4863                     &sc->sc_phyctxt[i], &ic->ic_channels[1], 1, 1)) != 0)
4864                         goto error;
4865         }
4866
4867         /* Initialize tx backoffs to the minimum. */
4868         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
4869                 iwm_mvm_tt_tx_backoff(sc, 0);
4870
4871         if (iwm_mvm_config_ltr(sc) != 0)
4872                 device_printf(sc->sc_dev, "PCIe LTR configuration failed\n");
4873
4874         error = iwm_mvm_power_update_device(sc);
4875         if (error)
4876                 goto error;
4877
4878         if ((error = iwm_send_update_mcc_cmd(sc, "ZZ")) != 0)
4879                 goto error;
4880
4881         if (iwm_fw_has_capa(sc, IWM_UCODE_TLV_CAPA_UMAC_SCAN)) {
4882                 if ((error = iwm_mvm_config_umac_scan(sc)) != 0)
4883                         goto error;
4884         }
4885
4886         /* Enable Tx queues. */
4887         for (ac = 0; ac < WME_NUM_AC; ac++) {
4888                 error = iwm_enable_txq(sc, IWM_STATION_ID, ac,
4889                     iwm_mvm_ac_to_tx_fifo[ac]);
4890                 if (error)
4891                         goto error;
4892         }
4893
4894         if ((error = iwm_mvm_disable_beacon_filter(sc)) != 0) {
4895                 device_printf(sc->sc_dev, "failed to disable beacon filter\n");
4896                 goto error;
4897         }
4898
4899         return 0;
4900
4901  error:
4902         iwm_stop_device(sc);
4903         return error;
4904 }
4905
4906 /* Allow multicast from our BSSID. */
4907 static int
4908 iwm_allow_mcast(struct ieee80211vap *vap, struct iwm_softc *sc)
4909 {
4910         struct ieee80211_node *ni = vap->iv_bss;
4911         struct iwm_mcast_filter_cmd *cmd;
4912         size_t size;
4913         int error;
4914
4915         size = roundup(sizeof(*cmd), 4);
4916         cmd = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
4917         if (cmd == NULL)
4918                 return ENOMEM;
4919         cmd->filter_own = 1;
4920         cmd->port_id = 0;
4921         cmd->count = 0;
4922         cmd->pass_all = 1;
4923         IEEE80211_ADDR_COPY(cmd->bssid, ni->ni_bssid);
4924
4925         error = iwm_mvm_send_cmd_pdu(sc, IWM_MCAST_FILTER_CMD,
4926             IWM_CMD_SYNC, size, cmd);
4927         free(cmd, M_DEVBUF);
4928
4929         return (error);
4930 }
4931
4932 /*
4933  * ifnet interfaces
4934  */
4935
4936 static void
4937 iwm_init(struct iwm_softc *sc)
4938 {
4939         int error;
4940
4941         if (sc->sc_flags & IWM_FLAG_HW_INITED) {
4942                 return;
4943         }
4944         sc->sc_generation++;
4945         sc->sc_flags &= ~IWM_FLAG_STOPPED;
4946
4947         if ((error = iwm_init_hw(sc)) != 0) {
4948                 printf("iwm_init_hw failed %d\n", error);
4949                 iwm_stop(sc);
4950                 return;
4951         }
4952
4953         /*
4954          * Ok, firmware loaded and we are jogging
4955          */
4956         sc->sc_flags |= IWM_FLAG_HW_INITED;
4957 }
4958
4959 static int
4960 iwm_transmit(struct ieee80211com *ic, struct mbuf *m)
4961 {
4962         struct iwm_softc *sc;
4963         int error;
4964
4965         sc = ic->ic_softc;
4966
4967         IWM_LOCK(sc);
4968         if ((sc->sc_flags & IWM_FLAG_HW_INITED) == 0) {
4969                 IWM_UNLOCK(sc);
4970                 return (ENXIO);
4971         }
4972         error = mbufq_enqueue(&sc->sc_snd, m);
4973         if (error) {
4974                 IWM_UNLOCK(sc);
4975                 return (error);
4976         }
4977         iwm_start(sc);
4978         IWM_UNLOCK(sc);
4979         return (0);
4980 }
4981
4982 /*
4983  * Dequeue packets from sendq and call send.
4984  */
4985 static void
4986 iwm_start(struct iwm_softc *sc)
4987 {
4988         struct ieee80211_node *ni;
4989         struct mbuf *m;
4990         int ac = 0;
4991
4992         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TRACE, "->%s\n", __func__);
4993         while (sc->qfullmsk == 0 &&
4994                 (m = mbufq_dequeue(&sc->sc_snd)) != NULL) {
4995                 ni = (struct ieee80211_node *)m->m_pkthdr.rcvif;
4996                 if (iwm_tx(sc, m, ni, ac) != 0) {
4997                         if_inc_counter(ni->ni_vap->iv_ifp,
4998                             IFCOUNTER_OERRORS, 1);
4999                         ieee80211_free_node(ni);
5000                         continue;
5001                 }
5002                 if (sc->sc_tx_timer == 0) {
5003                         callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog,
5004                             sc);
5005                 }
5006                 sc->sc_tx_timer = 15;
5007         }
5008         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TRACE, "<-%s\n", __func__);
5009 }
5010
5011 static void
5012 iwm_stop(struct iwm_softc *sc)
5013 {
5014
5015         sc->sc_flags &= ~IWM_FLAG_HW_INITED;
5016         sc->sc_flags |= IWM_FLAG_STOPPED;
5017         sc->sc_generation++;
5018         iwm_led_blink_stop(sc);
5019         sc->sc_tx_timer = 0;
5020         iwm_stop_device(sc);
5021         sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
5022 }
5023
5024 static void
5025 iwm_watchdog(void *arg)
5026 {
5027         struct iwm_softc *sc = arg;
5028         struct ieee80211com *ic = &sc->sc_ic;
5029
5030         if (sc->sc_attached == 0)
5031                 return;
5032
5033         if (sc->sc_tx_timer > 0) {
5034                 if (--sc->sc_tx_timer == 0) {
5035                         device_printf(sc->sc_dev, "device timeout\n");
5036 #ifdef IWM_DEBUG
5037                         iwm_nic_error(sc);
5038 #endif
5039                         ieee80211_restart_all(ic);
5040                         counter_u64_add(sc->sc_ic.ic_oerrors, 1);
5041                         return;
5042                 }
5043                 callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog, sc);
5044         }
5045 }
5046
5047 static void
5048 iwm_parent(struct ieee80211com *ic)
5049 {
5050         struct iwm_softc *sc = ic->ic_softc;
5051         int startall = 0;
5052
5053         IWM_LOCK(sc);
5054         if (ic->ic_nrunning > 0) {
5055                 if (!(sc->sc_flags & IWM_FLAG_HW_INITED)) {
5056                         iwm_init(sc);
5057                         startall = 1;
5058                 }
5059         } else if (sc->sc_flags & IWM_FLAG_HW_INITED)
5060                 iwm_stop(sc);
5061         IWM_UNLOCK(sc);
5062         if (startall)
5063                 ieee80211_start_all(ic);
5064 }
5065
5066 /*
5067  * The interrupt side of things
5068  */
5069
5070 /*
5071  * error dumping routines are from iwlwifi/mvm/utils.c
5072  */
5073
5074 /*
5075  * Note: This structure is read from the device with IO accesses,
5076  * and the reading already does the endian conversion. As it is
5077  * read with uint32_t-sized accesses, any members with a different size
5078  * need to be ordered correctly though!
5079  */
5080 struct iwm_error_event_table {
5081         uint32_t valid;         /* (nonzero) valid, (0) log is empty */
5082         uint32_t error_id;              /* type of error */
5083         uint32_t trm_hw_status0;        /* TRM HW status */
5084         uint32_t trm_hw_status1;        /* TRM HW status */
5085         uint32_t blink2;                /* branch link */
5086         uint32_t ilink1;                /* interrupt link */
5087         uint32_t ilink2;                /* interrupt link */
5088         uint32_t data1;         /* error-specific data */
5089         uint32_t data2;         /* error-specific data */
5090         uint32_t data3;         /* error-specific data */
5091         uint32_t bcon_time;             /* beacon timer */
5092         uint32_t tsf_low;               /* network timestamp function timer */
5093         uint32_t tsf_hi;                /* network timestamp function timer */
5094         uint32_t gp1;           /* GP1 timer register */
5095         uint32_t gp2;           /* GP2 timer register */
5096         uint32_t fw_rev_type;   /* firmware revision type */
5097         uint32_t major;         /* uCode version major */
5098         uint32_t minor;         /* uCode version minor */
5099         uint32_t hw_ver;                /* HW Silicon version */
5100         uint32_t brd_ver;               /* HW board version */
5101         uint32_t log_pc;                /* log program counter */
5102         uint32_t frame_ptr;             /* frame pointer */
5103         uint32_t stack_ptr;             /* stack pointer */
5104         uint32_t hcmd;          /* last host command header */
5105         uint32_t isr0;          /* isr status register LMPM_NIC_ISR0:
5106                                  * rxtx_flag */
5107         uint32_t isr1;          /* isr status register LMPM_NIC_ISR1:
5108                                  * host_flag */
5109         uint32_t isr2;          /* isr status register LMPM_NIC_ISR2:
5110                                  * enc_flag */
5111         uint32_t isr3;          /* isr status register LMPM_NIC_ISR3:
5112                                  * time_flag */
5113         uint32_t isr4;          /* isr status register LMPM_NIC_ISR4:
5114                                  * wico interrupt */
5115         uint32_t last_cmd_id;   /* last HCMD id handled by the firmware */
5116         uint32_t wait_event;            /* wait event() caller address */
5117         uint32_t l2p_control;   /* L2pControlField */
5118         uint32_t l2p_duration;  /* L2pDurationField */
5119         uint32_t l2p_mhvalid;   /* L2pMhValidBits */
5120         uint32_t l2p_addr_match;        /* L2pAddrMatchStat */
5121         uint32_t lmpm_pmg_sel;  /* indicate which clocks are turned on
5122                                  * (LMPM_PMG_SEL) */
5123         uint32_t u_timestamp;   /* indicate when the date and time of the
5124                                  * compilation */
5125         uint32_t flow_handler;  /* FH read/write pointers, RX credit */
5126 } __packed /* LOG_ERROR_TABLE_API_S_VER_3 */;
5127
5128 /*
5129  * UMAC error struct - relevant starting from family 8000 chip.
5130  * Note: This structure is read from the device with IO accesses,
5131  * and the reading already does the endian conversion. As it is
5132  * read with u32-sized accesses, any members with a different size
5133  * need to be ordered correctly though!
5134  */
5135 struct iwm_umac_error_event_table {
5136         uint32_t valid;         /* (nonzero) valid, (0) log is empty */
5137         uint32_t error_id;      /* type of error */
5138         uint32_t blink1;        /* branch link */
5139         uint32_t blink2;        /* branch link */
5140         uint32_t ilink1;        /* interrupt link */
5141         uint32_t ilink2;        /* interrupt link */
5142         uint32_t data1;         /* error-specific data */
5143         uint32_t data2;         /* error-specific data */
5144         uint32_t data3;         /* error-specific data */
5145         uint32_t umac_major;
5146         uint32_t umac_minor;
5147         uint32_t frame_pointer; /* core register 27*/
5148         uint32_t stack_pointer; /* core register 28 */
5149         uint32_t cmd_header;    /* latest host cmd sent to UMAC */
5150         uint32_t nic_isr_pref;  /* ISR status register */
5151 } __packed;
5152
5153 #define ERROR_START_OFFSET  (1 * sizeof(uint32_t))
5154 #define ERROR_ELEM_SIZE     (7 * sizeof(uint32_t))
5155
5156 #ifdef IWM_DEBUG
5157 struct {
5158         const char *name;
5159         uint8_t num;
5160 } advanced_lookup[] = {
5161         { "NMI_INTERRUPT_WDG", 0x34 },
5162         { "SYSASSERT", 0x35 },
5163         { "UCODE_VERSION_MISMATCH", 0x37 },
5164         { "BAD_COMMAND", 0x38 },
5165         { "NMI_INTERRUPT_DATA_ACTION_PT", 0x3C },
5166         { "FATAL_ERROR", 0x3D },
5167         { "NMI_TRM_HW_ERR", 0x46 },
5168         { "NMI_INTERRUPT_TRM", 0x4C },
5169         { "NMI_INTERRUPT_BREAK_POINT", 0x54 },
5170         { "NMI_INTERRUPT_WDG_RXF_FULL", 0x5C },
5171         { "NMI_INTERRUPT_WDG_NO_RBD_RXF_FULL", 0x64 },
5172         { "NMI_INTERRUPT_HOST", 0x66 },
5173         { "NMI_INTERRUPT_ACTION_PT", 0x7C },
5174         { "NMI_INTERRUPT_UNKNOWN", 0x84 },
5175         { "NMI_INTERRUPT_INST_ACTION_PT", 0x86 },
5176         { "ADVANCED_SYSASSERT", 0 },
5177 };
5178
5179 static const char *
5180 iwm_desc_lookup(uint32_t num)
5181 {
5182         int i;
5183
5184         for (i = 0; i < nitems(advanced_lookup) - 1; i++)
5185                 if (advanced_lookup[i].num == num)
5186                         return advanced_lookup[i].name;
5187
5188         /* No entry matches 'num', so it is the last: ADVANCED_SYSASSERT */
5189         return advanced_lookup[i].name;
5190 }
5191
5192 static void
5193 iwm_nic_umac_error(struct iwm_softc *sc)
5194 {
5195         struct iwm_umac_error_event_table table;
5196         uint32_t base;
5197
5198         base = sc->umac_error_event_table;
5199
5200         if (base < 0x800000) {
5201                 device_printf(sc->sc_dev, "Invalid error log pointer 0x%08x\n",
5202                     base);
5203                 return;
5204         }
5205
5206         if (iwm_read_mem(sc, base, &table, sizeof(table)/sizeof(uint32_t))) {
5207                 device_printf(sc->sc_dev, "reading errlog failed\n");
5208                 return;
5209         }
5210
5211         if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) {
5212                 device_printf(sc->sc_dev, "Start UMAC Error Log Dump:\n");
5213                 device_printf(sc->sc_dev, "Status: 0x%x, count: %d\n",
5214                     sc->sc_flags, table.valid);
5215         }
5216
5217         device_printf(sc->sc_dev, "0x%08X | %s\n", table.error_id,
5218                 iwm_desc_lookup(table.error_id));
5219         device_printf(sc->sc_dev, "0x%08X | umac branchlink1\n", table.blink1);
5220         device_printf(sc->sc_dev, "0x%08X | umac branchlink2\n", table.blink2);
5221         device_printf(sc->sc_dev, "0x%08X | umac interruptlink1\n",
5222             table.ilink1);
5223         device_printf(sc->sc_dev, "0x%08X | umac interruptlink2\n",
5224             table.ilink2);
5225         device_printf(sc->sc_dev, "0x%08X | umac data1\n", table.data1);
5226         device_printf(sc->sc_dev, "0x%08X | umac data2\n", table.data2);
5227         device_printf(sc->sc_dev, "0x%08X | umac data3\n", table.data3);
5228         device_printf(sc->sc_dev, "0x%08X | umac major\n", table.umac_major);
5229         device_printf(sc->sc_dev, "0x%08X | umac minor\n", table.umac_minor);
5230         device_printf(sc->sc_dev, "0x%08X | frame pointer\n",
5231             table.frame_pointer);
5232         device_printf(sc->sc_dev, "0x%08X | stack pointer\n",
5233             table.stack_pointer);
5234         device_printf(sc->sc_dev, "0x%08X | last host cmd\n", table.cmd_header);
5235         device_printf(sc->sc_dev, "0x%08X | isr status reg\n",
5236             table.nic_isr_pref);
5237 }
5238
5239 /*
5240  * Support for dumping the error log seemed like a good idea ...
5241  * but it's mostly hex junk and the only sensible thing is the
5242  * hw/ucode revision (which we know anyway).  Since it's here,
5243  * I'll just leave it in, just in case e.g. the Intel guys want to
5244  * help us decipher some "ADVANCED_SYSASSERT" later.
5245  */
5246 static void
5247 iwm_nic_error(struct iwm_softc *sc)
5248 {
5249         struct iwm_error_event_table table;
5250         uint32_t base;
5251
5252         device_printf(sc->sc_dev, "dumping device error log\n");
5253         base = sc->error_event_table[0];
5254         if (base < 0x800000) {
5255                 device_printf(sc->sc_dev,
5256                     "Invalid error log pointer 0x%08x\n", base);
5257                 return;
5258         }
5259
5260         if (iwm_read_mem(sc, base, &table, sizeof(table)/sizeof(uint32_t))) {
5261                 device_printf(sc->sc_dev, "reading errlog failed\n");
5262                 return;
5263         }
5264
5265         if (!table.valid) {
5266                 device_printf(sc->sc_dev, "errlog not found, skipping\n");
5267                 return;
5268         }
5269
5270         if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) {
5271                 device_printf(sc->sc_dev, "Start Error Log Dump:\n");
5272                 device_printf(sc->sc_dev, "Status: 0x%x, count: %d\n",
5273                     sc->sc_flags, table.valid);
5274         }
5275
5276         device_printf(sc->sc_dev, "0x%08X | %-28s\n", table.error_id,
5277             iwm_desc_lookup(table.error_id));
5278         device_printf(sc->sc_dev, "%08X | trm_hw_status0\n",
5279             table.trm_hw_status0);
5280         device_printf(sc->sc_dev, "%08X | trm_hw_status1\n",
5281             table.trm_hw_status1);
5282         device_printf(sc->sc_dev, "%08X | branchlink2\n", table.blink2);
5283         device_printf(sc->sc_dev, "%08X | interruptlink1\n", table.ilink1);
5284         device_printf(sc->sc_dev, "%08X | interruptlink2\n", table.ilink2);
5285         device_printf(sc->sc_dev, "%08X | data1\n", table.data1);
5286         device_printf(sc->sc_dev, "%08X | data2\n", table.data2);
5287         device_printf(sc->sc_dev, "%08X | data3\n", table.data3);
5288         device_printf(sc->sc_dev, "%08X | beacon time\n", table.bcon_time);
5289         device_printf(sc->sc_dev, "%08X | tsf low\n", table.tsf_low);
5290         device_printf(sc->sc_dev, "%08X | tsf hi\n", table.tsf_hi);
5291         device_printf(sc->sc_dev, "%08X | time gp1\n", table.gp1);
5292         device_printf(sc->sc_dev, "%08X | time gp2\n", table.gp2);
5293         device_printf(sc->sc_dev, "%08X | uCode revision type\n",
5294             table.fw_rev_type);
5295         device_printf(sc->sc_dev, "%08X | uCode version major\n", table.major);
5296         device_printf(sc->sc_dev, "%08X | uCode version minor\n", table.minor);
5297         device_printf(sc->sc_dev, "%08X | hw version\n", table.hw_ver);
5298         device_printf(sc->sc_dev, "%08X | board version\n", table.brd_ver);
5299         device_printf(sc->sc_dev, "%08X | hcmd\n", table.hcmd);
5300         device_printf(sc->sc_dev, "%08X | isr0\n", table.isr0);
5301         device_printf(sc->sc_dev, "%08X | isr1\n", table.isr1);
5302         device_printf(sc->sc_dev, "%08X | isr2\n", table.isr2);
5303         device_printf(sc->sc_dev, "%08X | isr3\n", table.isr3);
5304         device_printf(sc->sc_dev, "%08X | isr4\n", table.isr4);
5305         device_printf(sc->sc_dev, "%08X | last cmd Id\n", table.last_cmd_id);
5306         device_printf(sc->sc_dev, "%08X | wait_event\n", table.wait_event);
5307         device_printf(sc->sc_dev, "%08X | l2p_control\n", table.l2p_control);
5308         device_printf(sc->sc_dev, "%08X | l2p_duration\n", table.l2p_duration);
5309         device_printf(sc->sc_dev, "%08X | l2p_mhvalid\n", table.l2p_mhvalid);
5310         device_printf(sc->sc_dev, "%08X | l2p_addr_match\n", table.l2p_addr_match);
5311         device_printf(sc->sc_dev, "%08X | lmpm_pmg_sel\n", table.lmpm_pmg_sel);
5312         device_printf(sc->sc_dev, "%08X | timestamp\n", table.u_timestamp);
5313         device_printf(sc->sc_dev, "%08X | flow_handler\n", table.flow_handler);
5314
5315         if (sc->umac_error_event_table)
5316                 iwm_nic_umac_error(sc);
5317 }
5318 #endif
5319
5320 static void
5321 iwm_handle_rxb(struct iwm_softc *sc, struct mbuf *m)
5322 {
5323         struct ieee80211com *ic = &sc->sc_ic;
5324         struct iwm_cmd_response *cresp;
5325         struct mbuf *m1;
5326         uint32_t offset = 0;
5327         uint32_t maxoff = IWM_RBUF_SIZE;
5328         uint32_t nextoff;
5329         boolean_t stolen = FALSE;
5330
5331 #define HAVEROOM(a)     \
5332     ((a) + sizeof(uint32_t) + sizeof(struct iwm_cmd_header) < maxoff)
5333
5334         while (HAVEROOM(offset)) {
5335                 struct iwm_rx_packet *pkt = mtodoff(m, struct iwm_rx_packet *,
5336                     offset);
5337                 int qid, idx, code, len;
5338
5339                 qid = pkt->hdr.qid;
5340                 idx = pkt->hdr.idx;
5341
5342                 code = IWM_WIDE_ID(pkt->hdr.flags, pkt->hdr.code);
5343
5344                 /*
5345                  * randomly get these from the firmware, no idea why.
5346                  * they at least seem harmless, so just ignore them for now
5347                  */
5348                 if ((pkt->hdr.code == 0 && (qid & ~0x80) == 0 && idx == 0) ||
5349                     pkt->len_n_flags == htole32(IWM_FH_RSCSR_FRAME_INVALID)) {
5350                         break;
5351                 }
5352
5353                 IWM_DPRINTF(sc, IWM_DEBUG_INTR,
5354                     "rx packet qid=%d idx=%d type=%x\n",
5355                     qid & ~0x80, pkt->hdr.idx, code);
5356
5357                 len = iwm_rx_packet_len(pkt);
5358                 len += sizeof(uint32_t); /* account for status word */
5359                 nextoff = offset + roundup2(len, IWM_FH_RSCSR_FRAME_ALIGN);
5360
5361                 iwm_notification_wait_notify(sc->sc_notif_wait, code, pkt);
5362
5363                 switch (code) {
5364                 case IWM_REPLY_RX_PHY_CMD:
5365                         iwm_mvm_rx_rx_phy_cmd(sc, pkt);
5366                         break;
5367
5368                 case IWM_REPLY_RX_MPDU_CMD: {
5369                         /*
5370                          * If this is the last frame in the RX buffer, we
5371                          * can directly feed the mbuf to the sharks here.
5372                          */
5373                         struct iwm_rx_packet *nextpkt = mtodoff(m,
5374                             struct iwm_rx_packet *, nextoff);
5375                         if (!HAVEROOM(nextoff) ||
5376                             (nextpkt->hdr.code == 0 &&
5377                              (nextpkt->hdr.qid & ~0x80) == 0 &&
5378                              nextpkt->hdr.idx == 0) ||
5379                             (nextpkt->len_n_flags ==
5380                              htole32(IWM_FH_RSCSR_FRAME_INVALID))) {
5381                                 if (iwm_mvm_rx_mpdu(sc, m, offset, stolen)) {
5382                                         stolen = FALSE;
5383                                         /* Make sure we abort the loop */
5384                                         nextoff = maxoff;
5385                                 }
5386                                 break;
5387                         }
5388
5389                         /*
5390                          * Use m_copym instead of m_split, because that
5391                          * makes it easier to keep a valid rx buffer in
5392                          * the ring, when iwm_mvm_rx_mpdu() fails.
5393                          *
5394                          * We need to start m_copym() at offset 0, to get the
5395                          * M_PKTHDR flag preserved.
5396                          */
5397                         m1 = m_copym(m, 0, M_COPYALL, M_NOWAIT);
5398                         if (m1) {
5399                                 if (iwm_mvm_rx_mpdu(sc, m1, offset, stolen))
5400                                         stolen = TRUE;
5401                                 else
5402                                         m_freem(m1);
5403                         }
5404                         break;
5405                 }
5406
5407                 case IWM_TX_CMD:
5408                         iwm_mvm_rx_tx_cmd(sc, pkt);
5409                         break;
5410
5411                 case IWM_MISSED_BEACONS_NOTIFICATION: {
5412                         struct iwm_missed_beacons_notif *resp;
5413                         int missed;
5414
5415                         /* XXX look at mac_id to determine interface ID */
5416                         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
5417
5418                         resp = (void *)pkt->data;
5419                         missed = le32toh(resp->consec_missed_beacons);
5420
5421                         IWM_DPRINTF(sc, IWM_DEBUG_BEACON | IWM_DEBUG_STATE,
5422                             "%s: MISSED_BEACON: mac_id=%d, "
5423                             "consec_since_last_rx=%d, consec=%d, num_expect=%d "
5424                             "num_rx=%d\n",
5425                             __func__,
5426                             le32toh(resp->mac_id),
5427                             le32toh(resp->consec_missed_beacons_since_last_rx),
5428                             le32toh(resp->consec_missed_beacons),
5429                             le32toh(resp->num_expected_beacons),
5430                             le32toh(resp->num_recvd_beacons));
5431
5432                         /* Be paranoid */
5433                         if (vap == NULL)
5434                                 break;
5435
5436                         /* XXX no net80211 locking? */
5437                         if (vap->iv_state == IEEE80211_S_RUN &&
5438                             (ic->ic_flags & IEEE80211_F_SCAN) == 0) {
5439                                 if (missed > vap->iv_bmissthreshold) {
5440                                         /* XXX bad locking; turn into task */
5441                                         IWM_UNLOCK(sc);
5442                                         ieee80211_beacon_miss(ic);
5443                                         IWM_LOCK(sc);
5444                                 }
5445                         }
5446
5447                         break;
5448                 }
5449
5450                 case IWM_MFUART_LOAD_NOTIFICATION:
5451                         break;
5452
5453                 case IWM_MVM_ALIVE:
5454                         break;
5455
5456                 case IWM_CALIB_RES_NOTIF_PHY_DB:
5457                         break;
5458
5459                 case IWM_STATISTICS_NOTIFICATION:
5460                         iwm_mvm_handle_rx_statistics(sc, pkt);
5461                         break;
5462
5463                 case IWM_NVM_ACCESS_CMD:
5464                 case IWM_MCC_UPDATE_CMD:
5465                         if (sc->sc_wantresp == (((qid & ~0x80) << 16) | idx)) {
5466                                 memcpy(sc->sc_cmd_resp,
5467                                     pkt, sizeof(sc->sc_cmd_resp));
5468                         }
5469                         break;
5470
5471                 case IWM_MCC_CHUB_UPDATE_CMD: {
5472                         struct iwm_mcc_chub_notif *notif;
5473                         notif = (void *)pkt->data;
5474
5475                         sc->sc_fw_mcc[0] = (notif->mcc & 0xff00) >> 8;
5476                         sc->sc_fw_mcc[1] = notif->mcc & 0xff;
5477                         sc->sc_fw_mcc[2] = '\0';
5478                         IWM_DPRINTF(sc, IWM_DEBUG_LAR,
5479                             "fw source %d sent CC '%s'\n",
5480                             notif->source_id, sc->sc_fw_mcc);
5481                         break;
5482                 }
5483
5484                 case IWM_DTS_MEASUREMENT_NOTIFICATION:
5485                 case IWM_WIDE_ID(IWM_PHY_OPS_GROUP,
5486                                  IWM_DTS_MEASUREMENT_NOTIF_WIDE): {
5487                         struct iwm_dts_measurement_notif_v1 *notif;
5488
5489                         if (iwm_rx_packet_payload_len(pkt) < sizeof(*notif)) {
5490                                 device_printf(sc->sc_dev,
5491                                     "Invalid DTS_MEASUREMENT_NOTIFICATION\n");
5492                                 break;
5493                         }
5494                         notif = (void *)pkt->data;
5495                         IWM_DPRINTF(sc, IWM_DEBUG_TEMP,
5496                             "IWM_DTS_MEASUREMENT_NOTIFICATION - %d\n",
5497                             notif->temp);
5498                         break;
5499                 }
5500
5501                 case IWM_PHY_CONFIGURATION_CMD:
5502                 case IWM_TX_ANT_CONFIGURATION_CMD:
5503                 case IWM_ADD_STA:
5504                 case IWM_MAC_CONTEXT_CMD:
5505                 case IWM_REPLY_SF_CFG_CMD:
5506                 case IWM_POWER_TABLE_CMD:
5507                 case IWM_LTR_CONFIG:
5508                 case IWM_PHY_CONTEXT_CMD:
5509                 case IWM_BINDING_CONTEXT_CMD:
5510                 case IWM_TIME_EVENT_CMD:
5511                 case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_CFG_CMD):
5512                 case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_REQ_UMAC):
5513                 case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_ABORT_UMAC):
5514                 case IWM_SCAN_OFFLOAD_REQUEST_CMD:
5515                 case IWM_SCAN_OFFLOAD_ABORT_CMD:
5516                 case IWM_REPLY_BEACON_FILTERING_CMD:
5517                 case IWM_MAC_PM_POWER_TABLE:
5518                 case IWM_TIME_QUOTA_CMD:
5519                 case IWM_REMOVE_STA:
5520                 case IWM_TXPATH_FLUSH:
5521                 case IWM_LQ_CMD:
5522                 case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP,
5523                                  IWM_FW_PAGING_BLOCK_CMD):
5524                 case IWM_BT_CONFIG:
5525                 case IWM_REPLY_THERMAL_MNG_BACKOFF:
5526                         cresp = (void *)pkt->data;
5527                         if (sc->sc_wantresp == (((qid & ~0x80) << 16) | idx)) {
5528                                 memcpy(sc->sc_cmd_resp,
5529                                     pkt, sizeof(*pkt)+sizeof(*cresp));
5530                         }
5531                         break;
5532
5533                 /* ignore */
5534                 case IWM_PHY_DB_CMD:
5535                         break;
5536
5537                 case IWM_INIT_COMPLETE_NOTIF:
5538                         break;
5539
5540                 case IWM_SCAN_OFFLOAD_COMPLETE:
5541                         iwm_mvm_rx_lmac_scan_complete_notif(sc, pkt);
5542                         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
5543                                 sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
5544                                 ieee80211_runtask(ic, &sc->sc_es_task);
5545                         }
5546                         break;
5547
5548                 case IWM_SCAN_ITERATION_COMPLETE: {
5549                         struct iwm_lmac_scan_complete_notif *notif;
5550                         notif = (void *)pkt->data;
5551                         break;
5552                 }
5553
5554                 case IWM_SCAN_COMPLETE_UMAC:
5555                         iwm_mvm_rx_umac_scan_complete_notif(sc, pkt);
5556                         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
5557                                 sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
5558                                 ieee80211_runtask(ic, &sc->sc_es_task);
5559                         }
5560                         break;
5561
5562                 case IWM_SCAN_ITERATION_COMPLETE_UMAC: {
5563                         struct iwm_umac_scan_iter_complete_notif *notif;
5564                         notif = (void *)pkt->data;
5565
5566                         IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "UMAC scan iteration "
5567                             "complete, status=0x%x, %d channels scanned\n",
5568                             notif->status, notif->scanned_channels);
5569                         break;
5570                 }
5571
5572                 case IWM_REPLY_ERROR: {
5573                         struct iwm_error_resp *resp;
5574                         resp = (void *)pkt->data;
5575
5576                         device_printf(sc->sc_dev,
5577                             "firmware error 0x%x, cmd 0x%x\n",
5578                             le32toh(resp->error_type),
5579                             resp->cmd_id);
5580                         break;
5581                 }
5582
5583                 case IWM_TIME_EVENT_NOTIFICATION:
5584                         iwm_mvm_rx_time_event_notif(sc, pkt);
5585                         break;
5586
5587                 /*
5588                  * Firmware versions 21 and 22 generate some DEBUG_LOG_MSG
5589                  * messages. Just ignore them for now.
5590                  */
5591                 case IWM_DEBUG_LOG_MSG:
5592                         break;
5593
5594                 case IWM_MCAST_FILTER_CMD:
5595                         break;
5596
5597                 case IWM_SCD_QUEUE_CFG: {
5598                         struct iwm_scd_txq_cfg_rsp *rsp;
5599                         rsp = (void *)pkt->data;
5600
5601                         IWM_DPRINTF(sc, IWM_DEBUG_CMD,
5602                             "queue cfg token=0x%x sta_id=%d "
5603                             "tid=%d scd_queue=%d\n",
5604                             rsp->token, rsp->sta_id, rsp->tid,
5605                             rsp->scd_queue);
5606                         break;
5607                 }
5608
5609                 default:
5610                         device_printf(sc->sc_dev,
5611                             "frame %d/%d %x UNHANDLED (this should "
5612                             "not happen)\n", qid & ~0x80, idx,
5613                             pkt->len_n_flags);
5614                         break;
5615                 }
5616
5617                 /*
5618                  * Why test bit 0x80?  The Linux driver:
5619                  *
5620                  * There is one exception:  uCode sets bit 15 when it
5621                  * originates the response/notification, i.e. when the
5622                  * response/notification is not a direct response to a
5623                  * command sent by the driver.  For example, uCode issues
5624                  * IWM_REPLY_RX when it sends a received frame to the driver;
5625                  * it is not a direct response to any driver command.
5626                  *
5627                  * Ok, so since when is 7 == 15?  Well, the Linux driver
5628                  * uses a slightly different format for pkt->hdr, and "qid"
5629                  * is actually the upper byte of a two-byte field.
5630                  */
5631                 if (!(qid & (1 << 7)))
5632                         iwm_cmd_done(sc, pkt);
5633
5634                 offset = nextoff;
5635         }
5636         if (stolen)
5637                 m_freem(m);
5638 #undef HAVEROOM
5639 }
5640
5641 /*
5642  * Process an IWM_CSR_INT_BIT_FH_RX or IWM_CSR_INT_BIT_SW_RX interrupt.
5643  * Basic structure from if_iwn
5644  */
5645 static void
5646 iwm_notif_intr(struct iwm_softc *sc)
5647 {
5648         int count;
5649         uint32_t wreg;
5650         uint16_t hw;
5651
5652         bus_dmamap_sync(sc->rxq.stat_dma.tag, sc->rxq.stat_dma.map,
5653             BUS_DMASYNC_POSTREAD);
5654
5655         if (sc->cfg->mqrx_supported) {
5656                 count = IWM_RX_MQ_RING_COUNT;
5657                 wreg = IWM_RFH_Q0_FRBDCB_WIDX_TRG;
5658         } else {
5659                 count = IWM_RX_LEGACY_RING_COUNT;
5660                 wreg = IWM_FH_RSCSR_CHNL0_WPTR;
5661         }
5662
5663         hw = le16toh(sc->rxq.stat->closed_rb_num) & 0xfff;
5664
5665         /*
5666          * Process responses
5667          */
5668         while (sc->rxq.cur != hw) {
5669                 struct iwm_rx_ring *ring = &sc->rxq;
5670                 struct iwm_rx_data *data = &ring->data[ring->cur];
5671
5672                 bus_dmamap_sync(ring->data_dmat, data->map,
5673                     BUS_DMASYNC_POSTREAD);
5674
5675                 IWM_DPRINTF(sc, IWM_DEBUG_INTR,
5676                     "%s: hw = %d cur = %d\n", __func__, hw, ring->cur);
5677                 iwm_handle_rxb(sc, data->m);
5678
5679                 ring->cur = (ring->cur + 1) % count;
5680         }
5681
5682         /*
5683          * Tell the firmware that it can reuse the ring entries that
5684          * we have just processed.
5685          * Seems like the hardware gets upset unless we align
5686          * the write by 8??
5687          */
5688         hw = (hw == 0) ? count - 1 : hw - 1;
5689         IWM_WRITE(sc, wreg, rounddown2(hw, 8));
5690 }
5691
5692 static void
5693 iwm_intr(void *arg)
5694 {
5695         struct iwm_softc *sc = arg;
5696         int handled = 0;
5697         int r1, r2, rv = 0;
5698         int isperiodic = 0;
5699
5700         IWM_LOCK(sc);
5701         IWM_WRITE(sc, IWM_CSR_INT_MASK, 0);
5702
5703         if (sc->sc_flags & IWM_FLAG_USE_ICT) {
5704                 uint32_t *ict = sc->ict_dma.vaddr;
5705                 int tmp;
5706
5707                 tmp = htole32(ict[sc->ict_cur]);
5708                 if (!tmp)
5709                         goto out_ena;
5710
5711                 /*
5712                  * ok, there was something.  keep plowing until we have all.
5713                  */
5714                 r1 = r2 = 0;
5715                 while (tmp) {
5716                         r1 |= tmp;
5717                         ict[sc->ict_cur] = 0;
5718                         sc->ict_cur = (sc->ict_cur+1) % IWM_ICT_COUNT;
5719                         tmp = htole32(ict[sc->ict_cur]);
5720                 }
5721
5722                 /* this is where the fun begins.  don't ask */
5723                 if (r1 == 0xffffffff)
5724                         r1 = 0;
5725
5726                 /* i am not expected to understand this */
5727                 if (r1 & 0xc0000)
5728                         r1 |= 0x8000;
5729                 r1 = (0xff & r1) | ((0xff00 & r1) << 16);
5730         } else {
5731                 r1 = IWM_READ(sc, IWM_CSR_INT);
5732                 /* "hardware gone" (where, fishing?) */
5733                 if (r1 == 0xffffffff || (r1 & 0xfffffff0) == 0xa5a5a5a0)
5734                         goto out;
5735                 r2 = IWM_READ(sc, IWM_CSR_FH_INT_STATUS);
5736         }
5737         if (r1 == 0 && r2 == 0) {
5738                 goto out_ena;
5739         }
5740
5741         IWM_WRITE(sc, IWM_CSR_INT, r1 | ~sc->sc_intmask);
5742
5743         /* Safely ignore these bits for debug checks below */
5744         r1 &= ~(IWM_CSR_INT_BIT_ALIVE | IWM_CSR_INT_BIT_SCD);
5745
5746         if (r1 & IWM_CSR_INT_BIT_SW_ERR) {
5747                 int i;
5748                 struct ieee80211com *ic = &sc->sc_ic;
5749                 struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
5750
5751 #ifdef IWM_DEBUG
5752                 iwm_nic_error(sc);
5753 #endif
5754                 /* Dump driver status (TX and RX rings) while we're here. */
5755                 device_printf(sc->sc_dev, "driver status:\n");
5756                 for (i = 0; i < IWM_MVM_MAX_QUEUES; i++) {
5757                         struct iwm_tx_ring *ring = &sc->txq[i];
5758                         device_printf(sc->sc_dev,
5759                             "  tx ring %2d: qid=%-2d cur=%-3d "
5760                             "queued=%-3d\n",
5761                             i, ring->qid, ring->cur, ring->queued);
5762                 }
5763                 device_printf(sc->sc_dev,
5764                     "  rx ring: cur=%d\n", sc->rxq.cur);
5765                 device_printf(sc->sc_dev,
5766                     "  802.11 state %d\n", (vap == NULL) ? -1 : vap->iv_state);
5767
5768                 /* Reset our firmware state tracking. */
5769                 sc->sc_firmware_state = 0;
5770                 /* Don't stop the device; just do a VAP restart */
5771                 IWM_UNLOCK(sc);
5772
5773                 if (vap == NULL) {
5774                         printf("%s: null vap\n", __func__);
5775                         return;
5776                 }
5777
5778                 device_printf(sc->sc_dev, "%s: controller panicked, iv_state = %d; "
5779                     "restarting\n", __func__, vap->iv_state);
5780
5781                 ieee80211_restart_all(ic);
5782                 return;
5783         }
5784
5785         if (r1 & IWM_CSR_INT_BIT_HW_ERR) {
5786                 handled |= IWM_CSR_INT_BIT_HW_ERR;
5787                 device_printf(sc->sc_dev, "hardware error, stopping device\n");
5788                 iwm_stop(sc);
5789                 rv = 1;
5790                 goto out;
5791         }
5792
5793         /* firmware chunk loaded */
5794         if (r1 & IWM_CSR_INT_BIT_FH_TX) {
5795                 IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, IWM_CSR_FH_INT_TX_MASK);
5796                 handled |= IWM_CSR_INT_BIT_FH_TX;
5797                 sc->sc_fw_chunk_done = 1;
5798                 wakeup(&sc->sc_fw);
5799         }
5800
5801         if (r1 & IWM_CSR_INT_BIT_RF_KILL) {
5802                 handled |= IWM_CSR_INT_BIT_RF_KILL;
5803                 if (iwm_check_rfkill(sc)) {
5804                         device_printf(sc->sc_dev,
5805                             "%s: rfkill switch, disabling interface\n",
5806                             __func__);
5807                         iwm_stop(sc);
5808                 }
5809         }
5810
5811         /*
5812          * The Linux driver uses periodic interrupts to avoid races.
5813          * We cargo-cult like it's going out of fashion.
5814          */
5815         if (r1 & IWM_CSR_INT_BIT_RX_PERIODIC) {
5816                 handled |= IWM_CSR_INT_BIT_RX_PERIODIC;
5817                 IWM_WRITE(sc, IWM_CSR_INT, IWM_CSR_INT_BIT_RX_PERIODIC);
5818                 if ((r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX)) == 0)
5819                         IWM_WRITE_1(sc,
5820                             IWM_CSR_INT_PERIODIC_REG, IWM_CSR_INT_PERIODIC_DIS);
5821                 isperiodic = 1;
5822         }
5823
5824         if ((r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX)) || isperiodic) {
5825                 handled |= (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX);
5826                 IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, IWM_CSR_FH_INT_RX_MASK);
5827
5828                 iwm_notif_intr(sc);
5829
5830                 /* enable periodic interrupt, see above */
5831                 if (r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX) && !isperiodic)
5832                         IWM_WRITE_1(sc, IWM_CSR_INT_PERIODIC_REG,
5833                             IWM_CSR_INT_PERIODIC_ENA);
5834         }
5835
5836         if (__predict_false(r1 & ~handled))
5837                 IWM_DPRINTF(sc, IWM_DEBUG_INTR,
5838                     "%s: unhandled interrupts: %x\n", __func__, r1);
5839         rv = 1;
5840
5841  out_ena:
5842         iwm_restore_interrupts(sc);
5843  out:
5844         IWM_UNLOCK(sc);
5845         return;
5846 }
5847
5848 /*
5849  * Autoconf glue-sniffing
5850  */
5851 #define PCI_VENDOR_INTEL                0x8086
5852 #define PCI_PRODUCT_INTEL_WL_3160_1     0x08b3
5853 #define PCI_PRODUCT_INTEL_WL_3160_2     0x08b4
5854 #define PCI_PRODUCT_INTEL_WL_3165_1     0x3165
5855 #define PCI_PRODUCT_INTEL_WL_3165_2     0x3166
5856 #define PCI_PRODUCT_INTEL_WL_3168_1     0x24fb
5857 #define PCI_PRODUCT_INTEL_WL_7260_1     0x08b1
5858 #define PCI_PRODUCT_INTEL_WL_7260_2     0x08b2
5859 #define PCI_PRODUCT_INTEL_WL_7265_1     0x095a
5860 #define PCI_PRODUCT_INTEL_WL_7265_2     0x095b
5861 #define PCI_PRODUCT_INTEL_WL_8260_1     0x24f3
5862 #define PCI_PRODUCT_INTEL_WL_8260_2     0x24f4
5863 #define PCI_PRODUCT_INTEL_WL_8265_1     0x24fd
5864 #define PCI_PRODUCT_INTEL_WL_9560_1     0x9df0
5865 #define PCI_PRODUCT_INTEL_WL_9560_2     0xa370
5866 #define PCI_PRODUCT_INTEL_WL_9260_1     0x2526
5867
5868 static const struct iwm_devices {
5869         uint16_t                device;
5870         const struct iwm_cfg    *cfg;
5871 } iwm_devices[] = {
5872         { PCI_PRODUCT_INTEL_WL_3160_1, &iwm3160_cfg },
5873         { PCI_PRODUCT_INTEL_WL_3160_2, &iwm3160_cfg },
5874         { PCI_PRODUCT_INTEL_WL_3165_1, &iwm3165_cfg },
5875         { PCI_PRODUCT_INTEL_WL_3165_2, &iwm3165_cfg },
5876         { PCI_PRODUCT_INTEL_WL_3168_1, &iwm3168_cfg },
5877         { PCI_PRODUCT_INTEL_WL_7260_1, &iwm7260_cfg },
5878         { PCI_PRODUCT_INTEL_WL_7260_2, &iwm7260_cfg },
5879         { PCI_PRODUCT_INTEL_WL_7265_1, &iwm7265_cfg },
5880         { PCI_PRODUCT_INTEL_WL_7265_2, &iwm7265_cfg },
5881         { PCI_PRODUCT_INTEL_WL_8260_1, &iwm8260_cfg },
5882         { PCI_PRODUCT_INTEL_WL_8260_2, &iwm8260_cfg },
5883         { PCI_PRODUCT_INTEL_WL_8265_1, &iwm8265_cfg },
5884         { PCI_PRODUCT_INTEL_WL_9560_1, &iwm9560_cfg },
5885         { PCI_PRODUCT_INTEL_WL_9560_2, &iwm9560_cfg },
5886         { PCI_PRODUCT_INTEL_WL_9260_1, &iwm9260_cfg },
5887 };
5888
5889 static int
5890 iwm_probe(device_t dev)
5891 {
5892         int i;
5893
5894         for (i = 0; i < nitems(iwm_devices); i++) {
5895                 if (pci_get_vendor(dev) == PCI_VENDOR_INTEL &&
5896                     pci_get_device(dev) == iwm_devices[i].device) {
5897                         device_set_desc(dev, iwm_devices[i].cfg->name);
5898                         return (BUS_PROBE_DEFAULT);
5899                 }
5900         }
5901
5902         return (ENXIO);
5903 }
5904
5905 static int
5906 iwm_dev_check(device_t dev)
5907 {
5908         struct iwm_softc *sc;
5909         uint16_t devid;
5910         int i;
5911
5912         sc = device_get_softc(dev);
5913
5914         devid = pci_get_device(dev);
5915         for (i = 0; i < nitems(iwm_devices); i++) {
5916                 if (iwm_devices[i].device == devid) {
5917                         sc->cfg = iwm_devices[i].cfg;
5918                         return (0);
5919                 }
5920         }
5921         device_printf(dev, "unknown adapter type\n");
5922         return ENXIO;
5923 }
5924
5925 /* PCI registers */
5926 #define PCI_CFG_RETRY_TIMEOUT   0x041
5927
5928 static int
5929 iwm_pci_attach(device_t dev)
5930 {
5931         struct iwm_softc *sc;
5932         int count, error, rid;
5933         uint16_t reg;
5934
5935         sc = device_get_softc(dev);
5936
5937         /* We disable the RETRY_TIMEOUT register (0x41) to keep
5938          * PCI Tx retries from interfering with C3 CPU state */
5939         pci_write_config(dev, PCI_CFG_RETRY_TIMEOUT, 0x00, 1);
5940
5941         /* Enable bus-mastering and hardware bug workaround. */
5942         pci_enable_busmaster(dev);
5943         reg = pci_read_config(dev, PCIR_STATUS, sizeof(reg));
5944         /* if !MSI */
5945         if (reg & PCIM_STATUS_INTxSTATE) {
5946                 reg &= ~PCIM_STATUS_INTxSTATE;
5947         }
5948         pci_write_config(dev, PCIR_STATUS, reg, sizeof(reg));
5949
5950         rid = PCIR_BAR(0);
5951         sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
5952             RF_ACTIVE);
5953         if (sc->sc_mem == NULL) {
5954                 device_printf(sc->sc_dev, "can't map mem space\n");
5955                 return (ENXIO);
5956         }
5957         sc->sc_st = rman_get_bustag(sc->sc_mem);
5958         sc->sc_sh = rman_get_bushandle(sc->sc_mem);
5959
5960         /* Install interrupt handler. */
5961         count = 1;
5962         rid = 0;
5963         if (pci_alloc_msi(dev, &count) == 0)
5964                 rid = 1;
5965         sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE |
5966             (rid != 0 ? 0 : RF_SHAREABLE));
5967         if (sc->sc_irq == NULL) {
5968                 device_printf(dev, "can't map interrupt\n");
5969                         return (ENXIO);
5970         }
5971         error = bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
5972             NULL, iwm_intr, sc, &sc->sc_ih);
5973         if (sc->sc_ih == NULL) {
5974                 device_printf(dev, "can't establish interrupt");
5975                         return (ENXIO);
5976         }
5977         sc->sc_dmat = bus_get_dma_tag(sc->sc_dev);
5978
5979         return (0);
5980 }
5981
5982 static void
5983 iwm_pci_detach(device_t dev)
5984 {
5985         struct iwm_softc *sc = device_get_softc(dev);
5986
5987         if (sc->sc_irq != NULL) {
5988                 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
5989                 bus_release_resource(dev, SYS_RES_IRQ,
5990                     rman_get_rid(sc->sc_irq), sc->sc_irq);
5991                 pci_release_msi(dev);
5992         }
5993         if (sc->sc_mem != NULL)
5994                 bus_release_resource(dev, SYS_RES_MEMORY,
5995                     rman_get_rid(sc->sc_mem), sc->sc_mem);
5996 }
5997
5998 static int
5999 iwm_attach(device_t dev)
6000 {
6001         struct iwm_softc *sc = device_get_softc(dev);
6002         struct ieee80211com *ic = &sc->sc_ic;
6003         int error;
6004         int txq_i, i;
6005
6006         sc->sc_dev = dev;
6007         sc->sc_attached = 1;
6008         IWM_LOCK_INIT(sc);
6009         mbufq_init(&sc->sc_snd, ifqmaxlen);
6010         callout_init_mtx(&sc->sc_watchdog_to, &sc->sc_mtx, 0);
6011         callout_init_mtx(&sc->sc_led_blink_to, &sc->sc_mtx, 0);
6012         TASK_INIT(&sc->sc_es_task, 0, iwm_endscan_cb, sc);
6013
6014         error = iwm_dev_check(dev);
6015         if (error != 0)
6016                 goto fail;
6017
6018         sc->sc_notif_wait = iwm_notification_wait_init(sc);
6019         if (sc->sc_notif_wait == NULL) {
6020                 device_printf(dev, "failed to init notification wait struct\n");
6021                 goto fail;
6022         }
6023
6024         sc->sf_state = IWM_SF_UNINIT;
6025
6026         /* Init phy db */
6027         sc->sc_phy_db = iwm_phy_db_init(sc);
6028         if (!sc->sc_phy_db) {
6029                 device_printf(dev, "Cannot init phy_db\n");
6030                 goto fail;
6031         }
6032
6033         /* Set EBS as successful as long as not stated otherwise by the FW. */
6034         sc->last_ebs_successful = TRUE;
6035
6036         /* PCI attach */
6037         error = iwm_pci_attach(dev);
6038         if (error != 0)
6039                 goto fail;
6040
6041         sc->sc_wantresp = -1;
6042
6043         sc->sc_hw_rev = IWM_READ(sc, IWM_CSR_HW_REV);
6044         /*
6045          * In the 8000 HW family the format of the 4 bytes of CSR_HW_REV have
6046          * changed, and now the revision step also includes bit 0-1 (no more
6047          * "dash" value). To keep hw_rev backwards compatible - we'll store it
6048          * in the old format.
6049          */
6050         if (sc->cfg->device_family >= IWM_DEVICE_FAMILY_8000) {
6051                 int ret;
6052                 uint32_t hw_step;
6053
6054                 sc->sc_hw_rev = (sc->sc_hw_rev & 0xfff0) |
6055                                 (IWM_CSR_HW_REV_STEP(sc->sc_hw_rev << 2) << 2);
6056
6057                 if (iwm_prepare_card_hw(sc) != 0) {
6058                         device_printf(dev, "could not initialize hardware\n");
6059                         goto fail;
6060                 }
6061
6062                 /*
6063                  * In order to recognize C step the driver should read the
6064                  * chip version id located at the AUX bus MISC address.
6065                  */
6066                 IWM_SETBITS(sc, IWM_CSR_GP_CNTRL,
6067                             IWM_CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
6068                 DELAY(2);
6069
6070                 ret = iwm_poll_bit(sc, IWM_CSR_GP_CNTRL,
6071                                    IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
6072                                    IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
6073                                    25000);
6074                 if (!ret) {
6075                         device_printf(sc->sc_dev,
6076                             "Failed to wake up the nic\n");
6077                         goto fail;
6078                 }
6079
6080                 if (iwm_nic_lock(sc)) {
6081                         hw_step = iwm_read_prph(sc, IWM_WFPM_CTRL_REG);
6082                         hw_step |= IWM_ENABLE_WFPM;
6083                         iwm_write_prph(sc, IWM_WFPM_CTRL_REG, hw_step);
6084                         hw_step = iwm_read_prph(sc, IWM_AUX_MISC_REG);
6085                         hw_step = (hw_step >> IWM_HW_STEP_LOCATION_BITS) & 0xF;
6086                         if (hw_step == 0x3)
6087                                 sc->sc_hw_rev = (sc->sc_hw_rev & 0xFFFFFFF3) |
6088                                                 (IWM_SILICON_C_STEP << 2);
6089                         iwm_nic_unlock(sc);
6090                 } else {
6091                         device_printf(sc->sc_dev, "Failed to lock the nic\n");
6092                         goto fail;
6093                 }
6094         }
6095
6096         /* special-case 7265D, it has the same PCI IDs. */
6097         if (sc->cfg == &iwm7265_cfg &&
6098             (sc->sc_hw_rev & IWM_CSR_HW_REV_TYPE_MSK) == IWM_CSR_HW_REV_TYPE_7265D) {
6099                 sc->cfg = &iwm7265d_cfg;
6100         }
6101
6102         /* Allocate DMA memory for firmware transfers. */
6103         if ((error = iwm_alloc_fwmem(sc)) != 0) {
6104                 device_printf(dev, "could not allocate memory for firmware\n");
6105                 goto fail;
6106         }
6107
6108         /* Allocate "Keep Warm" page. */
6109         if ((error = iwm_alloc_kw(sc)) != 0) {
6110                 device_printf(dev, "could not allocate keep warm page\n");
6111                 goto fail;
6112         }
6113
6114         /* We use ICT interrupts */
6115         if ((error = iwm_alloc_ict(sc)) != 0) {
6116                 device_printf(dev, "could not allocate ICT table\n");
6117                 goto fail;
6118         }
6119
6120         /* Allocate TX scheduler "rings". */
6121         if ((error = iwm_alloc_sched(sc)) != 0) {
6122                 device_printf(dev, "could not allocate TX scheduler rings\n");
6123                 goto fail;
6124         }
6125
6126         /* Allocate TX rings */
6127         for (txq_i = 0; txq_i < nitems(sc->txq); txq_i++) {
6128                 if ((error = iwm_alloc_tx_ring(sc,
6129                     &sc->txq[txq_i], txq_i)) != 0) {
6130                         device_printf(dev,
6131                             "could not allocate TX ring %d\n",
6132                             txq_i);
6133                         goto fail;
6134                 }
6135         }
6136
6137         /* Allocate RX ring. */
6138         if ((error = iwm_alloc_rx_ring(sc, &sc->rxq)) != 0) {
6139                 device_printf(dev, "could not allocate RX ring\n");
6140                 goto fail;
6141         }
6142
6143         /* Clear pending interrupts. */
6144         IWM_WRITE(sc, IWM_CSR_INT, 0xffffffff);
6145
6146         ic->ic_softc = sc;
6147         ic->ic_name = device_get_nameunit(sc->sc_dev);
6148         ic->ic_phytype = IEEE80211_T_OFDM;      /* not only, but not used */
6149         ic->ic_opmode = IEEE80211_M_STA;        /* default to BSS mode */
6150
6151         /* Set device capabilities. */
6152         ic->ic_caps =
6153             IEEE80211_C_STA |
6154             IEEE80211_C_WPA |           /* WPA/RSN */
6155             IEEE80211_C_WME |
6156             IEEE80211_C_PMGT |
6157             IEEE80211_C_SHSLOT |        /* short slot time supported */
6158             IEEE80211_C_SHPREAMBLE      /* short preamble supported */
6159 //          IEEE80211_C_BGSCAN          /* capable of bg scanning */
6160             ;
6161         /* Advertise full-offload scanning */
6162         ic->ic_flags_ext = IEEE80211_FEXT_SCAN_OFFLOAD;
6163         for (i = 0; i < nitems(sc->sc_phyctxt); i++) {
6164                 sc->sc_phyctxt[i].id = i;
6165                 sc->sc_phyctxt[i].color = 0;
6166                 sc->sc_phyctxt[i].ref = 0;
6167                 sc->sc_phyctxt[i].channel = NULL;
6168         }
6169
6170         /* Default noise floor */
6171         sc->sc_noise = -96;
6172
6173         /* Max RSSI */
6174         sc->sc_max_rssi = IWM_MAX_DBM - IWM_MIN_DBM;
6175
6176 #ifdef IWM_DEBUG
6177         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
6178             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug",
6179             CTLFLAG_RW, &sc->sc_debug, 0, "control debugging");
6180 #endif
6181
6182         error = iwm_read_firmware(sc);
6183         if (error) {
6184                 goto fail;
6185         } else if (sc->sc_fw.fw_fp == NULL) {
6186                 /*
6187                  * XXX Add a solution for properly deferring firmware load
6188                  *     during bootup.
6189                  */
6190                 goto fail;
6191         } else {
6192                 sc->sc_preinit_hook.ich_func = iwm_preinit;
6193                 sc->sc_preinit_hook.ich_arg = sc;
6194                 if (config_intrhook_establish(&sc->sc_preinit_hook) != 0) {
6195                         device_printf(dev,
6196                             "config_intrhook_establish failed\n");
6197                         goto fail;
6198                 }
6199         }
6200
6201         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6202             "<-%s\n", __func__);
6203
6204         return 0;
6205
6206         /* Free allocated memory if something failed during attachment. */
6207 fail:
6208         iwm_detach_local(sc, 0);
6209
6210         return ENXIO;
6211 }
6212
6213 static int
6214 iwm_is_valid_ether_addr(uint8_t *addr)
6215 {
6216         char zero_addr[IEEE80211_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 };
6217
6218         if ((addr[0] & 1) || IEEE80211_ADDR_EQ(zero_addr, addr))
6219                 return (FALSE);
6220
6221         return (TRUE);
6222 }
6223
6224 static int
6225 iwm_wme_update(struct ieee80211com *ic)
6226 {
6227 #define IWM_EXP2(x)     ((1 << (x)) - 1)        /* CWmin = 2^ECWmin - 1 */
6228         struct iwm_softc *sc = ic->ic_softc;
6229         struct chanAccParams chp;
6230         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
6231         struct iwm_vap *ivp = IWM_VAP(vap);
6232         struct iwm_node *in;
6233         struct wmeParams tmp[WME_NUM_AC];
6234         int aci, error;
6235
6236         if (vap == NULL)
6237                 return (0);
6238
6239         ieee80211_wme_ic_getparams(ic, &chp);
6240
6241         IEEE80211_LOCK(ic);
6242         for (aci = 0; aci < WME_NUM_AC; aci++)
6243                 tmp[aci] = chp.cap_wmeParams[aci];
6244         IEEE80211_UNLOCK(ic);
6245
6246         IWM_LOCK(sc);
6247         for (aci = 0; aci < WME_NUM_AC; aci++) {
6248                 const struct wmeParams *ac = &tmp[aci];
6249                 ivp->queue_params[aci].aifsn = ac->wmep_aifsn;
6250                 ivp->queue_params[aci].cw_min = IWM_EXP2(ac->wmep_logcwmin);
6251                 ivp->queue_params[aci].cw_max = IWM_EXP2(ac->wmep_logcwmax);
6252                 ivp->queue_params[aci].edca_txop =
6253                     IEEE80211_TXOP_TO_US(ac->wmep_txopLimit);
6254         }
6255         ivp->have_wme = TRUE;
6256         if (ivp->is_uploaded && vap->iv_bss != NULL) {
6257                 in = IWM_NODE(vap->iv_bss);
6258                 if (in->in_assoc) {
6259                         if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) {
6260                                 device_printf(sc->sc_dev,
6261                                     "%s: failed to update MAC\n", __func__);
6262                         }
6263                 }
6264         }
6265         IWM_UNLOCK(sc);
6266
6267         return (0);
6268 #undef IWM_EXP2
6269 }
6270
6271 static void
6272 iwm_preinit(void *arg)
6273 {
6274         struct iwm_softc *sc = arg;
6275         device_t dev = sc->sc_dev;
6276         struct ieee80211com *ic = &sc->sc_ic;
6277         int error;
6278
6279         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6280             "->%s\n", __func__);
6281
6282         IWM_LOCK(sc);
6283         if ((error = iwm_start_hw(sc)) != 0) {
6284                 device_printf(dev, "could not initialize hardware\n");
6285                 IWM_UNLOCK(sc);
6286                 goto fail;
6287         }
6288
6289         error = iwm_run_init_mvm_ucode(sc, 1);
6290         iwm_stop_device(sc);
6291         if (error) {
6292                 IWM_UNLOCK(sc);
6293                 goto fail;
6294         }
6295         device_printf(dev,
6296             "hw rev 0x%x, fw ver %s, address %s\n",
6297             sc->sc_hw_rev & IWM_CSR_HW_REV_TYPE_MSK,
6298             sc->sc_fwver, ether_sprintf(sc->nvm_data->hw_addr));
6299
6300         /* not all hardware can do 5GHz band */
6301         if (!sc->nvm_data->sku_cap_band_52GHz_enable)
6302                 memset(&ic->ic_sup_rates[IEEE80211_MODE_11A], 0,
6303                     sizeof(ic->ic_sup_rates[IEEE80211_MODE_11A]));
6304         IWM_UNLOCK(sc);
6305
6306         iwm_init_channel_map(ic, IEEE80211_CHAN_MAX, &ic->ic_nchans,
6307             ic->ic_channels);
6308
6309         /*
6310          * At this point we've committed - if we fail to do setup,
6311          * we now also have to tear down the net80211 state.
6312          */
6313         ieee80211_ifattach(ic);
6314         ic->ic_vap_create = iwm_vap_create;
6315         ic->ic_vap_delete = iwm_vap_delete;
6316         ic->ic_raw_xmit = iwm_raw_xmit;
6317         ic->ic_node_alloc = iwm_node_alloc;
6318         ic->ic_scan_start = iwm_scan_start;
6319         ic->ic_scan_end = iwm_scan_end;
6320         ic->ic_update_mcast = iwm_update_mcast;
6321         ic->ic_getradiocaps = iwm_init_channel_map;
6322         ic->ic_set_channel = iwm_set_channel;
6323         ic->ic_scan_curchan = iwm_scan_curchan;
6324         ic->ic_scan_mindwell = iwm_scan_mindwell;
6325         ic->ic_wme.wme_update = iwm_wme_update;
6326         ic->ic_parent = iwm_parent;
6327         ic->ic_transmit = iwm_transmit;
6328         iwm_radiotap_attach(sc);
6329         if (bootverbose)
6330                 ieee80211_announce(ic);
6331
6332         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6333             "<-%s\n", __func__);
6334         config_intrhook_disestablish(&sc->sc_preinit_hook);
6335
6336         return;
6337 fail:
6338         config_intrhook_disestablish(&sc->sc_preinit_hook);
6339         iwm_detach_local(sc, 0);
6340 }
6341
6342 /*
6343  * Attach the interface to 802.11 radiotap.
6344  */
6345 static void
6346 iwm_radiotap_attach(struct iwm_softc *sc)
6347 {
6348         struct ieee80211com *ic = &sc->sc_ic;
6349
6350         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6351             "->%s begin\n", __func__);
6352         ieee80211_radiotap_attach(ic,
6353             &sc->sc_txtap.wt_ihdr, sizeof(sc->sc_txtap),
6354                 IWM_TX_RADIOTAP_PRESENT,
6355             &sc->sc_rxtap.wr_ihdr, sizeof(sc->sc_rxtap),
6356                 IWM_RX_RADIOTAP_PRESENT);
6357         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6358             "->%s end\n", __func__);
6359 }
6360
6361 static struct ieee80211vap *
6362 iwm_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], int unit,
6363     enum ieee80211_opmode opmode, int flags,
6364     const uint8_t bssid[IEEE80211_ADDR_LEN],
6365     const uint8_t mac[IEEE80211_ADDR_LEN])
6366 {
6367         struct iwm_vap *ivp;
6368         struct ieee80211vap *vap;
6369
6370         if (!TAILQ_EMPTY(&ic->ic_vaps))         /* only one at a time */
6371                 return NULL;
6372         ivp = malloc(sizeof(struct iwm_vap), M_80211_VAP, M_WAITOK | M_ZERO);
6373         vap = &ivp->iv_vap;
6374         ieee80211_vap_setup(ic, vap, name, unit, opmode, flags, bssid);
6375         vap->iv_bmissthreshold = 10;            /* override default */
6376         /* Override with driver methods. */
6377         ivp->iv_newstate = vap->iv_newstate;
6378         vap->iv_newstate = iwm_newstate;
6379
6380         ivp->id = IWM_DEFAULT_MACID;
6381         ivp->color = IWM_DEFAULT_COLOR;
6382
6383         ivp->have_wme = FALSE;
6384         ivp->ps_disabled = FALSE;
6385
6386         ieee80211_ratectl_init(vap);
6387         /* Complete setup. */
6388         ieee80211_vap_attach(vap, iwm_media_change, ieee80211_media_status,
6389             mac);
6390         ic->ic_opmode = opmode;
6391
6392         return vap;
6393 }
6394
6395 static void
6396 iwm_vap_delete(struct ieee80211vap *vap)
6397 {
6398         struct iwm_vap *ivp = IWM_VAP(vap);
6399
6400         ieee80211_ratectl_deinit(vap);
6401         ieee80211_vap_detach(vap);
6402         free(ivp, M_80211_VAP);
6403 }
6404
6405 static void
6406 iwm_xmit_queue_drain(struct iwm_softc *sc)
6407 {
6408         struct mbuf *m;
6409         struct ieee80211_node *ni;
6410
6411         while ((m = mbufq_dequeue(&sc->sc_snd)) != NULL) {
6412                 ni = (struct ieee80211_node *)m->m_pkthdr.rcvif;
6413                 ieee80211_free_node(ni);
6414                 m_freem(m);
6415         }
6416 }
6417
6418 static void
6419 iwm_scan_start(struct ieee80211com *ic)
6420 {
6421         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
6422         struct iwm_softc *sc = ic->ic_softc;
6423         int error;
6424
6425         IWM_LOCK(sc);
6426         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
6427                 /* This should not be possible */
6428                 device_printf(sc->sc_dev,
6429                     "%s: Previous scan not completed yet\n", __func__);
6430         }
6431         if (iwm_fw_has_capa(sc, IWM_UCODE_TLV_CAPA_UMAC_SCAN))
6432                 error = iwm_mvm_umac_scan(sc);
6433         else
6434                 error = iwm_mvm_lmac_scan(sc);
6435         if (error != 0) {
6436                 device_printf(sc->sc_dev, "could not initiate scan\n");
6437                 IWM_UNLOCK(sc);
6438                 ieee80211_cancel_scan(vap);
6439         } else {
6440                 sc->sc_flags |= IWM_FLAG_SCAN_RUNNING;
6441                 iwm_led_blink_start(sc);
6442                 IWM_UNLOCK(sc);
6443         }
6444 }
6445
6446 static void
6447 iwm_scan_end(struct ieee80211com *ic)
6448 {
6449         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
6450         struct iwm_softc *sc = ic->ic_softc;
6451
6452         IWM_LOCK(sc);
6453         iwm_led_blink_stop(sc);
6454         if (vap->iv_state == IEEE80211_S_RUN)
6455                 iwm_mvm_led_enable(sc);
6456         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
6457                 /*
6458                  * Removing IWM_FLAG_SCAN_RUNNING now, is fine because
6459                  * both iwm_scan_end and iwm_scan_start run in the ic->ic_tq
6460                  * taskqueue.
6461                  */
6462                 sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
6463                 iwm_mvm_scan_stop_wait(sc);
6464         }
6465         IWM_UNLOCK(sc);
6466
6467         /*
6468          * Make sure we don't race, if sc_es_task is still enqueued here.
6469          * This is to make sure that it won't call ieee80211_scan_done
6470          * when we have already started the next scan.
6471          */
6472         taskqueue_cancel(ic->ic_tq, &sc->sc_es_task, NULL);
6473 }
6474
6475 static void
6476 iwm_update_mcast(struct ieee80211com *ic)
6477 {
6478 }
6479
6480 static void
6481 iwm_set_channel(struct ieee80211com *ic)
6482 {
6483 }
6484
6485 static void
6486 iwm_scan_curchan(struct ieee80211_scan_state *ss, unsigned long maxdwell)
6487 {
6488 }
6489
6490 static void
6491 iwm_scan_mindwell(struct ieee80211_scan_state *ss)
6492 {
6493         return;
6494 }
6495
6496 void
6497 iwm_init_task(void *arg1)
6498 {
6499         struct iwm_softc *sc = arg1;
6500
6501         IWM_LOCK(sc);
6502         while (sc->sc_flags & IWM_FLAG_BUSY)
6503                 msleep(&sc->sc_flags, &sc->sc_mtx, 0, "iwmpwr", 0);
6504         sc->sc_flags |= IWM_FLAG_BUSY;
6505         iwm_stop(sc);
6506         if (sc->sc_ic.ic_nrunning > 0)
6507                 iwm_init(sc);
6508         sc->sc_flags &= ~IWM_FLAG_BUSY;
6509         wakeup(&sc->sc_flags);
6510         IWM_UNLOCK(sc);
6511 }
6512
6513 static int
6514 iwm_resume(device_t dev)
6515 {
6516         struct iwm_softc *sc = device_get_softc(dev);
6517         int do_reinit = 0;
6518
6519         /*
6520          * We disable the RETRY_TIMEOUT register (0x41) to keep
6521          * PCI Tx retries from interfering with C3 CPU state.
6522          */
6523         pci_write_config(dev, PCI_CFG_RETRY_TIMEOUT, 0x00, 1);
6524
6525         if (!sc->sc_attached)
6526                 return 0;
6527
6528         iwm_init_task(device_get_softc(dev));
6529
6530         IWM_LOCK(sc);
6531         if (sc->sc_flags & IWM_FLAG_SCANNING) {
6532                 sc->sc_flags &= ~IWM_FLAG_SCANNING;
6533                 do_reinit = 1;
6534         }
6535         IWM_UNLOCK(sc);
6536
6537         if (do_reinit)
6538                 ieee80211_resume_all(&sc->sc_ic);
6539
6540         return 0;
6541 }
6542
6543 static int
6544 iwm_suspend(device_t dev)
6545 {
6546         int do_stop = 0;
6547         struct iwm_softc *sc = device_get_softc(dev);
6548
6549         do_stop = !! (sc->sc_ic.ic_nrunning > 0);
6550
6551         if (!sc->sc_attached)
6552                 return (0);
6553
6554         ieee80211_suspend_all(&sc->sc_ic);
6555
6556         if (do_stop) {
6557                 IWM_LOCK(sc);
6558                 iwm_stop(sc);
6559                 sc->sc_flags |= IWM_FLAG_SCANNING;
6560                 IWM_UNLOCK(sc);
6561         }
6562
6563         return (0);
6564 }
6565
6566 static int
6567 iwm_detach_local(struct iwm_softc *sc, int do_net80211)
6568 {
6569         struct iwm_fw_info *fw = &sc->sc_fw;
6570         device_t dev = sc->sc_dev;
6571         int i;
6572
6573         if (!sc->sc_attached)
6574                 return 0;
6575         sc->sc_attached = 0;
6576         if (do_net80211) {
6577                 ieee80211_draintask(&sc->sc_ic, &sc->sc_es_task);
6578         }
6579         iwm_stop_device(sc);
6580         if (do_net80211) {
6581                 IWM_LOCK(sc);
6582                 iwm_xmit_queue_drain(sc);
6583                 IWM_UNLOCK(sc);
6584                 ieee80211_ifdetach(&sc->sc_ic);
6585         }
6586         callout_drain(&sc->sc_led_blink_to);
6587         callout_drain(&sc->sc_watchdog_to);
6588
6589         iwm_phy_db_free(sc->sc_phy_db);
6590         sc->sc_phy_db = NULL;
6591
6592         iwm_free_nvm_data(sc->nvm_data);
6593
6594         /* Free descriptor rings */
6595         iwm_free_rx_ring(sc, &sc->rxq);
6596         for (i = 0; i < nitems(sc->txq); i++)
6597                 iwm_free_tx_ring(sc, &sc->txq[i]);
6598
6599         /* Free firmware */
6600         if (fw->fw_fp != NULL)
6601                 iwm_fw_info_free(fw);
6602
6603         /* Free scheduler */
6604         iwm_dma_contig_free(&sc->sched_dma);
6605         iwm_dma_contig_free(&sc->ict_dma);
6606         iwm_dma_contig_free(&sc->kw_dma);
6607         iwm_dma_contig_free(&sc->fw_dma);
6608
6609         iwm_free_fw_paging(sc);
6610
6611         /* Finished with the hardware - detach things */
6612         iwm_pci_detach(dev);
6613
6614         if (sc->sc_notif_wait != NULL) {
6615                 iwm_notification_wait_free(sc->sc_notif_wait);
6616                 sc->sc_notif_wait = NULL;
6617         }
6618
6619         IWM_LOCK_DESTROY(sc);
6620
6621         return (0);
6622 }
6623
6624 static int
6625 iwm_detach(device_t dev)
6626 {
6627         struct iwm_softc *sc = device_get_softc(dev);
6628
6629         return (iwm_detach_local(sc, 1));
6630 }
6631
6632 static device_method_t iwm_pci_methods[] = {
6633         /* Device interface */
6634         DEVMETHOD(device_probe,         iwm_probe),
6635         DEVMETHOD(device_attach,        iwm_attach),
6636         DEVMETHOD(device_detach,        iwm_detach),
6637         DEVMETHOD(device_suspend,       iwm_suspend),
6638         DEVMETHOD(device_resume,        iwm_resume),
6639
6640         DEVMETHOD_END
6641 };
6642
6643 static driver_t iwm_pci_driver = {
6644         "iwm",
6645         iwm_pci_methods,
6646         sizeof (struct iwm_softc)
6647 };
6648
6649 static devclass_t iwm_devclass;
6650
6651 DRIVER_MODULE(iwm, pci, iwm_pci_driver, iwm_devclass, NULL, NULL);
6652 MODULE_PNP_INFO("U16:device;P:#;T:vendor=0x8086", pci, iwm_pci_driver,
6653     iwm_devices, nitems(iwm_devices));
6654 MODULE_DEPEND(iwm, firmware, 1, 1, 1);
6655 MODULE_DEPEND(iwm, pci, 1, 1, 1);
6656 MODULE_DEPEND(iwm, wlan, 1, 1, 1);