sys/dev/iwm/if_iwm.c

   1 /*      $OpenBSD: if_iwm.c,v 1.167 2017/04/04 00:40:52 claudio Exp $    */
   2
   3 /*
   4  * Copyright (c) 2014 genua mbh <info@genua.de>
   5  * Copyright (c) 2014 Fixup Software Ltd.
   6  *
   7  * Permission to use, copy, modify, and distribute this software for any
   8  * purpose with or without fee is hereby granted, provided that the above
   9  * copyright notice and this permission notice appear in all copies.
  10  *
  11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  18  */
  19
  20 /*-
  21  * Based on BSD-licensed source modules in the Linux iwlwifi driver,
  22  * which were used as the reference documentation for this implementation.
  23  *
  24  * Driver version we are currently based off of is
  25  * Linux 3.14.3 (tag id a2df521e42b1d9a23f620ac79dbfe8655a8391dd)
  26  *
  27  ***********************************************************************
  28  *
  29  * This file is provided under a dual BSD/GPLv2 license.  When using or
  30  * redistributing this file, you may do so under either license.
  31  *
  32  * GPL LICENSE SUMMARY
  33  *
  34  * Copyright(c) 2007 - 2013 Intel Corporation. All rights reserved.
  35  *
  36  * This program is free software; you can redistribute it and/or modify
  37  * it under the terms of version 2 of the GNU General Public License as
  38  * published by the Free Software Foundation.
  39  *
  40  * This program is distributed in the hope that it will be useful, but
  41  * WITHOUT ANY WARRANTY; without even the implied warranty of
  42  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  43  * General Public License for more details.
  44  *
  45  * You should have received a copy of the GNU General Public License
  46  * along with this program; if not, write to the Free Software
  47  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
  48  * USA
  49  *
  50  * The full GNU General Public License is included in this distribution
  51  * in the file called COPYING.
  52  *
  53  * Contact Information:
  54  *  Intel Linux Wireless <ilw@linux.intel.com>
  55  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  56  *
  57  *
  58  * BSD LICENSE
  59  *
  60  * Copyright(c) 2005 - 2013 Intel Corporation. All rights reserved.
  61  * All rights reserved.
  62  *
  63  * Redistribution and use in source and binary forms, with or without
  64  * modification, are permitted provided that the following conditions
  65  * are met:
  66  *
  67  *  * Redistributions of source code must retain the above copyright
  68  *    notice, this list of conditions and the following disclaimer.
  69  *  * Redistributions in binary form must reproduce the above copyright
  70  *    notice, this list of conditions and the following disclaimer in
  71  *    the documentation and/or other materials provided with the
  72  *    distribution.
  73  *  * Neither the name Intel Corporation nor the names of its
  74  *    contributors may be used to endorse or promote products derived
  75  *    from this software without specific prior written permission.
  76  *
  77  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  78  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  79  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  80  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  81  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  82  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  83  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  84  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  85  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  86  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  87  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  88  */
  89
  90 /*-
  91  * Copyright (c) 2007-2010 Damien Bergamini <damien.bergamini@free.fr>
  92  *
  93  * Permission to use, copy, modify, and distribute this software for any
  94  * purpose with or without fee is hereby granted, provided that the above
  95  * copyright notice and this permission notice appear in all copies.
  96  *
  97  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  98  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  99  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 100  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 101  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 102  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 103  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 104  */
 105 #include <sys/cdefs.h>
 106 __FBSDID("$FreeBSD$");
 107
 108 #include "opt_wlan.h"
 109 #include "opt_iwm.h"
 110
 111 #include <sys/param.h>
 112 #include <sys/bus.h>
 113 #include <sys/conf.h>
 114 #include <sys/endian.h>
 115 #include <sys/firmware.h>
 116 #include <sys/kernel.h>
 117 #include <sys/malloc.h>
 118 #include <sys/mbuf.h>
 119 #include <sys/mutex.h>
 120 #include <sys/module.h>
 121 #include <sys/proc.h>
 122 #include <sys/rman.h>
 123 #include <sys/socket.h>
 124 #include <sys/sockio.h>
 125 #include <sys/sysctl.h>
 126 #include <sys/linker.h>
 127
 128 #include <machine/bus.h>
 129 #include <machine/endian.h>
 130 #include <machine/resource.h>
 131
 132 #include <dev/pci/pcivar.h>
 133 #include <dev/pci/pcireg.h>
 134
 135 #include <net/bpf.h>
 136
 137 #include <net/if.h>
 138 #include <net/if_var.h>
 139 #include <net/if_arp.h>
 140 #include <net/if_dl.h>
 141 #include <net/if_media.h>
 142 #include <net/if_types.h>
 143
 144 #include <netinet/in.h>
 145 #include <netinet/in_systm.h>
 146 #include <netinet/if_ether.h>
 147 #include <netinet/ip.h>
 148
 149 #include <net80211/ieee80211_var.h>
 150 #include <net80211/ieee80211_regdomain.h>
 151 #include <net80211/ieee80211_ratectl.h>
 152 #include <net80211/ieee80211_radiotap.h>
 153
 154 #include <dev/iwm/if_iwmreg.h>
 155 #include <dev/iwm/if_iwmvar.h>
 156 #include <dev/iwm/if_iwm_config.h>
 157 #include <dev/iwm/if_iwm_debug.h>
 158 #include <dev/iwm/if_iwm_notif_wait.h>
 159 #include <dev/iwm/if_iwm_util.h>
 160 #include <dev/iwm/if_iwm_binding.h>
 161 #include <dev/iwm/if_iwm_phy_db.h>
 162 #include <dev/iwm/if_iwm_mac_ctxt.h>
 163 #include <dev/iwm/if_iwm_phy_ctxt.h>
 164 #include <dev/iwm/if_iwm_time_event.h>
 165 #include <dev/iwm/if_iwm_power.h>
 166 #include <dev/iwm/if_iwm_scan.h>
 167 #include <dev/iwm/if_iwm_sf.h>
 168 #include <dev/iwm/if_iwm_sta.h>
 169
 170 #include <dev/iwm/if_iwm_pcie_trans.h>
 171 #include <dev/iwm/if_iwm_led.h>
 172 #include <dev/iwm/if_iwm_fw.h>
 173
 174 /* From DragonflyBSD */
 175 #define mtodoff(m, t, off)      ((t)((m)->m_data + (off)))
 176
 177 const uint8_t iwm_nvm_channels[] = {
 178         /* 2.4 GHz */
 179         1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
 180         /* 5 GHz */
 181         36, 40, 44, 48, 52, 56, 60, 64,
 182         100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144,
 183         149, 153, 157, 161, 165
 184 };
 185 _Static_assert(nitems(iwm_nvm_channels) <= IWM_NUM_CHANNELS,
 186     "IWM_NUM_CHANNELS is too small");
 187
 188 const uint8_t iwm_nvm_channels_8000[] = {
 189         /* 2.4 GHz */
 190         1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
 191         /* 5 GHz */
 192         36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92,
 193         96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144,
 194         149, 153, 157, 161, 165, 169, 173, 177, 181
 195 };
 196 _Static_assert(nitems(iwm_nvm_channels_8000) <= IWM_NUM_CHANNELS_8000,
 197     "IWM_NUM_CHANNELS_8000 is too small");
 198
 199 #define IWM_NUM_2GHZ_CHANNELS   14
 200 #define IWM_N_HW_ADDR_MASK      0xF
 201
 202 /*
 203  * XXX For now, there's simply a fixed set of rate table entries
 204  * that are populated.
 205  */
 206 const struct iwm_rate {
 207         uint8_t rate;
 208         uint8_t plcp;
 209 } iwm_rates[] = {
 210         {   2,  IWM_RATE_1M_PLCP  },
 211         {   4,  IWM_RATE_2M_PLCP  },
 212         {  11,  IWM_RATE_5M_PLCP  },
 213         {  22,  IWM_RATE_11M_PLCP },
 214         {  12,  IWM_RATE_6M_PLCP  },
 215         {  18,  IWM_RATE_9M_PLCP  },
 216         {  24,  IWM_RATE_12M_PLCP },
 217         {  36,  IWM_RATE_18M_PLCP },
 218         {  48,  IWM_RATE_24M_PLCP },
 219         {  72,  IWM_RATE_36M_PLCP },
 220         {  96,  IWM_RATE_48M_PLCP },
 221         { 108,  IWM_RATE_54M_PLCP },
 222 };
 223 #define IWM_RIDX_CCK    0
 224 #define IWM_RIDX_OFDM   4
 225 #define IWM_RIDX_MAX    (nitems(iwm_rates)-1)
 226 #define IWM_RIDX_IS_CCK(_i_) ((_i_) < IWM_RIDX_OFDM)
 227 #define IWM_RIDX_IS_OFDM(_i_) ((_i_) >= IWM_RIDX_OFDM)
 228
 229 struct iwm_nvm_section {
 230         uint16_t length;
 231         uint8_t *data;
 232 };
 233
 234 #define IWM_MVM_UCODE_ALIVE_TIMEOUT     hz
 235 #define IWM_MVM_UCODE_CALIB_TIMEOUT     (2*hz)
 236
 237 struct iwm_mvm_alive_data {
 238         int valid;
 239         uint32_t scd_base_addr;
 240 };
 241
 242 static int      iwm_store_cscheme(struct iwm_softc *, const uint8_t *, size_t);
 243 static int      iwm_firmware_store_section(struct iwm_softc *,
 244                                            enum iwm_ucode_type,
 245                                            const uint8_t *, size_t);
 246 static int      iwm_set_default_calib(struct iwm_softc *, const void *);
 247 static void     iwm_fw_info_free(struct iwm_fw_info *);
 248 static int      iwm_read_firmware(struct iwm_softc *);
 249 static int      iwm_alloc_fwmem(struct iwm_softc *);
 250 static int      iwm_alloc_sched(struct iwm_softc *);
 251 static int      iwm_alloc_kw(struct iwm_softc *);
 252 static int      iwm_alloc_ict(struct iwm_softc *);
 253 static int      iwm_alloc_rx_ring(struct iwm_softc *, struct iwm_rx_ring *);
 254 static void     iwm_reset_rx_ring(struct iwm_softc *, struct iwm_rx_ring *);
 255 static void     iwm_free_rx_ring(struct iwm_softc *, struct iwm_rx_ring *);
 256 static int      iwm_alloc_tx_ring(struct iwm_softc *, struct iwm_tx_ring *,
 257                                   int);
 258 static void     iwm_reset_tx_ring(struct iwm_softc *, struct iwm_tx_ring *);
 259 static void     iwm_free_tx_ring(struct iwm_softc *, struct iwm_tx_ring *);
 260 static void     iwm_enable_interrupts(struct iwm_softc *);
 261 static void     iwm_restore_interrupts(struct iwm_softc *);
 262 static void     iwm_disable_interrupts(struct iwm_softc *);
 263 static void     iwm_ict_reset(struct iwm_softc *);
 264 static int      iwm_allow_mcast(struct ieee80211vap *, struct iwm_softc *);
 265 static void     iwm_stop_device(struct iwm_softc *);
 266 static void     iwm_mvm_nic_config(struct iwm_softc *);
 267 static int      iwm_nic_rx_init(struct iwm_softc *);
 268 static int      iwm_nic_tx_init(struct iwm_softc *);
 269 static int      iwm_nic_init(struct iwm_softc *);
 270 static int      iwm_trans_pcie_fw_alive(struct iwm_softc *, uint32_t);
 271 static int      iwm_nvm_read_chunk(struct iwm_softc *, uint16_t, uint16_t,
 272                                    uint16_t, uint8_t *, uint16_t *);
 273 static int      iwm_nvm_read_section(struct iwm_softc *, uint16_t, uint8_t *,
 274                                      uint16_t *, uint32_t);
 275 static uint32_t iwm_eeprom_channel_flags(uint16_t);
 276 static void     iwm_add_channel_band(struct iwm_softc *,
 277                     struct ieee80211_channel[], int, int *, int, size_t,
 278                     const uint8_t[]);
 279 static void     iwm_init_channel_map(struct ieee80211com *, int, int *,
 280                     struct ieee80211_channel[]);
 281 static struct iwm_nvm_data *
 282         iwm_parse_nvm_data(struct iwm_softc *, const uint16_t *,
 283                            const uint16_t *, const uint16_t *,
 284                            const uint16_t *, const uint16_t *,
 285                            const uint16_t *);
 286 static void     iwm_free_nvm_data(struct iwm_nvm_data *);
 287 static void     iwm_set_hw_address_family_8000(struct iwm_softc *,
 288                                                struct iwm_nvm_data *,
 289                                                const uint16_t *,
 290                                                const uint16_t *);
 291 static int      iwm_get_sku(const struct iwm_softc *, const uint16_t *,
 292                             const uint16_t *);
 293 static int      iwm_get_nvm_version(const struct iwm_softc *, const uint16_t *);
 294 static int      iwm_get_radio_cfg(const struct iwm_softc *, const uint16_t *,
 295                                   const uint16_t *);
 296 static int      iwm_get_n_hw_addrs(const struct iwm_softc *,
 297                                    const uint16_t *);
 298 static void     iwm_set_radio_cfg(const struct iwm_softc *,
 299                                   struct iwm_nvm_data *, uint32_t);
 300 static struct iwm_nvm_data *
 301         iwm_parse_nvm_sections(struct iwm_softc *, struct iwm_nvm_section *);
 302 static int      iwm_nvm_init(struct iwm_softc *);
 303 static int      iwm_pcie_load_section(struct iwm_softc *, uint8_t,
 304                                       const struct iwm_fw_desc *);
 305 static int      iwm_pcie_load_firmware_chunk(struct iwm_softc *, uint32_t,
 306                                              bus_addr_t, uint32_t);
 307 static int      iwm_pcie_load_cpu_sections_8000(struct iwm_softc *sc,
 308                                                 const struct iwm_fw_sects *,
 309                                                 int, int *);
 310 static int      iwm_pcie_load_cpu_sections(struct iwm_softc *,
 311                                            const struct iwm_fw_sects *,
 312                                            int, int *);
 313 static int      iwm_pcie_load_given_ucode_8000(struct iwm_softc *,
 314                                                const struct iwm_fw_sects *);
 315 static int      iwm_pcie_load_given_ucode(struct iwm_softc *,
 316                                           const struct iwm_fw_sects *);
 317 static int      iwm_start_fw(struct iwm_softc *, const struct iwm_fw_sects *);
 318 static int      iwm_send_tx_ant_cfg(struct iwm_softc *, uint8_t);
 319 static int      iwm_send_phy_cfg_cmd(struct iwm_softc *);
 320 static int      iwm_mvm_load_ucode_wait_alive(struct iwm_softc *,
 321                                               enum iwm_ucode_type);
 322 static int      iwm_run_init_mvm_ucode(struct iwm_softc *, int);
 323 static int      iwm_rx_addbuf(struct iwm_softc *, int, int);
 324 static int      iwm_mvm_get_signal_strength(struct iwm_softc *,
 325                                             struct iwm_rx_phy_info *);
 326 static void     iwm_mvm_rx_rx_phy_cmd(struct iwm_softc *,
 327                                       struct iwm_rx_packet *);
 328 static int      iwm_get_noise(struct iwm_softc *,
 329                     const struct iwm_mvm_statistics_rx_non_phy *);
 330 static void     iwm_mvm_handle_rx_statistics(struct iwm_softc *,
 331                     struct iwm_rx_packet *);
 332 static boolean_t iwm_mvm_rx_rx_mpdu(struct iwm_softc *, struct mbuf *,
 333                                     uint32_t, boolean_t);
 334 static int      iwm_mvm_rx_tx_cmd_single(struct iwm_softc *,
 335                                          struct iwm_rx_packet *,
 336                                          struct iwm_node *);
 337 static void     iwm_mvm_rx_tx_cmd(struct iwm_softc *, struct iwm_rx_packet *);
 338 static void     iwm_cmd_done(struct iwm_softc *, struct iwm_rx_packet *);
 339 #if 0
 340 static void     iwm_update_sched(struct iwm_softc *, int, int, uint8_t,
 341                                  uint16_t);
 342 #endif
 343 static const struct iwm_rate *
 344         iwm_tx_fill_cmd(struct iwm_softc *, struct iwm_node *,
 345                         struct mbuf *, struct iwm_tx_cmd *);
 346 static int      iwm_tx(struct iwm_softc *, struct mbuf *,
 347                        struct ieee80211_node *, int);
 348 static int      iwm_raw_xmit(struct ieee80211_node *, struct mbuf *,
 349                              const struct ieee80211_bpf_params *);
 350 static int      iwm_mvm_update_quotas(struct iwm_softc *, struct iwm_vap *);
 351 static int      iwm_auth(struct ieee80211vap *, struct iwm_softc *);
 352 static int      iwm_release(struct iwm_softc *, struct iwm_node *);
 353 static struct ieee80211_node *
 354                 iwm_node_alloc(struct ieee80211vap *,
 355                                const uint8_t[IEEE80211_ADDR_LEN]);
 356 static uint8_t  iwm_rate_from_ucode_rate(uint32_t);
 357 static int      iwm_rate2ridx(struct iwm_softc *, uint8_t);
 358 static void     iwm_setrates(struct iwm_softc *, struct iwm_node *, int);
 359 static int      iwm_media_change(struct ifnet *);
 360 static int      iwm_newstate(struct ieee80211vap *, enum ieee80211_state, int);
 361 static void     iwm_endscan_cb(void *, int);
 362 static int      iwm_send_bt_init_conf(struct iwm_softc *);
 363 static boolean_t iwm_mvm_is_lar_supported(struct iwm_softc *);
 364 static boolean_t iwm_mvm_is_wifi_mcc_supported(struct iwm_softc *);
 365 static int      iwm_send_update_mcc_cmd(struct iwm_softc *, const char *);
 366 static void     iwm_mvm_tt_tx_backoff(struct iwm_softc *, uint32_t);
 367 static int      iwm_init_hw(struct iwm_softc *);
 368 static void     iwm_init(struct iwm_softc *);
 369 static void     iwm_start(struct iwm_softc *);
 370 static void     iwm_stop(struct iwm_softc *);
 371 static void     iwm_watchdog(void *);
 372 static void     iwm_parent(struct ieee80211com *);
 373 #ifdef IWM_DEBUG
 374 static const char *
 375                 iwm_desc_lookup(uint32_t);
 376 static void     iwm_nic_error(struct iwm_softc *);
 377 static void     iwm_nic_umac_error(struct iwm_softc *);
 378 #endif
 379 static void     iwm_handle_rxb(struct iwm_softc *, struct mbuf *);
 380 static void     iwm_notif_intr(struct iwm_softc *);
 381 static void     iwm_intr(void *);
 382 static int      iwm_attach(device_t);
 383 static int      iwm_is_valid_ether_addr(uint8_t *);
 384 static void     iwm_preinit(void *);
 385 static int      iwm_detach_local(struct iwm_softc *sc, int);
 386 static void     iwm_init_task(void *);
 387 static void     iwm_radiotap_attach(struct iwm_softc *);
 388 static struct ieee80211vap *
 389                 iwm_vap_create(struct ieee80211com *,
 390                                const char [IFNAMSIZ], int,
 391                                enum ieee80211_opmode, int,
 392                                const uint8_t [IEEE80211_ADDR_LEN],
 393                                const uint8_t [IEEE80211_ADDR_LEN]);
 394 static void     iwm_vap_delete(struct ieee80211vap *);
 395 static void     iwm_xmit_queue_drain(struct iwm_softc *);
 396 static void     iwm_scan_start(struct ieee80211com *);
 397 static void     iwm_scan_end(struct ieee80211com *);
 398 static void     iwm_update_mcast(struct ieee80211com *);
 399 static void     iwm_set_channel(struct ieee80211com *);
 400 static void     iwm_scan_curchan(struct ieee80211_scan_state *, unsigned long);
 401 static void     iwm_scan_mindwell(struct ieee80211_scan_state *);
 402 static int      iwm_detach(device_t);
 403
 404 static int      iwm_lar_disable = 0;
 405 TUNABLE_INT("hw.iwm.lar.disable", &iwm_lar_disable);
 406
 407 /*
 408  * Firmware parser.
 409  */
 410
 411 static int
 412 iwm_store_cscheme(struct iwm_softc *sc, const uint8_t *data, size_t dlen)
 413 {
 414         const struct iwm_fw_cscheme_list *l = (const void *)data;
 415
 416         if (dlen < sizeof(*l) ||
 417             dlen < sizeof(l->size) + l->size * sizeof(*l->cs))
 418                 return EINVAL;
 419
 420         /* we don't actually store anything for now, always use s/w crypto */
 421
 422         return 0;
 423 }
 424
 425 static int
 426 iwm_firmware_store_section(struct iwm_softc *sc,
 427     enum iwm_ucode_type type, const uint8_t *data, size_t dlen)
 428 {
 429         struct iwm_fw_sects *fws;
 430         struct iwm_fw_desc *fwone;
 431
 432         if (type >= IWM_UCODE_TYPE_MAX)
 433                 return EINVAL;
 434         if (dlen < sizeof(uint32_t))
 435                 return EINVAL;
 436
 437         fws = &sc->sc_fw.fw_sects[type];
 438         if (fws->fw_count >= IWM_UCODE_SECTION_MAX)
 439                 return EINVAL;
 440
 441         fwone = &fws->fw_sect[fws->fw_count];
 442
 443         /* first 32bit are device load offset */
 444         memcpy(&fwone->offset, data, sizeof(uint32_t));
 445
 446         /* rest is data */
 447         fwone->data = data + sizeof(uint32_t);
 448         fwone->len = dlen - sizeof(uint32_t);
 449
 450         fws->fw_count++;
 451
 452         return 0;
 453 }
 454
 455 #define IWM_DEFAULT_SCAN_CHANNELS 40
 456
 457 /* iwlwifi: iwl-drv.c */
 458 struct iwm_tlv_calib_data {
 459         uint32_t ucode_type;
 460         struct iwm_tlv_calib_ctrl calib;
 461 } __packed;
 462
 463 static int
 464 iwm_set_default_calib(struct iwm_softc *sc, const void *data)
 465 {
 466         const struct iwm_tlv_calib_data *def_calib = data;
 467         uint32_t ucode_type = le32toh(def_calib->ucode_type);
 468
 469         if (ucode_type >= IWM_UCODE_TYPE_MAX) {
 470                 device_printf(sc->sc_dev,
 471                     "Wrong ucode_type %u for default "
 472                     "calibration.\n", ucode_type);
 473                 return EINVAL;
 474         }
 475
 476         sc->sc_default_calib[ucode_type].flow_trigger =
 477             def_calib->calib.flow_trigger;
 478         sc->sc_default_calib[ucode_type].event_trigger =
 479             def_calib->calib.event_trigger;
 480
 481         return 0;
 482 }
 483
 484 static int
 485 iwm_set_ucode_api_flags(struct iwm_softc *sc, const uint8_t *data,
 486                         struct iwm_ucode_capabilities *capa)
 487 {
 488         const struct iwm_ucode_api *ucode_api = (const void *)data;
 489         uint32_t api_index = le32toh(ucode_api->api_index);
 490         uint32_t api_flags = le32toh(ucode_api->api_flags);
 491         int i;
 492
 493         if (api_index >= howmany(IWM_NUM_UCODE_TLV_API, 32)) {
 494                 device_printf(sc->sc_dev,
 495                     "api flags index %d larger than supported by driver\n",
 496                     api_index);
 497                 /* don't return an error so we can load FW that has more bits */
 498                 return 0;
 499         }
 500
 501         for (i = 0; i < 32; i++) {
 502                 if (api_flags & (1U << i))
 503                         setbit(capa->enabled_api, i + 32 * api_index);
 504         }
 505
 506         return 0;
 507 }
 508
 509 static int
 510 iwm_set_ucode_capabilities(struct iwm_softc *sc, const uint8_t *data,
 511                            struct iwm_ucode_capabilities *capa)
 512 {
 513         const struct iwm_ucode_capa *ucode_capa = (const void *)data;
 514         uint32_t api_index = le32toh(ucode_capa->api_index);
 515         uint32_t api_flags = le32toh(ucode_capa->api_capa);
 516         int i;
 517
 518         if (api_index >= howmany(IWM_NUM_UCODE_TLV_CAPA, 32)) {
 519                 device_printf(sc->sc_dev,
 520                     "capa flags index %d larger than supported by driver\n",
 521                     api_index);
 522                 /* don't return an error so we can load FW that has more bits */
 523                 return 0;
 524         }
 525
 526         for (i = 0; i < 32; i++) {
 527                 if (api_flags & (1U << i))
 528                         setbit(capa->enabled_capa, i + 32 * api_index);
 529         }
 530
 531         return 0;
 532 }
 533
 534 static void
 535 iwm_fw_info_free(struct iwm_fw_info *fw)
 536 {
 537         firmware_put(fw->fw_fp, FIRMWARE_UNLOAD);
 538         fw->fw_fp = NULL;
 539         memset(fw->fw_sects, 0, sizeof(fw->fw_sects));
 540 }
 541
 542 static int
 543 iwm_read_firmware(struct iwm_softc *sc)
 544 {
 545         struct iwm_fw_info *fw = &sc->sc_fw;
 546         const struct iwm_tlv_ucode_header *uhdr;
 547         const struct iwm_ucode_tlv *tlv;
 548         struct iwm_ucode_capabilities *capa = &sc->ucode_capa;
 549         enum iwm_ucode_tlv_type tlv_type;
 550         const struct firmware *fwp;
 551         const uint8_t *data;
 552         uint32_t tlv_len;
 553         uint32_t usniffer_img;
 554         const uint8_t *tlv_data;
 555         uint32_t paging_mem_size;
 556         int num_of_cpus;
 557         int error = 0;
 558         size_t len;
 559
 560         /*
 561          * Load firmware into driver memory.
 562          * fw_fp will be set.
 563          */
 564         fwp = firmware_get(sc->cfg->fw_name);
 565         if (fwp == NULL) {
 566                 device_printf(sc->sc_dev,
 567                     "could not read firmware %s (error %d)\n",
 568                     sc->cfg->fw_name, error);
 569                 goto out;
 570         }
 571         fw->fw_fp = fwp;
 572
 573         /* (Re-)Initialize default values. */
 574         capa->flags = 0;
 575         capa->max_probe_length = IWM_DEFAULT_MAX_PROBE_LENGTH;
 576         capa->n_scan_channels = IWM_DEFAULT_SCAN_CHANNELS;
 577         memset(capa->enabled_capa, 0, sizeof(capa->enabled_capa));
 578         memset(capa->enabled_api, 0, sizeof(capa->enabled_api));
 579         memset(sc->sc_fw_mcc, 0, sizeof(sc->sc_fw_mcc));
 580
 581         /*
 582          * Parse firmware contents
 583          */
 584
 585         uhdr = (const void *)fw->fw_fp->data;
 586         if (*(const uint32_t *)fw->fw_fp->data != 0
 587             || le32toh(uhdr->magic) != IWM_TLV_UCODE_MAGIC) {
 588                 device_printf(sc->sc_dev, "invalid firmware %s\n",
 589                     sc->cfg->fw_name);
 590                 error = EINVAL;
 591                 goto out;
 592         }
 593
 594         snprintf(sc->sc_fwver, sizeof(sc->sc_fwver), "%u.%u (API ver %u)",
 595             IWM_UCODE_MAJOR(le32toh(uhdr->ver)),
 596             IWM_UCODE_MINOR(le32toh(uhdr->ver)),
 597             IWM_UCODE_API(le32toh(uhdr->ver)));
 598         data = uhdr->data;
 599         len = fw->fw_fp->datasize - sizeof(*uhdr);
 600
 601         while (len >= sizeof(*tlv)) {
 602                 len -= sizeof(*tlv);
 603                 tlv = (const void *)data;
 604
 605                 tlv_len = le32toh(tlv->length);
 606                 tlv_type = le32toh(tlv->type);
 607                 tlv_data = tlv->data;
 608
 609                 if (len < tlv_len) {
 610                         device_printf(sc->sc_dev,
 611                             "firmware too short: %zu bytes\n",
 612                             len);
 613                         error = EINVAL;
 614                         goto parse_out;
 615                 }
 616                 len -= roundup2(tlv_len, 4);
 617                 data += sizeof(*tlv) + roundup2(tlv_len, 4);
 618
 619                 switch ((int)tlv_type) {
 620                 case IWM_UCODE_TLV_PROBE_MAX_LEN:
 621                         if (tlv_len != sizeof(uint32_t)) {
 622                                 device_printf(sc->sc_dev,
 623                                     "%s: PROBE_MAX_LEN (%u) != sizeof(uint32_t)\n",
 624                                     __func__, tlv_len);
 625                                 error = EINVAL;
 626                                 goto parse_out;
 627                         }
 628                         capa->max_probe_length =
 629                             le32_to_cpup((const uint32_t *)tlv_data);
 630                         /* limit it to something sensible */
 631                         if (capa->max_probe_length >
 632                             IWM_SCAN_OFFLOAD_PROBE_REQ_SIZE) {
 633                                 IWM_DPRINTF(sc, IWM_DEBUG_FIRMWARE_TLV,
 634                                     "%s: IWM_UCODE_TLV_PROBE_MAX_LEN "
 635                                     "ridiculous\n", __func__);
 636                                 error = EINVAL;
 637                                 goto parse_out;
 638                         }
 639                         break;
 640                 case IWM_UCODE_TLV_PAN:
 641                         if (tlv_len) {
 642                                 device_printf(sc->sc_dev,
 643                                     "%s: IWM_UCODE_TLV_PAN: tlv_len (%u) > 0\n",
 644                                     __func__, tlv_len);
 645                                 error = EINVAL;
 646                                 goto parse_out;
 647                         }
 648                         capa->flags |= IWM_UCODE_TLV_FLAGS_PAN;
 649                         break;
 650                 case IWM_UCODE_TLV_FLAGS:
 651                         if (tlv_len < sizeof(uint32_t)) {
 652                                 device_printf(sc->sc_dev,
 653                                     "%s: IWM_UCODE_TLV_FLAGS: tlv_len (%u) < sizeof(uint32_t)\n",
 654                                     __func__, tlv_len);
 655                                 error = EINVAL;
 656                                 goto parse_out;
 657                         }
 658                         if (tlv_len % sizeof(uint32_t)) {
 659                                 device_printf(sc->sc_dev,
 660                                     "%s: IWM_UCODE_TLV_FLAGS: tlv_len (%u) %% sizeof(uint32_t)\n",
 661                                     __func__, tlv_len);
 662                                 error = EINVAL;
 663                                 goto parse_out;
 664                         }
 665                         /*
 666                          * Apparently there can be many flags, but Linux driver
 667                          * parses only the first one, and so do we.
 668                          *
 669                          * XXX: why does this override IWM_UCODE_TLV_PAN?
 670                          * Intentional or a bug?  Observations from
 671                          * current firmware file:
 672                          *  1) TLV_PAN is parsed first
 673                          *  2) TLV_FLAGS contains TLV_FLAGS_PAN
 674                          * ==> this resets TLV_PAN to itself... hnnnk
 675                          */
 676                         capa->flags = le32_to_cpup((const uint32_t *)tlv_data);
 677                         break;
 678                 case IWM_UCODE_TLV_CSCHEME:
 679                         if ((error = iwm_store_cscheme(sc,
 680                             tlv_data, tlv_len)) != 0) {
 681                                 device_printf(sc->sc_dev,
 682                                     "%s: iwm_store_cscheme(): returned %d\n",
 683                                     __func__, error);
 684                                 goto parse_out;
 685                         }
 686                         break;
 687                 case IWM_UCODE_TLV_NUM_OF_CPU:
 688                         if (tlv_len != sizeof(uint32_t)) {
 689                                 device_printf(sc->sc_dev,
 690                                     "%s: IWM_UCODE_TLV_NUM_OF_CPU: tlv_len (%u) != sizeof(uint32_t)\n",
 691                                     __func__, tlv_len);
 692                                 error = EINVAL;
 693                                 goto parse_out;
 694                         }
 695                         num_of_cpus = le32_to_cpup((const uint32_t *)tlv_data);
 696                         if (num_of_cpus == 2) {
 697                                 fw->fw_sects[IWM_UCODE_REGULAR].is_dual_cpus =
 698                                         TRUE;
 699                                 fw->fw_sects[IWM_UCODE_INIT].is_dual_cpus =
 700                                         TRUE;
 701                                 fw->fw_sects[IWM_UCODE_WOWLAN].is_dual_cpus =
 702                                         TRUE;
 703                         } else if ((num_of_cpus > 2) || (num_of_cpus < 1)) {
 704                                 device_printf(sc->sc_dev,
 705                                     "%s: Driver supports only 1 or 2 CPUs\n",
 706                                     __func__);
 707                                 error = EINVAL;
 708                                 goto parse_out;
 709                         }
 710                         break;
 711                 case IWM_UCODE_TLV_SEC_RT:
 712                         if ((error = iwm_firmware_store_section(sc,
 713                             IWM_UCODE_REGULAR, tlv_data, tlv_len)) != 0) {
 714                                 device_printf(sc->sc_dev,
 715                                     "%s: IWM_UCODE_REGULAR: iwm_firmware_store_section() failed; %d\n",
 716                                     __func__, error);
 717                                 goto parse_out;
 718                         }
 719                         break;
 720                 case IWM_UCODE_TLV_SEC_INIT:
 721                         if ((error = iwm_firmware_store_section(sc,
 722                             IWM_UCODE_INIT, tlv_data, tlv_len)) != 0) {
 723                                 device_printf(sc->sc_dev,
 724                                     "%s: IWM_UCODE_INIT: iwm_firmware_store_section() failed; %d\n",
 725                                     __func__, error);
 726                                 goto parse_out;
 727                         }
 728                         break;
 729                 case IWM_UCODE_TLV_SEC_WOWLAN:
 730                         if ((error = iwm_firmware_store_section(sc,
 731                             IWM_UCODE_WOWLAN, tlv_data, tlv_len)) != 0) {
 732                                 device_printf(sc->sc_dev,
 733                                     "%s: IWM_UCODE_WOWLAN: iwm_firmware_store_section() failed; %d\n",
 734                                     __func__, error);
 735                                 goto parse_out;
 736                         }
 737                         break;
 738                 case IWM_UCODE_TLV_DEF_CALIB:
 739                         if (tlv_len != sizeof(struct iwm_tlv_calib_data)) {
 740                                 device_printf(sc->sc_dev,
 741                                     "%s: IWM_UCODE_TLV_DEV_CALIB: tlv_len (%u) < sizeof(iwm_tlv_calib_data) (%zu)\n",
 742                                     __func__, tlv_len,
 743                                     sizeof(struct iwm_tlv_calib_data));
 744                                 error = EINVAL;
 745                                 goto parse_out;
 746                         }
 747                         if ((error = iwm_set_default_calib(sc, tlv_data)) != 0) {
 748                                 device_printf(sc->sc_dev,
 749                                     "%s: iwm_set_default_calib() failed: %d\n",
 750                                     __func__, error);
 751                                 goto parse_out;
 752                         }
 753                         break;
 754                 case IWM_UCODE_TLV_PHY_SKU:
 755                         if (tlv_len != sizeof(uint32_t)) {
 756                                 error = EINVAL;
 757                                 device_printf(sc->sc_dev,
 758                                     "%s: IWM_UCODE_TLV_PHY_SKU: tlv_len (%u) < sizeof(uint32_t)\n",
 759                                     __func__, tlv_len);
 760                                 goto parse_out;
 761                         }
 762                         sc->sc_fw.phy_config =
 763                             le32_to_cpup((const uint32_t *)tlv_data);
 764                         sc->sc_fw.valid_tx_ant = (sc->sc_fw.phy_config &
 765                                                   IWM_FW_PHY_CFG_TX_CHAIN) >>
 766                                                   IWM_FW_PHY_CFG_TX_CHAIN_POS;
 767                         sc->sc_fw.valid_rx_ant = (sc->sc_fw.phy_config &
 768                                                   IWM_FW_PHY_CFG_RX_CHAIN) >>
 769                                                   IWM_FW_PHY_CFG_RX_CHAIN_POS;
 770                         break;
 771
 772                 case IWM_UCODE_TLV_API_CHANGES_SET: {
 773                         if (tlv_len != sizeof(struct iwm_ucode_api)) {
 774                                 error = EINVAL;
 775                                 goto parse_out;
 776                         }
 777                         if (iwm_set_ucode_api_flags(sc, tlv_data, capa)) {
 778                                 error = EINVAL;
 779                                 goto parse_out;
 780                         }
 781                         break;
 782                 }
 783
 784                 case IWM_UCODE_TLV_ENABLED_CAPABILITIES: {
 785                         if (tlv_len != sizeof(struct iwm_ucode_capa)) {
 786                                 error = EINVAL;
 787                                 goto parse_out;
 788                         }
 789                         if (iwm_set_ucode_capabilities(sc, tlv_data, capa)) {
 790                                 error = EINVAL;
 791                                 goto parse_out;
 792                         }
 793                         break;
 794                 }
 795
 796                 case 48: /* undocumented TLV */
 797                 case IWM_UCODE_TLV_SDIO_ADMA_ADDR:
 798                 case IWM_UCODE_TLV_FW_GSCAN_CAPA:
 799                         /* ignore, not used by current driver */
 800                         break;
 801
 802                 case IWM_UCODE_TLV_SEC_RT_USNIFFER:
 803                         if ((error = iwm_firmware_store_section(sc,
 804                             IWM_UCODE_REGULAR_USNIFFER, tlv_data,
 805                             tlv_len)) != 0)
 806                                 goto parse_out;
 807                         break;
 808
 809                 case IWM_UCODE_TLV_PAGING:
 810                         if (tlv_len != sizeof(uint32_t)) {
 811                                 error = EINVAL;
 812                                 goto parse_out;
 813                         }
 814                         paging_mem_size = le32_to_cpup((const uint32_t *)tlv_data);
 815
 816                         IWM_DPRINTF(sc, IWM_DEBUG_FIRMWARE_TLV,
 817                             "%s: Paging: paging enabled (size = %u bytes)\n",
 818                             __func__, paging_mem_size);
 819                         if (paging_mem_size > IWM_MAX_PAGING_IMAGE_SIZE) {
 820                                 device_printf(sc->sc_dev,
 821                                         "%s: Paging: driver supports up to %u bytes for paging image\n",
 822                                         __func__, IWM_MAX_PAGING_IMAGE_SIZE);
 823                                 error = EINVAL;
 824                                 goto out;
 825                         }
 826                         if (paging_mem_size & (IWM_FW_PAGING_SIZE - 1)) {
 827                                 device_printf(sc->sc_dev,
 828                                     "%s: Paging: image isn't multiple %u\n",
 829                                     __func__, IWM_FW_PAGING_SIZE);
 830                                 error = EINVAL;
 831                                 goto out;
 832                         }
 833
 834                         sc->sc_fw.fw_sects[IWM_UCODE_REGULAR].paging_mem_size =
 835                             paging_mem_size;
 836                         usniffer_img = IWM_UCODE_REGULAR_USNIFFER;
 837                         sc->sc_fw.fw_sects[usniffer_img].paging_mem_size =
 838                             paging_mem_size;
 839                         break;
 840
 841                 case IWM_UCODE_TLV_N_SCAN_CHANNELS:
 842                         if (tlv_len != sizeof(uint32_t)) {
 843                                 error = EINVAL;
 844                                 goto parse_out;
 845                         }
 846                         capa->n_scan_channels =
 847                             le32_to_cpup((const uint32_t *)tlv_data);
 848                         break;
 849
 850                 case IWM_UCODE_TLV_FW_VERSION:
 851                         if (tlv_len != sizeof(uint32_t) * 3) {
 852                                 error = EINVAL;
 853                                 goto parse_out;
 854                         }
 855                         snprintf(sc->sc_fwver, sizeof(sc->sc_fwver),
 856                             "%d.%d.%d",
 857                             le32toh(((const uint32_t *)tlv_data)[0]),
 858                             le32toh(((const uint32_t *)tlv_data)[1]),
 859                             le32toh(((const uint32_t *)tlv_data)[2]));
 860                         break;
 861
 862                 case IWM_UCODE_TLV_FW_MEM_SEG:
 863                         break;
 864
 865                 default:
 866                         device_printf(sc->sc_dev,
 867                             "%s: unknown firmware section %d, abort\n",
 868                             __func__, tlv_type);
 869                         error = EINVAL;
 870                         goto parse_out;
 871                 }
 872         }
 873
 874         KASSERT(error == 0, ("unhandled error"));
 875
 876  parse_out:
 877         if (error) {
 878                 device_printf(sc->sc_dev, "firmware parse error %d, "
 879                     "section type %d\n", error, tlv_type);
 880         }
 881
 882  out:
 883         if (error) {
 884                 if (fw->fw_fp != NULL)
 885                         iwm_fw_info_free(fw);
 886         }
 887
 888         return error;
 889 }
 890
 891 /*
 892  * DMA resource routines
 893  */
 894
 895 /* fwmem is used to load firmware onto the card */
 896 static int
 897 iwm_alloc_fwmem(struct iwm_softc *sc)
 898 {
 899         /* Must be aligned on a 16-byte boundary. */
 900         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->fw_dma,
 901             IWM_FH_MEM_TB_MAX_LENGTH, 16);
 902 }
 903
 904 /* tx scheduler rings.  not used? */
 905 static int
 906 iwm_alloc_sched(struct iwm_softc *sc)
 907 {
 908         /* TX scheduler rings must be aligned on a 1KB boundary. */
 909         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->sched_dma,
 910             nitems(sc->txq) * sizeof(struct iwm_agn_scd_bc_tbl), 1024);
 911 }
 912
 913 /* keep-warm page is used internally by the card.  see iwl-fh.h for more info */
 914 static int
 915 iwm_alloc_kw(struct iwm_softc *sc)
 916 {
 917         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->kw_dma, 4096, 4096);
 918 }
 919
 920 /* interrupt cause table */
 921 static int
 922 iwm_alloc_ict(struct iwm_softc *sc)
 923 {
 924         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->ict_dma,
 925             IWM_ICT_SIZE, 1<<IWM_ICT_PADDR_SHIFT);
 926 }
 927
 928 static int
 929 iwm_alloc_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring)
 930 {
 931         bus_size_t size;
 932         int i, error;
 933
 934         ring->cur = 0;
 935
 936         /* Allocate RX descriptors (256-byte aligned). */
 937         size = IWM_RX_RING_COUNT * sizeof(uint32_t);
 938         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->desc_dma, size, 256);
 939         if (error != 0) {
 940                 device_printf(sc->sc_dev,
 941                     "could not allocate RX ring DMA memory\n");
 942                 goto fail;
 943         }
 944         ring->desc = ring->desc_dma.vaddr;
 945
 946         /* Allocate RX status area (16-byte aligned). */
 947         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->stat_dma,
 948             sizeof(*ring->stat), 16);
 949         if (error != 0) {
 950                 device_printf(sc->sc_dev,
 951                     "could not allocate RX status DMA memory\n");
 952                 goto fail;
 953         }
 954         ring->stat = ring->stat_dma.vaddr;
 955
 956         /* Create RX buffer DMA tag. */
 957         error = bus_dma_tag_create(sc->sc_dmat, 1, 0,
 958             BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
 959             IWM_RBUF_SIZE, 1, IWM_RBUF_SIZE, 0, NULL, NULL, &ring->data_dmat);
 960         if (error != 0) {
 961                 device_printf(sc->sc_dev,
 962                     "%s: could not create RX buf DMA tag, error %d\n",
 963                     __func__, error);
 964                 goto fail;
 965         }
 966
 967         /* Allocate spare bus_dmamap_t for iwm_rx_addbuf() */
 968         error = bus_dmamap_create(ring->data_dmat, 0, &ring->spare_map);
 969         if (error != 0) {
 970                 device_printf(sc->sc_dev,
 971                     "%s: could not create RX buf DMA map, error %d\n",
 972                     __func__, error);
 973                 goto fail;
 974         }
 975         /*
 976          * Allocate and map RX buffers.
 977          */
 978         for (i = 0; i < IWM_RX_RING_COUNT; i++) {
 979                 struct iwm_rx_data *data = &ring->data[i];
 980                 error = bus_dmamap_create(ring->data_dmat, 0, &data->map);
 981                 if (error != 0) {
 982                         device_printf(sc->sc_dev,
 983                             "%s: could not create RX buf DMA map, error %d\n",
 984                             __func__, error);
 985                         goto fail;
 986                 }
 987                 data->m = NULL;
 988
 989                 if ((error = iwm_rx_addbuf(sc, IWM_RBUF_SIZE, i)) != 0) {
 990                         goto fail;
 991                 }
 992         }
 993         return 0;
 994
 995 fail:   iwm_free_rx_ring(sc, ring);
 996         return error;
 997 }
 998
 999 static void
1000 iwm_reset_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring)
1001 {
1002         /* Reset the ring state */
1003         ring->cur = 0;
1004
1005         /*
1006          * The hw rx ring index in shared memory must also be cleared,
1007          * otherwise the discrepancy can cause reprocessing chaos.
1008          */
1009         if (sc->rxq.stat)
1010                 memset(sc->rxq.stat, 0, sizeof(*sc->rxq.stat));
1011 }
1012
1013 static void
1014 iwm_free_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring)
1015 {
1016         int i;
1017
1018         iwm_dma_contig_free(&ring->desc_dma);
1019         iwm_dma_contig_free(&ring->stat_dma);
1020
1021         for (i = 0; i < IWM_RX_RING_COUNT; i++) {
1022                 struct iwm_rx_data *data = &ring->data[i];
1023
1024                 if (data->m != NULL) {
1025                         bus_dmamap_sync(ring->data_dmat, data->map,
1026                             BUS_DMASYNC_POSTREAD);
1027                         bus_dmamap_unload(ring->data_dmat, data->map);
1028                         m_freem(data->m);
1029                         data->m = NULL;
1030                 }
1031                 if (data->map != NULL) {
1032                         bus_dmamap_destroy(ring->data_dmat, data->map);
1033                         data->map = NULL;
1034                 }
1035         }
1036         if (ring->spare_map != NULL) {
1037                 bus_dmamap_destroy(ring->data_dmat, ring->spare_map);
1038                 ring->spare_map = NULL;
1039         }
1040         if (ring->data_dmat != NULL) {
1041                 bus_dma_tag_destroy(ring->data_dmat);
1042                 ring->data_dmat = NULL;
1043         }
1044 }
1045
1046 static int
1047 iwm_alloc_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring, int qid)
1048 {
1049         bus_addr_t paddr;
1050         bus_size_t size;
1051         size_t maxsize;
1052         int nsegments;
1053         int i, error;
1054
1055         ring->qid = qid;
1056         ring->queued = 0;
1057         ring->cur = 0;
1058
1059         /* Allocate TX descriptors (256-byte aligned). */
1060         size = IWM_TX_RING_COUNT * sizeof (struct iwm_tfd);
1061         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->desc_dma, size, 256);
1062         if (error != 0) {
1063                 device_printf(sc->sc_dev,
1064                     "could not allocate TX ring DMA memory\n");
1065                 goto fail;
1066         }
1067         ring->desc = ring->desc_dma.vaddr;
1068
1069         /*
1070          * We only use rings 0 through 9 (4 EDCA + cmd) so there is no need
1071          * to allocate commands space for other rings.
1072          */
1073         if (qid > IWM_MVM_CMD_QUEUE)
1074                 return 0;
1075
1076         size = IWM_TX_RING_COUNT * sizeof(struct iwm_device_cmd);
1077         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->cmd_dma, size, 4);
1078         if (error != 0) {
1079                 device_printf(sc->sc_dev,
1080                     "could not allocate TX cmd DMA memory\n");
1081                 goto fail;
1082         }
1083         ring->cmd = ring->cmd_dma.vaddr;
1084
1085         /* FW commands may require more mapped space than packets. */
1086         if (qid == IWM_MVM_CMD_QUEUE) {
1087                 maxsize = IWM_RBUF_SIZE;
1088                 nsegments = 1;
1089         } else {
1090                 maxsize = MCLBYTES;
1091                 nsegments = IWM_MAX_SCATTER - 2;
1092         }
1093
1094         error = bus_dma_tag_create(sc->sc_dmat, 1, 0,
1095             BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, maxsize,
1096             nsegments, maxsize, 0, NULL, NULL, &ring->data_dmat);
1097         if (error != 0) {
1098                 device_printf(sc->sc_dev, "could not create TX buf DMA tag\n");
1099                 goto fail;
1100         }
1101
1102         paddr = ring->cmd_dma.paddr;
1103         for (i = 0; i < IWM_TX_RING_COUNT; i++) {
1104                 struct iwm_tx_data *data = &ring->data[i];
1105
1106                 data->cmd_paddr = paddr;
1107                 data->scratch_paddr = paddr + sizeof(struct iwm_cmd_header)
1108                     + offsetof(struct iwm_tx_cmd, scratch);
1109                 paddr += sizeof(struct iwm_device_cmd);
1110
1111                 error = bus_dmamap_create(ring->data_dmat, 0, &data->map);
1112                 if (error != 0) {
1113                         device_printf(sc->sc_dev,
1114                             "could not create TX buf DMA map\n");
1115                         goto fail;
1116                 }
1117         }
1118         KASSERT(paddr == ring->cmd_dma.paddr + size,
1119             ("invalid physical address"));
1120         return 0;
1121
1122 fail:   iwm_free_tx_ring(sc, ring);
1123         return error;
1124 }
1125
1126 static void
1127 iwm_reset_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring)
1128 {
1129         int i;
1130
1131         for (i = 0; i < IWM_TX_RING_COUNT; i++) {
1132                 struct iwm_tx_data *data = &ring->data[i];
1133
1134                 if (data->m != NULL) {
1135                         bus_dmamap_sync(ring->data_dmat, data->map,
1136                             BUS_DMASYNC_POSTWRITE);
1137                         bus_dmamap_unload(ring->data_dmat, data->map);
1138                         m_freem(data->m);
1139                         data->m = NULL;
1140                 }
1141         }
1142         /* Clear TX descriptors. */
1143         memset(ring->desc, 0, ring->desc_dma.size);
1144         bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map,
1145             BUS_DMASYNC_PREWRITE);
1146         sc->qfullmsk &= ~(1 << ring->qid);
1147         ring->queued = 0;
1148         ring->cur = 0;
1149
1150         if (ring->qid == IWM_MVM_CMD_QUEUE && sc->cmd_hold_nic_awake)
1151                 iwm_pcie_clear_cmd_in_flight(sc);
1152 }
1153
1154 static void
1155 iwm_free_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring)
1156 {
1157         int i;
1158
1159         iwm_dma_contig_free(&ring->desc_dma);
1160         iwm_dma_contig_free(&ring->cmd_dma);
1161
1162         for (i = 0; i < IWM_TX_RING_COUNT; i++) {
1163                 struct iwm_tx_data *data = &ring->data[i];
1164
1165                 if (data->m != NULL) {
1166                         bus_dmamap_sync(ring->data_dmat, data->map,
1167                             BUS_DMASYNC_POSTWRITE);
1168                         bus_dmamap_unload(ring->data_dmat, data->map);
1169                         m_freem(data->m);
1170                         data->m = NULL;
1171                 }
1172                 if (data->map != NULL) {
1173                         bus_dmamap_destroy(ring->data_dmat, data->map);
1174                         data->map = NULL;
1175                 }
1176         }
1177         if (ring->data_dmat != NULL) {
1178                 bus_dma_tag_destroy(ring->data_dmat);
1179                 ring->data_dmat = NULL;
1180         }
1181 }
1182
1183 /*
1184  * High-level hardware frobbing routines
1185  */
1186
1187 static void
1188 iwm_enable_interrupts(struct iwm_softc *sc)
1189 {
1190         sc->sc_intmask = IWM_CSR_INI_SET_MASK;
1191         IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask);
1192 }
1193
1194 static void
1195 iwm_restore_interrupts(struct iwm_softc *sc)
1196 {
1197         IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask);
1198 }
1199
1200 static void
1201 iwm_disable_interrupts(struct iwm_softc *sc)
1202 {
1203         /* disable interrupts */
1204         IWM_WRITE(sc, IWM_CSR_INT_MASK, 0);
1205
1206         /* acknowledge all interrupts */
1207         IWM_WRITE(sc, IWM_CSR_INT, ~0);
1208         IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, ~0);
1209 }
1210
1211 static void
1212 iwm_ict_reset(struct iwm_softc *sc)
1213 {
1214         iwm_disable_interrupts(sc);
1215
1216         /* Reset ICT table. */
1217         memset(sc->ict_dma.vaddr, 0, IWM_ICT_SIZE);
1218         sc->ict_cur = 0;
1219
1220         /* Set physical address of ICT table (4KB aligned). */
1221         IWM_WRITE(sc, IWM_CSR_DRAM_INT_TBL_REG,
1222             IWM_CSR_DRAM_INT_TBL_ENABLE
1223             | IWM_CSR_DRAM_INIT_TBL_WRITE_POINTER
1224             | IWM_CSR_DRAM_INIT_TBL_WRAP_CHECK
1225             | sc->ict_dma.paddr >> IWM_ICT_PADDR_SHIFT);
1226
1227         /* Switch to ICT interrupt mode in driver. */
1228         sc->sc_flags |= IWM_FLAG_USE_ICT;
1229
1230         /* Re-enable interrupts. */
1231         IWM_WRITE(sc, IWM_CSR_INT, ~0);
1232         iwm_enable_interrupts(sc);
1233 }
1234
1235 /* iwlwifi pcie/trans.c */
1236
1237 /*
1238  * Since this .. hard-resets things, it's time to actually
1239  * mark the first vap (if any) as having no mac context.
1240  * It's annoying, but since the driver is potentially being
1241  * stop/start'ed whilst active (thanks openbsd port!) we
1242  * have to correctly track this.
1243  */
1244 static void
1245 iwm_stop_device(struct iwm_softc *sc)
1246 {
1247         struct ieee80211com *ic = &sc->sc_ic;
1248         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
1249         int chnl, qid;
1250         uint32_t mask = 0;
1251
1252         /* tell the device to stop sending interrupts */
1253         iwm_disable_interrupts(sc);
1254
1255         /*
1256          * FreeBSD-local: mark the first vap as not-uploaded,
1257          * so the next transition through auth/assoc
1258          * will correctly populate the MAC context.
1259          */
1260         if (vap) {
1261                 struct iwm_vap *iv = IWM_VAP(vap);
1262                 iv->phy_ctxt = NULL;
1263                 iv->is_uploaded = 0;
1264         }
1265
1266         /* device going down, Stop using ICT table */
1267         sc->sc_flags &= ~IWM_FLAG_USE_ICT;
1268
1269         /* stop tx and rx.  tx and rx bits, as usual, are from if_iwn */
1270
1271         if (iwm_nic_lock(sc)) {
1272                 iwm_write_prph(sc, IWM_SCD_TXFACT, 0);
1273
1274                 /* Stop each Tx DMA channel */
1275                 for (chnl = 0; chnl < IWM_FH_TCSR_CHNL_NUM; chnl++) {
1276                         IWM_WRITE(sc,
1277                             IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl), 0);
1278                         mask |= IWM_FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(chnl);
1279                 }
1280
1281                 /* Wait for DMA channels to be idle */
1282                 if (!iwm_poll_bit(sc, IWM_FH_TSSR_TX_STATUS_REG, mask, mask,
1283                     5000)) {
1284                         device_printf(sc->sc_dev,
1285                             "Failing on timeout while stopping DMA channel: [0x%08x]\n",
1286                             IWM_READ(sc, IWM_FH_TSSR_TX_STATUS_REG));
1287                 }
1288                 iwm_nic_unlock(sc);
1289         }
1290         iwm_pcie_rx_stop(sc);
1291
1292         /* Stop RX ring. */
1293         iwm_reset_rx_ring(sc, &sc->rxq);
1294
1295         /* Reset all TX rings. */
1296         for (qid = 0; qid < nitems(sc->txq); qid++)
1297                 iwm_reset_tx_ring(sc, &sc->txq[qid]);
1298
1299         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
1300                 /* Power-down device's busmaster DMA clocks */
1301                 if (iwm_nic_lock(sc)) {
1302                         iwm_write_prph(sc, IWM_APMG_CLK_DIS_REG,
1303                             IWM_APMG_CLK_VAL_DMA_CLK_RQT);
1304                         iwm_nic_unlock(sc);
1305                 }
1306                 DELAY(5);
1307         }
1308
1309         /* Make sure (redundant) we've released our request to stay awake */
1310         IWM_CLRBITS(sc, IWM_CSR_GP_CNTRL,
1311             IWM_CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1312
1313         /* Stop the device, and put it in low power state */
1314         iwm_apm_stop(sc);
1315
1316         /* Upon stop, the APM issues an interrupt if HW RF kill is set.
1317          * Clean again the interrupt here
1318          */
1319         iwm_disable_interrupts(sc);
1320         /* stop and reset the on-board processor */
1321         IWM_WRITE(sc, IWM_CSR_RESET, IWM_CSR_RESET_REG_FLAG_SW_RESET);
1322
1323         /*
1324          * Even if we stop the HW, we still want the RF kill
1325          * interrupt
1326          */
1327         iwm_enable_rfkill_int(sc);
1328         iwm_check_rfkill(sc);
1329 }
1330
1331 /* iwlwifi: mvm/ops.c */
1332 static void
1333 iwm_mvm_nic_config(struct iwm_softc *sc)
1334 {
1335         uint8_t radio_cfg_type, radio_cfg_step, radio_cfg_dash;
1336         uint32_t reg_val = 0;
1337         uint32_t phy_config = iwm_mvm_get_phy_config(sc);
1338
1339         radio_cfg_type = (phy_config & IWM_FW_PHY_CFG_RADIO_TYPE) >>
1340             IWM_FW_PHY_CFG_RADIO_TYPE_POS;
1341         radio_cfg_step = (phy_config & IWM_FW_PHY_CFG_RADIO_STEP) >>
1342             IWM_FW_PHY_CFG_RADIO_STEP_POS;
1343         radio_cfg_dash = (phy_config & IWM_FW_PHY_CFG_RADIO_DASH) >>
1344             IWM_FW_PHY_CFG_RADIO_DASH_POS;
1345
1346         /* SKU control */
1347         reg_val |= IWM_CSR_HW_REV_STEP(sc->sc_hw_rev) <<
1348             IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_STEP;
1349         reg_val |= IWM_CSR_HW_REV_DASH(sc->sc_hw_rev) <<
1350             IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_DASH;
1351
1352         /* radio configuration */
1353         reg_val |= radio_cfg_type << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_TYPE;
1354         reg_val |= radio_cfg_step << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_STEP;
1355         reg_val |= radio_cfg_dash << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_DASH;
1356
1357         IWM_WRITE(sc, IWM_CSR_HW_IF_CONFIG_REG, reg_val);
1358
1359         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
1360             "Radio type=0x%x-0x%x-0x%x\n", radio_cfg_type,
1361             radio_cfg_step, radio_cfg_dash);
1362
1363         /*
1364          * W/A : NIC is stuck in a reset state after Early PCIe power off
1365          * (PCIe power is lost before PERST# is asserted), causing ME FW
1366          * to lose ownership and not being able to obtain it back.
1367          */
1368         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
1369                 iwm_set_bits_mask_prph(sc, IWM_APMG_PS_CTRL_REG,
1370                     IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS,
1371                     ~IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS);
1372         }
1373 }
1374
1375 static int
1376 iwm_nic_rx_init(struct iwm_softc *sc)
1377 {
1378         /*
1379          * Initialize RX ring.  This is from the iwn driver.
1380          */
1381         memset(sc->rxq.stat, 0, sizeof(*sc->rxq.stat));
1382
1383         /* Stop Rx DMA */
1384         iwm_pcie_rx_stop(sc);
1385
1386         if (!iwm_nic_lock(sc))
1387                 return EBUSY;
1388
1389         /* reset and flush pointers */
1390         IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_RBDCB_WPTR, 0);
1391         IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_FLUSH_RB_REQ, 0);
1392         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_RDPTR, 0);
1393         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_RBDCB_WPTR_REG, 0);
1394
1395         /* Set physical address of RX ring (256-byte aligned). */
1396         IWM_WRITE(sc,
1397             IWM_FH_RSCSR_CHNL0_RBDCB_BASE_REG, sc->rxq.desc_dma.paddr >> 8);
1398
1399         /* Set physical address of RX status (16-byte aligned). */
1400         IWM_WRITE(sc,
1401             IWM_FH_RSCSR_CHNL0_STTS_WPTR_REG, sc->rxq.stat_dma.paddr >> 4);
1402
1403         /* Enable Rx DMA
1404          * XXX 5000 HW isn't supported by the iwm(4) driver.
1405          * IWM_FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY is set because of HW bug in
1406          *      the credit mechanism in 5000 HW RX FIFO
1407          * Direct rx interrupts to hosts
1408          * Rx buffer size 4 or 8k or 12k
1409          * RB timeout 0x10
1410          * 256 RBDs
1411          */
1412         IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_CONFIG_REG,
1413             IWM_FH_RCSR_RX_CONFIG_CHNL_EN_ENABLE_VAL            |
1414             IWM_FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY               |  /* HW bug */
1415             IWM_FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_INT_HOST_VAL   |
1416             IWM_FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K            |
1417             (IWM_RX_RB_TIMEOUT << IWM_FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS) |
1418             IWM_RX_QUEUE_SIZE_LOG << IWM_FH_RCSR_RX_CONFIG_RBDCB_SIZE_POS);
1419
1420         IWM_WRITE_1(sc, IWM_CSR_INT_COALESCING, IWM_HOST_INT_TIMEOUT_DEF);
1421
1422         /* W/A for interrupt coalescing bug in 7260 and 3160 */
1423         if (sc->cfg->host_interrupt_operation_mode)
1424                 IWM_SETBITS(sc, IWM_CSR_INT_COALESCING, IWM_HOST_INT_OPER_MODE);
1425
1426         /*
1427          * Thus sayeth el jefe (iwlwifi) via a comment:
1428          *
1429          * This value should initially be 0 (before preparing any
1430          * RBs), should be 8 after preparing the first 8 RBs (for example)
1431          */
1432         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_WPTR, 8);
1433
1434         iwm_nic_unlock(sc);
1435
1436         return 0;
1437 }
1438
1439 static int
1440 iwm_nic_tx_init(struct iwm_softc *sc)
1441 {
1442         int qid;
1443
1444         if (!iwm_nic_lock(sc))
1445                 return EBUSY;
1446
1447         /* Deactivate TX scheduler. */
1448         iwm_write_prph(sc, IWM_SCD_TXFACT, 0);
1449
1450         /* Set physical address of "keep warm" page (16-byte aligned). */
1451         IWM_WRITE(sc, IWM_FH_KW_MEM_ADDR_REG, sc->kw_dma.paddr >> 4);
1452
1453         /* Initialize TX rings. */
1454         for (qid = 0; qid < nitems(sc->txq); qid++) {
1455                 struct iwm_tx_ring *txq = &sc->txq[qid];
1456
1457                 /* Set physical address of TX ring (256-byte aligned). */
1458                 IWM_WRITE(sc, IWM_FH_MEM_CBBC_QUEUE(qid),
1459                     txq->desc_dma.paddr >> 8);
1460                 IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
1461                     "%s: loading ring %d descriptors (%p) at %lx\n",
1462                     __func__,
1463                     qid, txq->desc,
1464                     (unsigned long) (txq->desc_dma.paddr >> 8));
1465         }
1466
1467         iwm_write_prph(sc, IWM_SCD_GP_CTRL, IWM_SCD_GP_CTRL_AUTO_ACTIVE_MODE);
1468
1469         iwm_nic_unlock(sc);
1470
1471         return 0;
1472 }
1473
1474 static int
1475 iwm_nic_init(struct iwm_softc *sc)
1476 {
1477         int error;
1478
1479         iwm_apm_init(sc);
1480         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
1481                 iwm_set_pwr(sc);
1482
1483         iwm_mvm_nic_config(sc);
1484
1485         if ((error = iwm_nic_rx_init(sc)) != 0)
1486                 return error;
1487
1488         /*
1489          * Ditto for TX, from iwn
1490          */
1491         if ((error = iwm_nic_tx_init(sc)) != 0)
1492                 return error;
1493
1494         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
1495             "%s: shadow registers enabled\n", __func__);
1496         IWM_SETBITS(sc, IWM_CSR_MAC_SHADOW_REG_CTRL, 0x800fffff);
1497
1498         return 0;
1499 }
1500
1501 int
1502 iwm_enable_txq(struct iwm_softc *sc, int sta_id, int qid, int fifo)
1503 {
1504         if (!iwm_nic_lock(sc)) {
1505                 device_printf(sc->sc_dev,
1506                     "%s: cannot enable txq %d\n",
1507                     __func__,
1508                     qid);
1509                 return EBUSY;
1510         }
1511
1512         IWM_WRITE(sc, IWM_HBUS_TARG_WRPTR, qid << 8 | 0);
1513
1514         if (qid == IWM_MVM_CMD_QUEUE) {
1515                 /* unactivate before configuration */
1516                 iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid),
1517                     (0 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE)
1518                     | (1 << IWM_SCD_QUEUE_STTS_REG_POS_SCD_ACT_EN));
1519
1520                 iwm_nic_unlock(sc);
1521
1522                 iwm_clear_bits_prph(sc, IWM_SCD_AGGR_SEL, (1 << qid));
1523
1524                 if (!iwm_nic_lock(sc)) {
1525                         device_printf(sc->sc_dev,
1526                             "%s: cannot enable txq %d\n", __func__, qid);
1527                         return EBUSY;
1528                 }
1529                 iwm_write_prph(sc, IWM_SCD_QUEUE_RDPTR(qid), 0);
1530                 iwm_nic_unlock(sc);
1531
1532                 iwm_write_mem32(sc, sc->scd_base_addr + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid), 0);
1533                 /* Set scheduler window size and frame limit. */
1534                 iwm_write_mem32(sc,
1535                     sc->scd_base_addr + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid) +
1536                     sizeof(uint32_t),
1537                     ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_POS) &
1538                     IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_MSK) |
1539                     ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_POS) &
1540                     IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_MSK));
1541
1542                 if (!iwm_nic_lock(sc)) {
1543                         device_printf(sc->sc_dev,
1544                             "%s: cannot enable txq %d\n", __func__, qid);
1545                         return EBUSY;
1546                 }
1547                 iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid),
1548                     (1 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE) |
1549                     (fifo << IWM_SCD_QUEUE_STTS_REG_POS_TXF) |
1550                     (1 << IWM_SCD_QUEUE_STTS_REG_POS_WSL) |
1551                     IWM_SCD_QUEUE_STTS_REG_MSK);
1552         } else {
1553                 struct iwm_scd_txq_cfg_cmd cmd;
1554                 int error;
1555
1556                 iwm_nic_unlock(sc);
1557
1558                 memset(&cmd, 0, sizeof(cmd));
1559                 cmd.scd_queue = qid;
1560                 cmd.enable = 1;
1561                 cmd.sta_id = sta_id;
1562                 cmd.tx_fifo = fifo;
1563                 cmd.aggregate = 0;
1564                 cmd.window = IWM_FRAME_LIMIT;
1565
1566                 error = iwm_mvm_send_cmd_pdu(sc, IWM_SCD_QUEUE_CFG, IWM_CMD_SYNC,
1567                     sizeof(cmd), &cmd);
1568                 if (error) {
1569                         device_printf(sc->sc_dev,
1570                             "cannot enable txq %d\n", qid);
1571                         return error;
1572                 }
1573
1574                 if (!iwm_nic_lock(sc))
1575                         return EBUSY;
1576         }
1577
1578         iwm_write_prph(sc, IWM_SCD_EN_CTRL,
1579             iwm_read_prph(sc, IWM_SCD_EN_CTRL) | qid);
1580
1581         iwm_nic_unlock(sc);
1582
1583         IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "%s: enabled txq %d FIFO %d\n",
1584             __func__, qid, fifo);
1585
1586         return 0;
1587 }
1588
1589 static int
1590 iwm_trans_pcie_fw_alive(struct iwm_softc *sc, uint32_t scd_base_addr)
1591 {
1592         int error, chnl;
1593
1594         int clear_dwords = (IWM_SCD_TRANS_TBL_MEM_UPPER_BOUND -
1595             IWM_SCD_CONTEXT_MEM_LOWER_BOUND) / sizeof(uint32_t);
1596
1597         if (!iwm_nic_lock(sc))
1598                 return EBUSY;
1599
1600         iwm_ict_reset(sc);
1601
1602         sc->scd_base_addr = iwm_read_prph(sc, IWM_SCD_SRAM_BASE_ADDR);
1603         if (scd_base_addr != 0 &&
1604             scd_base_addr != sc->scd_base_addr) {
1605                 device_printf(sc->sc_dev,
1606                     "%s: sched addr mismatch: alive: 0x%x prph: 0x%x\n",
1607                     __func__, sc->scd_base_addr, scd_base_addr);
1608         }
1609
1610         iwm_nic_unlock(sc);
1611
1612         /* reset context data, TX status and translation data */
1613         error = iwm_write_mem(sc,
1614             sc->scd_base_addr + IWM_SCD_CONTEXT_MEM_LOWER_BOUND,
1615             NULL, clear_dwords);
1616         if (error)
1617                 return EBUSY;
1618
1619         if (!iwm_nic_lock(sc))
1620                 return EBUSY;
1621
1622         /* Set physical address of TX scheduler rings (1KB aligned). */
1623         iwm_write_prph(sc, IWM_SCD_DRAM_BASE_ADDR, sc->sched_dma.paddr >> 10);
1624
1625         iwm_write_prph(sc, IWM_SCD_CHAINEXT_EN, 0);
1626
1627         iwm_nic_unlock(sc);
1628
1629         /* enable command channel */
1630         error = iwm_enable_txq(sc, 0 /* unused */, IWM_MVM_CMD_QUEUE, 7);
1631         if (error)
1632                 return error;
1633
1634         if (!iwm_nic_lock(sc))
1635                 return EBUSY;
1636
1637         iwm_write_prph(sc, IWM_SCD_TXFACT, 0xff);
1638
1639         /* Enable DMA channels. */
1640         for (chnl = 0; chnl < IWM_FH_TCSR_CHNL_NUM; chnl++) {
1641                 IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl),
1642                     IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE |
1643                     IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE);
1644         }
1645
1646         IWM_SETBITS(sc, IWM_FH_TX_CHICKEN_BITS_REG,
1647             IWM_FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN);
1648
1649         iwm_nic_unlock(sc);
1650
1651         /* Enable L1-Active */
1652         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
1653                 iwm_clear_bits_prph(sc, IWM_APMG_PCIDEV_STT_REG,
1654                     IWM_APMG_PCIDEV_STT_VAL_L1_ACT_DIS);
1655         }
1656
1657         return error;
1658 }
1659
1660 /*
1661  * NVM read access and content parsing.  We do not support
1662  * external NVM or writing NVM.
1663  * iwlwifi/mvm/nvm.c
1664  */
1665
1666 /* Default NVM size to read */
1667 #define IWM_NVM_DEFAULT_CHUNK_SIZE      (2*1024)
1668
1669 #define IWM_NVM_WRITE_OPCODE 1
1670 #define IWM_NVM_READ_OPCODE 0
1671
1672 /* load nvm chunk response */
1673 enum {
1674         IWM_READ_NVM_CHUNK_SUCCEED = 0,
1675         IWM_READ_NVM_CHUNK_NOT_VALID_ADDRESS = 1
1676 };
1677
1678 static int
1679 iwm_nvm_read_chunk(struct iwm_softc *sc, uint16_t section,
1680         uint16_t offset, uint16_t length, uint8_t *data, uint16_t *len)
1681 {
1682         struct iwm_nvm_access_cmd nvm_access_cmd = {
1683                 .offset = htole16(offset),
1684                 .length = htole16(length),
1685                 .type = htole16(section),
1686                 .op_code = IWM_NVM_READ_OPCODE,
1687         };
1688         struct iwm_nvm_access_resp *nvm_resp;
1689         struct iwm_rx_packet *pkt;
1690         struct iwm_host_cmd cmd = {
1691                 .id = IWM_NVM_ACCESS_CMD,
1692                 .flags = IWM_CMD_WANT_SKB | IWM_CMD_SEND_IN_RFKILL,
1693                 .data = { &nvm_access_cmd, },
1694         };
1695         int ret, bytes_read, offset_read;
1696         uint8_t *resp_data;
1697
1698         cmd.len[0] = sizeof(struct iwm_nvm_access_cmd);
1699
1700         ret = iwm_send_cmd(sc, &cmd);
1701         if (ret) {
1702                 device_printf(sc->sc_dev,
1703                     "Could not send NVM_ACCESS command (error=%d)\n", ret);
1704                 return ret;
1705         }
1706
1707         pkt = cmd.resp_pkt;
1708
1709         /* Extract NVM response */
1710         nvm_resp = (void *)pkt->data;
1711         ret = le16toh(nvm_resp->status);
1712         bytes_read = le16toh(nvm_resp->length);
1713         offset_read = le16toh(nvm_resp->offset);
1714         resp_data = nvm_resp->data;
1715         if (ret) {
1716                 if ((offset != 0) &&
1717                     (ret == IWM_READ_NVM_CHUNK_NOT_VALID_ADDRESS)) {
1718                         /*
1719                          * meaning of NOT_VALID_ADDRESS:
1720                          * driver try to read chunk from address that is
1721                          * multiple of 2K and got an error since addr is empty.
1722                          * meaning of (offset != 0): driver already
1723                          * read valid data from another chunk so this case
1724                          * is not an error.
1725                          */
1726                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1727                                     "NVM access command failed on offset 0x%x since that section size is multiple 2K\n",
1728                                     offset);
1729                         *len = 0;
1730                         ret = 0;
1731                 } else {
1732                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1733                                     "NVM access command failed with status %d\n", ret);
1734                         ret = EIO;
1735                 }
1736                 goto exit;
1737         }
1738
1739         if (offset_read != offset) {
1740                 device_printf(sc->sc_dev,
1741                     "NVM ACCESS response with invalid offset %d\n",
1742                     offset_read);
1743                 ret = EINVAL;
1744                 goto exit;
1745         }
1746
1747         if (bytes_read > length) {
1748                 device_printf(sc->sc_dev,
1749                     "NVM ACCESS response with too much data "
1750                     "(%d bytes requested, %d bytes received)\n",
1751                     length, bytes_read);
1752                 ret = EINVAL;
1753                 goto exit;
1754         }
1755
1756         /* Write data to NVM */
1757         memcpy(data + offset, resp_data, bytes_read);
1758         *len = bytes_read;
1759
1760  exit:
1761         iwm_free_resp(sc, &cmd);
1762         return ret;
1763 }
1764
1765 /*
1766  * Reads an NVM section completely.
1767  * NICs prior to 7000 family don't have a real NVM, but just read
1768  * section 0 which is the EEPROM. Because the EEPROM reading is unlimited
1769  * by uCode, we need to manually check in this case that we don't
1770  * overflow and try to read more than the EEPROM size.
1771  * For 7000 family NICs, we supply the maximal size we can read, and
1772  * the uCode fills the response with as much data as we can,
1773  * without overflowing, so no check is needed.
1774  */
1775 static int
1776 iwm_nvm_read_section(struct iwm_softc *sc,
1777         uint16_t section, uint8_t *data, uint16_t *len, uint32_t size_read)
1778 {
1779         uint16_t seglen, length, offset = 0;
1780         int ret;
1781
1782         /* Set nvm section read length */
1783         length = IWM_NVM_DEFAULT_CHUNK_SIZE;
1784
1785         seglen = length;
1786
1787         /* Read the NVM until exhausted (reading less than requested) */
1788         while (seglen == length) {
1789                 /* Check no memory assumptions fail and cause an overflow */
1790                 if ((size_read + offset + length) >
1791                     sc->cfg->eeprom_size) {
1792                         device_printf(sc->sc_dev,
1793                             "EEPROM size is too small for NVM\n");
1794                         return ENOBUFS;
1795                 }
1796
1797                 ret = iwm_nvm_read_chunk(sc, section, offset, length, data, &seglen);
1798                 if (ret) {
1799                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1800                                     "Cannot read NVM from section %d offset %d, length %d\n",
1801                                     section, offset, length);
1802                         return ret;
1803                 }
1804                 offset += seglen;
1805         }
1806
1807         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1808                     "NVM section %d read completed\n", section);
1809         *len = offset;
1810         return 0;
1811 }
1812
1813 /*
1814  * BEGIN IWM_NVM_PARSE
1815  */
1816
1817 /* iwlwifi/iwl-nvm-parse.c */
1818
1819 /* NVM offsets (in words) definitions */
1820 enum iwm_nvm_offsets {
1821         /* NVM HW-Section offset (in words) definitions */
1822         IWM_HW_ADDR = 0x15,
1823
1824 /* NVM SW-Section offset (in words) definitions */
1825         IWM_NVM_SW_SECTION = 0x1C0,
1826         IWM_NVM_VERSION = 0,
1827         IWM_RADIO_CFG = 1,
1828         IWM_SKU = 2,
1829         IWM_N_HW_ADDRS = 3,
1830         IWM_NVM_CHANNELS = 0x1E0 - IWM_NVM_SW_SECTION,
1831
1832 /* NVM calibration section offset (in words) definitions */
1833         IWM_NVM_CALIB_SECTION = 0x2B8,
1834         IWM_XTAL_CALIB = 0x316 - IWM_NVM_CALIB_SECTION
1835 };
1836
1837 enum iwm_8000_nvm_offsets {
1838         /* NVM HW-Section offset (in words) definitions */
1839         IWM_HW_ADDR0_WFPM_8000 = 0x12,
1840         IWM_HW_ADDR1_WFPM_8000 = 0x16,
1841         IWM_HW_ADDR0_PCIE_8000 = 0x8A,
1842         IWM_HW_ADDR1_PCIE_8000 = 0x8E,
1843         IWM_MAC_ADDRESS_OVERRIDE_8000 = 1,
1844
1845         /* NVM SW-Section offset (in words) definitions */
1846         IWM_NVM_SW_SECTION_8000 = 0x1C0,
1847         IWM_NVM_VERSION_8000 = 0,
1848         IWM_RADIO_CFG_8000 = 0,
1849         IWM_SKU_8000 = 2,
1850         IWM_N_HW_ADDRS_8000 = 3,
1851
1852         /* NVM REGULATORY -Section offset (in words) definitions */
1853         IWM_NVM_CHANNELS_8000 = 0,
1854         IWM_NVM_LAR_OFFSET_8000_OLD = 0x4C7,
1855         IWM_NVM_LAR_OFFSET_8000 = 0x507,
1856         IWM_NVM_LAR_ENABLED_8000 = 0x7,
1857
1858         /* NVM calibration section offset (in words) definitions */
1859         IWM_NVM_CALIB_SECTION_8000 = 0x2B8,
1860         IWM_XTAL_CALIB_8000 = 0x316 - IWM_NVM_CALIB_SECTION_8000
1861 };
1862
1863 /* SKU Capabilities (actual values from NVM definition) */
1864 enum nvm_sku_bits {
1865         IWM_NVM_SKU_CAP_BAND_24GHZ      = (1 << 0),
1866         IWM_NVM_SKU_CAP_BAND_52GHZ      = (1 << 1),
1867         IWM_NVM_SKU_CAP_11N_ENABLE      = (1 << 2),
1868         IWM_NVM_SKU_CAP_11AC_ENABLE     = (1 << 3),
1869 };
1870
1871 /* radio config bits (actual values from NVM definition) */
1872 #define IWM_NVM_RF_CFG_DASH_MSK(x)   (x & 0x3)         /* bits 0-1   */
1873 #define IWM_NVM_RF_CFG_STEP_MSK(x)   ((x >> 2)  & 0x3) /* bits 2-3   */
1874 #define IWM_NVM_RF_CFG_TYPE_MSK(x)   ((x >> 4)  & 0x3) /* bits 4-5   */
1875 #define IWM_NVM_RF_CFG_PNUM_MSK(x)   ((x >> 6)  & 0x3) /* bits 6-7   */
1876 #define IWM_NVM_RF_CFG_TX_ANT_MSK(x) ((x >> 8)  & 0xF) /* bits 8-11  */
1877 #define IWM_NVM_RF_CFG_RX_ANT_MSK(x) ((x >> 12) & 0xF) /* bits 12-15 */
1878
1879 #define IWM_NVM_RF_CFG_FLAVOR_MSK_8000(x)       (x & 0xF)
1880 #define IWM_NVM_RF_CFG_DASH_MSK_8000(x)         ((x >> 4) & 0xF)
1881 #define IWM_NVM_RF_CFG_STEP_MSK_8000(x)         ((x >> 8) & 0xF)
1882 #define IWM_NVM_RF_CFG_TYPE_MSK_8000(x)         ((x >> 12) & 0xFFF)
1883 #define IWM_NVM_RF_CFG_TX_ANT_MSK_8000(x)       ((x >> 24) & 0xF)
1884 #define IWM_NVM_RF_CFG_RX_ANT_MSK_8000(x)       ((x >> 28) & 0xF)
1885
1886 /**
1887  * enum iwm_nvm_channel_flags - channel flags in NVM
1888  * @IWM_NVM_CHANNEL_VALID: channel is usable for this SKU/geo
1889  * @IWM_NVM_CHANNEL_IBSS: usable as an IBSS channel
1890  * @IWM_NVM_CHANNEL_ACTIVE: active scanning allowed
1891  * @IWM_NVM_CHANNEL_RADAR: radar detection required
1892  * XXX cannot find this (DFS) flag in iwm-nvm-parse.c
1893  * @IWM_NVM_CHANNEL_DFS: dynamic freq selection candidate
1894  * @IWM_NVM_CHANNEL_WIDE: 20 MHz channel okay (?)
1895  * @IWM_NVM_CHANNEL_40MHZ: 40 MHz channel okay (?)
1896  * @IWM_NVM_CHANNEL_80MHZ: 80 MHz channel okay (?)
1897  * @IWM_NVM_CHANNEL_160MHZ: 160 MHz channel okay (?)
1898  */
1899 enum iwm_nvm_channel_flags {
1900         IWM_NVM_CHANNEL_VALID = (1 << 0),
1901         IWM_NVM_CHANNEL_IBSS = (1 << 1),
1902         IWM_NVM_CHANNEL_ACTIVE = (1 << 3),
1903         IWM_NVM_CHANNEL_RADAR = (1 << 4),
1904         IWM_NVM_CHANNEL_DFS = (1 << 7),
1905         IWM_NVM_CHANNEL_WIDE = (1 << 8),
1906         IWM_NVM_CHANNEL_40MHZ = (1 << 9),
1907         IWM_NVM_CHANNEL_80MHZ = (1 << 10),
1908         IWM_NVM_CHANNEL_160MHZ = (1 << 11),
1909 };
1910
1911 /*
1912  * Translate EEPROM flags to net80211.
1913  */
1914 static uint32_t
1915 iwm_eeprom_channel_flags(uint16_t ch_flags)
1916 {
1917         uint32_t nflags;
1918
1919         nflags = 0;
1920         if ((ch_flags & IWM_NVM_CHANNEL_ACTIVE) == 0)
1921                 nflags |= IEEE80211_CHAN_PASSIVE;
1922         if ((ch_flags & IWM_NVM_CHANNEL_IBSS) == 0)
1923                 nflags |= IEEE80211_CHAN_NOADHOC;
1924         if (ch_flags & IWM_NVM_CHANNEL_RADAR) {
1925                 nflags |= IEEE80211_CHAN_DFS;
1926                 /* Just in case. */
1927                 nflags |= IEEE80211_CHAN_NOADHOC;
1928         }
1929
1930         return (nflags);
1931 }
1932
1933 static void
1934 iwm_add_channel_band(struct iwm_softc *sc, struct ieee80211_channel chans[],
1935     int maxchans, int *nchans, int ch_idx, size_t ch_num,
1936     const uint8_t bands[])
1937 {
1938         const uint16_t * const nvm_ch_flags = sc->nvm_data->nvm_ch_flags;
1939         uint32_t nflags;
1940         uint16_t ch_flags;
1941         uint8_t ieee;
1942         int error;
1943
1944         for (; ch_idx < ch_num; ch_idx++) {
1945                 ch_flags = le16_to_cpup(nvm_ch_flags + ch_idx);
1946                 if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
1947                         ieee = iwm_nvm_channels[ch_idx];
1948                 else
1949                         ieee = iwm_nvm_channels_8000[ch_idx];
1950
1951                 if (!(ch_flags & IWM_NVM_CHANNEL_VALID)) {
1952                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM,
1953                             "Ch. %d Flags %x [%sGHz] - No traffic\n",
1954                             ieee, ch_flags,
1955                             (ch_idx >= IWM_NUM_2GHZ_CHANNELS) ?
1956                             "5.2" : "2.4");
1957                         continue;
1958                 }
1959
1960                 nflags = iwm_eeprom_channel_flags(ch_flags);
1961                 error = ieee80211_add_channel(chans, maxchans, nchans,
1962                     ieee, 0, 0, nflags, bands);
1963                 if (error != 0)
1964                         break;
1965
1966                 IWM_DPRINTF(sc, IWM_DEBUG_EEPROM,
1967                     "Ch. %d Flags %x [%sGHz] - Added\n",
1968                     ieee, ch_flags,
1969                     (ch_idx >= IWM_NUM_2GHZ_CHANNELS) ?
1970                     "5.2" : "2.4");
1971         }
1972 }
1973
1974 static void
1975 iwm_init_channel_map(struct ieee80211com *ic, int maxchans, int *nchans,
1976     struct ieee80211_channel chans[])
1977 {
1978         struct iwm_softc *sc = ic->ic_softc;
1979         struct iwm_nvm_data *data = sc->nvm_data;
1980         uint8_t bands[IEEE80211_MODE_BYTES];
1981         size_t ch_num;
1982
1983         memset(bands, 0, sizeof(bands));
1984         /* 1-13: 11b/g channels. */
1985         setbit(bands, IEEE80211_MODE_11B);
1986         setbit(bands, IEEE80211_MODE_11G);
1987         iwm_add_channel_band(sc, chans, maxchans, nchans, 0,
1988             IWM_NUM_2GHZ_CHANNELS - 1, bands);
1989
1990         /* 14: 11b channel only. */
1991         clrbit(bands, IEEE80211_MODE_11G);
1992         iwm_add_channel_band(sc, chans, maxchans, nchans,
1993             IWM_NUM_2GHZ_CHANNELS - 1, IWM_NUM_2GHZ_CHANNELS, bands);
1994
1995         if (data->sku_cap_band_52GHz_enable) {
1996                 if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
1997                         ch_num = nitems(iwm_nvm_channels);
1998                 else
1999                         ch_num = nitems(iwm_nvm_channels_8000);
2000                 memset(bands, 0, sizeof(bands));
2001                 setbit(bands, IEEE80211_MODE_11A);
2002                 iwm_add_channel_band(sc, chans, maxchans, nchans,
2003                     IWM_NUM_2GHZ_CHANNELS, ch_num, bands);
2004         }
2005 }
2006
2007 static void
2008 iwm_set_hw_address_family_8000(struct iwm_softc *sc, struct iwm_nvm_data *data,
2009         const uint16_t *mac_override, const uint16_t *nvm_hw)
2010 {
2011         const uint8_t *hw_addr;
2012
2013         if (mac_override) {
2014                 static const uint8_t reserved_mac[] = {
2015                         0x02, 0xcc, 0xaa, 0xff, 0xee, 0x00
2016                 };
2017
2018                 hw_addr = (const uint8_t *)(mac_override +
2019                                  IWM_MAC_ADDRESS_OVERRIDE_8000);
2020
2021                 /*
2022                  * Store the MAC address from MAO section.
2023                  * No byte swapping is required in MAO section
2024                  */
2025                 IEEE80211_ADDR_COPY(data->hw_addr, hw_addr);
2026
2027                 /*
2028                  * Force the use of the OTP MAC address in case of reserved MAC
2029                  * address in the NVM, or if address is given but invalid.
2030                  */
2031                 if (!IEEE80211_ADDR_EQ(reserved_mac, hw_addr) &&
2032                     !IEEE80211_ADDR_EQ(ieee80211broadcastaddr, data->hw_addr) &&
2033                     iwm_is_valid_ether_addr(data->hw_addr) &&
2034                     !IEEE80211_IS_MULTICAST(data->hw_addr))
2035                         return;
2036
2037                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2038                     "%s: mac address from nvm override section invalid\n",
2039                     __func__);
2040         }
2041
2042         if (nvm_hw) {
2043                 /* read the mac address from WFMP registers */
2044                 uint32_t mac_addr0 =
2045                     htole32(iwm_read_prph(sc, IWM_WFMP_MAC_ADDR_0));
2046                 uint32_t mac_addr1 =
2047                     htole32(iwm_read_prph(sc, IWM_WFMP_MAC_ADDR_1));
2048
2049                 hw_addr = (const uint8_t *)&mac_addr0;
2050                 data->hw_addr[0] = hw_addr[3];
2051                 data->hw_addr[1] = hw_addr[2];
2052                 data->hw_addr[2] = hw_addr[1];
2053                 data->hw_addr[3] = hw_addr[0];
2054
2055                 hw_addr = (const uint8_t *)&mac_addr1;
2056                 data->hw_addr[4] = hw_addr[1];
2057                 data->hw_addr[5] = hw_addr[0];
2058
2059                 return;
2060         }
2061
2062         device_printf(sc->sc_dev, "%s: mac address not found\n", __func__);
2063         memset(data->hw_addr, 0, sizeof(data->hw_addr));
2064 }
2065
2066 static int
2067 iwm_get_sku(const struct iwm_softc *sc, const uint16_t *nvm_sw,
2068             const uint16_t *phy_sku)
2069 {
2070         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2071                 return le16_to_cpup(nvm_sw + IWM_SKU);
2072
2073         return le32_to_cpup((const uint32_t *)(phy_sku + IWM_SKU_8000));
2074 }
2075
2076 static int
2077 iwm_get_nvm_version(const struct iwm_softc *sc, const uint16_t *nvm_sw)
2078 {
2079         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2080                 return le16_to_cpup(nvm_sw + IWM_NVM_VERSION);
2081         else
2082                 return le32_to_cpup((const uint32_t *)(nvm_sw +
2083                                                 IWM_NVM_VERSION_8000));
2084 }
2085
2086 static int
2087 iwm_get_radio_cfg(const struct iwm_softc *sc, const uint16_t *nvm_sw,
2088                   const uint16_t *phy_sku)
2089 {
2090         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2091                 return le16_to_cpup(nvm_sw + IWM_RADIO_CFG);
2092
2093         return le32_to_cpup((const uint32_t *)(phy_sku + IWM_RADIO_CFG_8000));
2094 }
2095
2096 static int
2097 iwm_get_n_hw_addrs(const struct iwm_softc *sc, const uint16_t *nvm_sw)
2098 {
2099         int n_hw_addr;
2100
2101         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2102                 return le16_to_cpup(nvm_sw + IWM_N_HW_ADDRS);
2103
2104         n_hw_addr = le32_to_cpup((const uint32_t *)(nvm_sw + IWM_N_HW_ADDRS_8000));
2105
2106         return n_hw_addr & IWM_N_HW_ADDR_MASK;
2107 }
2108
2109 static void
2110 iwm_set_radio_cfg(const struct iwm_softc *sc, struct iwm_nvm_data *data,
2111                   uint32_t radio_cfg)
2112 {
2113         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
2114                 data->radio_cfg_type = IWM_NVM_RF_CFG_TYPE_MSK(radio_cfg);
2115                 data->radio_cfg_step = IWM_NVM_RF_CFG_STEP_MSK(radio_cfg);
2116                 data->radio_cfg_dash = IWM_NVM_RF_CFG_DASH_MSK(radio_cfg);
2117                 data->radio_cfg_pnum = IWM_NVM_RF_CFG_PNUM_MSK(radio_cfg);
2118                 return;
2119         }
2120
2121         /* set the radio configuration for family 8000 */
2122         data->radio_cfg_type = IWM_NVM_RF_CFG_TYPE_MSK_8000(radio_cfg);
2123         data->radio_cfg_step = IWM_NVM_RF_CFG_STEP_MSK_8000(radio_cfg);
2124         data->radio_cfg_dash = IWM_NVM_RF_CFG_DASH_MSK_8000(radio_cfg);
2125         data->radio_cfg_pnum = IWM_NVM_RF_CFG_FLAVOR_MSK_8000(radio_cfg);
2126         data->valid_tx_ant = IWM_NVM_RF_CFG_TX_ANT_MSK_8000(radio_cfg);
2127         data->valid_rx_ant = IWM_NVM_RF_CFG_RX_ANT_MSK_8000(radio_cfg);
2128 }
2129
2130 static int
2131 iwm_set_hw_address(struct iwm_softc *sc, struct iwm_nvm_data *data,
2132                    const uint16_t *nvm_hw, const uint16_t *mac_override)
2133 {
2134 #ifdef notyet /* for FAMILY 9000 */
2135         if (cfg->mac_addr_from_csr) {
2136                 iwm_set_hw_address_from_csr(sc, data);
2137         } else
2138 #endif
2139         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
2140                 const uint8_t *hw_addr = (const uint8_t *)(nvm_hw + IWM_HW_ADDR);
2141
2142                 /* The byte order is little endian 16 bit, meaning 214365 */
2143                 data->hw_addr[0] = hw_addr[1];
2144                 data->hw_addr[1] = hw_addr[0];
2145                 data->hw_addr[2] = hw_addr[3];
2146                 data->hw_addr[3] = hw_addr[2];
2147                 data->hw_addr[4] = hw_addr[5];
2148                 data->hw_addr[5] = hw_addr[4];
2149         } else {
2150                 iwm_set_hw_address_family_8000(sc, data, mac_override, nvm_hw);
2151         }
2152
2153         if (!iwm_is_valid_ether_addr(data->hw_addr)) {
2154                 device_printf(sc->sc_dev, "no valid mac address was found\n");
2155                 return EINVAL;
2156         }
2157
2158         return 0;
2159 }
2160
2161 static struct iwm_nvm_data *
2162 iwm_parse_nvm_data(struct iwm_softc *sc,
2163                    const uint16_t *nvm_hw, const uint16_t *nvm_sw,
2164                    const uint16_t *nvm_calib, const uint16_t *mac_override,
2165                    const uint16_t *phy_sku, const uint16_t *regulatory)
2166 {
2167         struct iwm_nvm_data *data;
2168         uint32_t sku, radio_cfg;
2169         uint16_t lar_config;
2170
2171         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
2172                 data = malloc(sizeof(*data) +
2173                     IWM_NUM_CHANNELS * sizeof(uint16_t),
2174                     M_DEVBUF, M_NOWAIT | M_ZERO);
2175         } else {
2176                 data = malloc(sizeof(*data) +
2177                     IWM_NUM_CHANNELS_8000 * sizeof(uint16_t),
2178                     M_DEVBUF, M_NOWAIT | M_ZERO);
2179         }
2180         if (!data)
2181                 return NULL;
2182
2183         data->nvm_version = iwm_get_nvm_version(sc, nvm_sw);
2184
2185         radio_cfg = iwm_get_radio_cfg(sc, nvm_sw, phy_sku);
2186         iwm_set_radio_cfg(sc, data, radio_cfg);
2187
2188         sku = iwm_get_sku(sc, nvm_sw, phy_sku);
2189         data->sku_cap_band_24GHz_enable = sku & IWM_NVM_SKU_CAP_BAND_24GHZ;
2190         data->sku_cap_band_52GHz_enable = sku & IWM_NVM_SKU_CAP_BAND_52GHZ;
2191         data->sku_cap_11n_enable = 0;
2192
2193         data->n_hw_addrs = iwm_get_n_hw_addrs(sc, nvm_sw);
2194
2195         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) {
2196                 uint16_t lar_offset = data->nvm_version < 0xE39 ?
2197                                        IWM_NVM_LAR_OFFSET_8000_OLD :
2198                                        IWM_NVM_LAR_OFFSET_8000;
2199
2200                 lar_config = le16_to_cpup(regulatory + lar_offset);
2201                 data->lar_enabled = !!(lar_config &
2202                                        IWM_NVM_LAR_ENABLED_8000);
2203         }
2204
2205         /* If no valid mac address was found - bail out */
2206         if (iwm_set_hw_address(sc, data, nvm_hw, mac_override)) {
2207                 free(data, M_DEVBUF);
2208                 return NULL;
2209         }
2210
2211         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
2212                 memcpy(data->nvm_ch_flags, &nvm_sw[IWM_NVM_CHANNELS],
2213                     IWM_NUM_CHANNELS * sizeof(uint16_t));
2214         } else {
2215                 memcpy(data->nvm_ch_flags, &regulatory[IWM_NVM_CHANNELS_8000],
2216                     IWM_NUM_CHANNELS_8000 * sizeof(uint16_t));
2217         }
2218
2219         return data;
2220 }
2221
2222 static void
2223 iwm_free_nvm_data(struct iwm_nvm_data *data)
2224 {
2225         if (data != NULL)
2226                 free(data, M_DEVBUF);
2227 }
2228
2229 static struct iwm_nvm_data *
2230 iwm_parse_nvm_sections(struct iwm_softc *sc, struct iwm_nvm_section *sections)
2231 {
2232         const uint16_t *hw, *sw, *calib, *regulatory, *mac_override, *phy_sku;
2233
2234         /* Checking for required sections */
2235         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
2236                 if (!sections[IWM_NVM_SECTION_TYPE_SW].data ||
2237                     !sections[sc->cfg->nvm_hw_section_num].data) {
2238                         device_printf(sc->sc_dev,
2239                             "Can't parse empty OTP/NVM sections\n");
2240                         return NULL;
2241                 }
2242         } else if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) {
2243                 /* SW and REGULATORY sections are mandatory */
2244                 if (!sections[IWM_NVM_SECTION_TYPE_SW].data ||
2245                     !sections[IWM_NVM_SECTION_TYPE_REGULATORY].data) {
2246                         device_printf(sc->sc_dev,
2247                             "Can't parse empty OTP/NVM sections\n");
2248                         return NULL;
2249                 }
2250                 /* MAC_OVERRIDE or at least HW section must exist */
2251                 if (!sections[sc->cfg->nvm_hw_section_num].data &&
2252                     !sections[IWM_NVM_SECTION_TYPE_MAC_OVERRIDE].data) {
2253                         device_printf(sc->sc_dev,
2254                             "Can't parse mac_address, empty sections\n");
2255                         return NULL;
2256                 }
2257
2258                 /* PHY_SKU section is mandatory in B0 */
2259                 if (!sections[IWM_NVM_SECTION_TYPE_PHY_SKU].data) {
2260                         device_printf(sc->sc_dev,
2261                             "Can't parse phy_sku in B0, empty sections\n");
2262                         return NULL;
2263                 }
2264         } else {
2265                 panic("unknown device family %d\n", sc->cfg->device_family);
2266         }
2267
2268         hw = (const uint16_t *) sections[sc->cfg->nvm_hw_section_num].data;
2269         sw = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_SW].data;
2270         calib = (const uint16_t *)
2271             sections[IWM_NVM_SECTION_TYPE_CALIBRATION].data;
2272         regulatory = (const uint16_t *)
2273             sections[IWM_NVM_SECTION_TYPE_REGULATORY].data;
2274         mac_override = (const uint16_t *)
2275             sections[IWM_NVM_SECTION_TYPE_MAC_OVERRIDE].data;
2276         phy_sku = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_PHY_SKU].data;
2277
2278         return iwm_parse_nvm_data(sc, hw, sw, calib, mac_override,
2279             phy_sku, regulatory);
2280 }
2281
2282 static int
2283 iwm_nvm_init(struct iwm_softc *sc)
2284 {
2285         struct iwm_nvm_section nvm_sections[IWM_NVM_MAX_NUM_SECTIONS];
2286         int i, ret, section;
2287         uint32_t size_read = 0;
2288         uint8_t *nvm_buffer, *temp;
2289         uint16_t len;
2290
2291         memset(nvm_sections, 0, sizeof(nvm_sections));
2292
2293         if (sc->cfg->nvm_hw_section_num >= IWM_NVM_MAX_NUM_SECTIONS)
2294                 return EINVAL;
2295
2296         /* load NVM values from nic */
2297         /* Read From FW NVM */
2298         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM, "Read from NVM\n");
2299
2300         nvm_buffer = malloc(sc->cfg->eeprom_size, M_DEVBUF, M_NOWAIT | M_ZERO);
2301         if (!nvm_buffer)
2302                 return ENOMEM;
2303         for (section = 0; section < IWM_NVM_MAX_NUM_SECTIONS; section++) {
2304                 /* we override the constness for initial read */
2305                 ret = iwm_nvm_read_section(sc, section, nvm_buffer,
2306                                            &len, size_read);
2307                 if (ret)
2308                         continue;
2309                 size_read += len;
2310                 temp = malloc(len, M_DEVBUF, M_NOWAIT);
2311                 if (!temp) {
2312                         ret = ENOMEM;
2313                         break;
2314                 }
2315                 memcpy(temp, nvm_buffer, len);
2316
2317                 nvm_sections[section].data = temp;
2318                 nvm_sections[section].length = len;
2319         }
2320         if (!size_read)
2321                 device_printf(sc->sc_dev, "OTP is blank\n");
2322         free(nvm_buffer, M_DEVBUF);
2323
2324         sc->nvm_data = iwm_parse_nvm_sections(sc, nvm_sections);
2325         if (!sc->nvm_data)
2326                 return EINVAL;
2327         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
2328                     "nvm version = %x\n", sc->nvm_data->nvm_version);
2329
2330         for (i = 0; i < IWM_NVM_MAX_NUM_SECTIONS; i++) {
2331                 if (nvm_sections[i].data != NULL)
2332                         free(nvm_sections[i].data, M_DEVBUF);
2333         }
2334
2335         return 0;
2336 }
2337
2338 static int
2339 iwm_pcie_load_section(struct iwm_softc *sc, uint8_t section_num,
2340         const struct iwm_fw_desc *section)
2341 {
2342         struct iwm_dma_info *dma = &sc->fw_dma;
2343         uint8_t *v_addr;
2344         bus_addr_t p_addr;
2345         uint32_t offset, chunk_sz = MIN(IWM_FH_MEM_TB_MAX_LENGTH, section->len);
2346         int ret = 0;
2347
2348         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2349                     "%s: [%d] uCode section being loaded...\n",
2350                     __func__, section_num);
2351
2352         v_addr = dma->vaddr;
2353         p_addr = dma->paddr;
2354
2355         for (offset = 0; offset < section->len; offset += chunk_sz) {
2356                 uint32_t copy_size, dst_addr;
2357                 int extended_addr = FALSE;
2358
2359                 copy_size = MIN(chunk_sz, section->len - offset);
2360                 dst_addr = section->offset + offset;
2361
2362                 if (dst_addr >= IWM_FW_MEM_EXTENDED_START &&
2363                     dst_addr <= IWM_FW_MEM_EXTENDED_END)
2364                         extended_addr = TRUE;
2365
2366                 if (extended_addr)
2367                         iwm_set_bits_prph(sc, IWM_LMPM_CHICK,
2368                                           IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE);
2369
2370                 memcpy(v_addr, (const uint8_t *)section->data + offset,
2371                     copy_size);
2372                 bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREWRITE);
2373                 ret = iwm_pcie_load_firmware_chunk(sc, dst_addr, p_addr,
2374                                                    copy_size);
2375
2376                 if (extended_addr)
2377                         iwm_clear_bits_prph(sc, IWM_LMPM_CHICK,
2378                                             IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE);
2379
2380                 if (ret) {
2381                         device_printf(sc->sc_dev,
2382                             "%s: Could not load the [%d] uCode section\n",
2383                             __func__, section_num);
2384                         break;
2385                 }
2386         }
2387
2388         return ret;
2389 }
2390
2391 /*
2392  * ucode
2393  */
2394 static int
2395 iwm_pcie_load_firmware_chunk(struct iwm_softc *sc, uint32_t dst_addr,
2396                              bus_addr_t phy_addr, uint32_t byte_cnt)
2397 {
2398         int ret;
2399
2400         sc->sc_fw_chunk_done = 0;
2401
2402         if (!iwm_nic_lock(sc))
2403                 return EBUSY;
2404
2405         IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(IWM_FH_SRVC_CHNL),
2406             IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE);
2407
2408         IWM_WRITE(sc, IWM_FH_SRVC_CHNL_SRAM_ADDR_REG(IWM_FH_SRVC_CHNL),
2409             dst_addr);
2410
2411         IWM_WRITE(sc, IWM_FH_TFDIB_CTRL0_REG(IWM_FH_SRVC_CHNL),
2412             phy_addr & IWM_FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK);
2413
2414         IWM_WRITE(sc, IWM_FH_TFDIB_CTRL1_REG(IWM_FH_SRVC_CHNL),
2415             (iwm_get_dma_hi_addr(phy_addr)
2416              << IWM_FH_MEM_TFDIB_REG1_ADDR_BITSHIFT) | byte_cnt);
2417
2418         IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_BUF_STS_REG(IWM_FH_SRVC_CHNL),
2419             1 << IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM |
2420             1 << IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX |
2421             IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID);
2422
2423         IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(IWM_FH_SRVC_CHNL),
2424             IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE    |
2425             IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE |
2426             IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD);
2427
2428         iwm_nic_unlock(sc);
2429
2430         /* wait up to 5s for this segment to load */
2431         ret = 0;
2432         while (!sc->sc_fw_chunk_done) {
2433                 ret = msleep(&sc->sc_fw, &sc->sc_mtx, 0, "iwmfw", hz);
2434                 if (ret)
2435                         break;
2436         }
2437
2438         if (ret != 0) {
2439                 device_printf(sc->sc_dev,
2440                     "fw chunk addr 0x%x len %d failed to load\n",
2441                     dst_addr, byte_cnt);
2442                 return ETIMEDOUT;
2443         }
2444
2445         return 0;
2446 }
2447
2448 static int
2449 iwm_pcie_load_cpu_sections_8000(struct iwm_softc *sc,
2450         const struct iwm_fw_sects *image, int cpu, int *first_ucode_section)
2451 {
2452         int shift_param;
2453         int i, ret = 0, sec_num = 0x1;
2454         uint32_t val, last_read_idx = 0;
2455
2456         if (cpu == 1) {
2457                 shift_param = 0;
2458                 *first_ucode_section = 0;
2459         } else {
2460                 shift_param = 16;
2461                 (*first_ucode_section)++;
2462         }
2463
2464         for (i = *first_ucode_section; i < IWM_UCODE_SECTION_MAX; i++) {
2465                 last_read_idx = i;
2466
2467                 /*
2468                  * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
2469                  * CPU1 to CPU2.
2470                  * PAGING_SEPARATOR_SECTION delimiter - separate between
2471                  * CPU2 non paged to CPU2 paging sec.
2472                  */
2473                 if (!image->fw_sect[i].data ||
2474                     image->fw_sect[i].offset == IWM_CPU1_CPU2_SEPARATOR_SECTION ||
2475                     image->fw_sect[i].offset == IWM_PAGING_SEPARATOR_SECTION) {
2476                         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2477                                     "Break since Data not valid or Empty section, sec = %d\n",
2478                                     i);
2479                         break;
2480                 }
2481                 ret = iwm_pcie_load_section(sc, i, &image->fw_sect[i]);
2482                 if (ret)
2483                         return ret;
2484
2485                 /* Notify the ucode of the loaded section number and status */
2486                 if (iwm_nic_lock(sc)) {
2487                         val = IWM_READ(sc, IWM_FH_UCODE_LOAD_STATUS);
2488                         val = val | (sec_num << shift_param);
2489                         IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, val);
2490                         sec_num = (sec_num << 1) | 0x1;
2491                         iwm_nic_unlock(sc);
2492                 }
2493         }
2494
2495         *first_ucode_section = last_read_idx;
2496
2497         iwm_enable_interrupts(sc);
2498
2499         if (iwm_nic_lock(sc)) {
2500                 if (cpu == 1)
2501                         IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, 0xFFFF);
2502                 else
2503                         IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, 0xFFFFFFFF);
2504                 iwm_nic_unlock(sc);
2505         }
2506
2507         return 0;
2508 }
2509
2510 static int
2511 iwm_pcie_load_cpu_sections(struct iwm_softc *sc,
2512         const struct iwm_fw_sects *image, int cpu, int *first_ucode_section)
2513 {
2514         int shift_param;
2515         int i, ret = 0;
2516         uint32_t last_read_idx = 0;
2517
2518         if (cpu == 1) {
2519                 shift_param = 0;
2520                 *first_ucode_section = 0;
2521         } else {
2522                 shift_param = 16;
2523                 (*first_ucode_section)++;
2524         }
2525
2526         for (i = *first_ucode_section; i < IWM_UCODE_SECTION_MAX; i++) {
2527                 last_read_idx = i;
2528
2529                 /*
2530                  * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
2531                  * CPU1 to CPU2.
2532                  * PAGING_SEPARATOR_SECTION delimiter - separate between
2533                  * CPU2 non paged to CPU2 paging sec.
2534                  */
2535                 if (!image->fw_sect[i].data ||
2536                     image->fw_sect[i].offset == IWM_CPU1_CPU2_SEPARATOR_SECTION ||
2537                     image->fw_sect[i].offset == IWM_PAGING_SEPARATOR_SECTION) {
2538                         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2539                                     "Break since Data not valid or Empty section, sec = %d\n",
2540                                      i);
2541                         break;
2542                 }
2543
2544                 ret = iwm_pcie_load_section(sc, i, &image->fw_sect[i]);
2545                 if (ret)
2546                         return ret;
2547         }
2548
2549         *first_ucode_section = last_read_idx;
2550
2551         return 0;
2552
2553 }
2554
2555 static int
2556 iwm_pcie_load_given_ucode(struct iwm_softc *sc,
2557         const struct iwm_fw_sects *image)
2558 {
2559         int ret = 0;
2560         int first_ucode_section;
2561
2562         IWM_DPRINTF(sc, IWM_DEBUG_RESET, "working with %s CPU\n",
2563                      image->is_dual_cpus ? "Dual" : "Single");
2564
2565         /* load to FW the binary non secured sections of CPU1 */
2566         ret = iwm_pcie_load_cpu_sections(sc, image, 1, &first_ucode_section);
2567         if (ret)
2568                 return ret;
2569
2570         if (image->is_dual_cpus) {
2571                 /* set CPU2 header address */
2572                 if (iwm_nic_lock(sc)) {
2573                         iwm_write_prph(sc,
2574                                        IWM_LMPM_SECURE_UCODE_LOAD_CPU2_HDR_ADDR,
2575                                        IWM_LMPM_SECURE_CPU2_HDR_MEM_SPACE);
2576                         iwm_nic_unlock(sc);
2577                 }
2578
2579                 /* load to FW the binary sections of CPU2 */
2580                 ret = iwm_pcie_load_cpu_sections(sc, image, 2,
2581                                                  &first_ucode_section);
2582                 if (ret)
2583                         return ret;
2584         }
2585
2586         iwm_enable_interrupts(sc);
2587
2588         /* release CPU reset */
2589         IWM_WRITE(sc, IWM_CSR_RESET, 0);
2590
2591         return 0;
2592 }
2593
2594 int
2595 iwm_pcie_load_given_ucode_8000(struct iwm_softc *sc,
2596         const struct iwm_fw_sects *image)
2597 {
2598         int ret = 0;
2599         int first_ucode_section;
2600
2601         IWM_DPRINTF(sc, IWM_DEBUG_RESET, "working with %s CPU\n",
2602                     image->is_dual_cpus ? "Dual" : "Single");
2603
2604         /* configure the ucode to be ready to get the secured image */
2605         /* release CPU reset */
2606         if (iwm_nic_lock(sc)) {
2607                 iwm_write_prph(sc, IWM_RELEASE_CPU_RESET,
2608                     IWM_RELEASE_CPU_RESET_BIT);
2609                 iwm_nic_unlock(sc);
2610         }
2611
2612         /* load to FW the binary Secured sections of CPU1 */
2613         ret = iwm_pcie_load_cpu_sections_8000(sc, image, 1,
2614             &first_ucode_section);
2615         if (ret)
2616                 return ret;
2617
2618         /* load to FW the binary sections of CPU2 */
2619         return iwm_pcie_load_cpu_sections_8000(sc, image, 2,
2620             &first_ucode_section);
2621 }
2622
2623 /* XXX Get rid of this definition */
2624 static inline void
2625 iwm_enable_fw_load_int(struct iwm_softc *sc)
2626 {
2627         IWM_DPRINTF(sc, IWM_DEBUG_INTR, "Enabling FW load interrupt\n");
2628         sc->sc_intmask = IWM_CSR_INT_BIT_FH_TX;
2629         IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask);
2630 }
2631
2632 /* XXX Add proper rfkill support code */
2633 static int
2634 iwm_start_fw(struct iwm_softc *sc,
2635         const struct iwm_fw_sects *fw)
2636 {
2637         int ret;
2638
2639         /* This may fail if AMT took ownership of the device */
2640         if (iwm_prepare_card_hw(sc)) {
2641                 device_printf(sc->sc_dev,
2642                     "%s: Exit HW not ready\n", __func__);
2643                 ret = EIO;
2644                 goto out;
2645         }
2646
2647         IWM_WRITE(sc, IWM_CSR_INT, 0xFFFFFFFF);
2648
2649         iwm_disable_interrupts(sc);
2650
2651         /* make sure rfkill handshake bits are cleared */
2652         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL);
2653         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR,
2654             IWM_CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED);
2655
2656         /* clear (again), then enable host interrupts */
2657         IWM_WRITE(sc, IWM_CSR_INT, 0xFFFFFFFF);
2658
2659         ret = iwm_nic_init(sc);
2660         if (ret) {
2661                 device_printf(sc->sc_dev, "%s: Unable to init nic\n", __func__);
2662                 goto out;
2663         }
2664
2665         /*
2666          * Now, we load the firmware and don't want to be interrupted, even
2667          * by the RF-Kill interrupt (hence mask all the interrupt besides the
2668          * FH_TX interrupt which is needed to load the firmware). If the
2669          * RF-Kill switch is toggled, we will find out after having loaded
2670          * the firmware and return the proper value to the caller.
2671          */
2672         iwm_enable_fw_load_int(sc);
2673
2674         /* really make sure rfkill handshake bits are cleared */
2675         /* maybe we should write a few times more?  just to make sure */
2676         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL);
2677         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL);
2678
2679         /* Load the given image to the HW */
2680         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000)
2681                 ret = iwm_pcie_load_given_ucode_8000(sc, fw);
2682         else
2683                 ret = iwm_pcie_load_given_ucode(sc, fw);
2684
2685         /* XXX re-check RF-Kill state */
2686
2687 out:
2688         return ret;
2689 }
2690
2691 static int
2692 iwm_send_tx_ant_cfg(struct iwm_softc *sc, uint8_t valid_tx_ant)
2693 {
2694         struct iwm_tx_ant_cfg_cmd tx_ant_cmd = {
2695                 .valid = htole32(valid_tx_ant),
2696         };
2697
2698         return iwm_mvm_send_cmd_pdu(sc, IWM_TX_ANT_CONFIGURATION_CMD,
2699             IWM_CMD_SYNC, sizeof(tx_ant_cmd), &tx_ant_cmd);
2700 }
2701
2702 /* iwlwifi: mvm/fw.c */
2703 static int
2704 iwm_send_phy_cfg_cmd(struct iwm_softc *sc)
2705 {
2706         struct iwm_phy_cfg_cmd phy_cfg_cmd;
2707         enum iwm_ucode_type ucode_type = sc->cur_ucode;
2708
2709         /* Set parameters */
2710         phy_cfg_cmd.phy_cfg = htole32(iwm_mvm_get_phy_config(sc));
2711         phy_cfg_cmd.calib_control.event_trigger =
2712             sc->sc_default_calib[ucode_type].event_trigger;
2713         phy_cfg_cmd.calib_control.flow_trigger =
2714             sc->sc_default_calib[ucode_type].flow_trigger;
2715
2716         IWM_DPRINTF(sc, IWM_DEBUG_CMD | IWM_DEBUG_RESET,
2717             "Sending Phy CFG command: 0x%x\n", phy_cfg_cmd.phy_cfg);
2718         return iwm_mvm_send_cmd_pdu(sc, IWM_PHY_CONFIGURATION_CMD, IWM_CMD_SYNC,
2719             sizeof(phy_cfg_cmd), &phy_cfg_cmd);
2720 }
2721
2722 static int
2723 iwm_alive_fn(struct iwm_softc *sc, struct iwm_rx_packet *pkt, void *data)
2724 {
2725         struct iwm_mvm_alive_data *alive_data = data;
2726         struct iwm_mvm_alive_resp_ver1 *palive1;
2727         struct iwm_mvm_alive_resp_ver2 *palive2;
2728         struct iwm_mvm_alive_resp *palive;
2729
2730         if (iwm_rx_packet_payload_len(pkt) == sizeof(*palive1)) {
2731                 palive1 = (void *)pkt->data;
2732
2733                 sc->support_umac_log = FALSE;
2734                 sc->error_event_table =
2735                         le32toh(palive1->error_event_table_ptr);
2736                 sc->log_event_table =
2737                         le32toh(palive1->log_event_table_ptr);
2738                 alive_data->scd_base_addr = le32toh(palive1->scd_base_ptr);
2739
2740                 alive_data->valid = le16toh(palive1->status) ==
2741                                     IWM_ALIVE_STATUS_OK;
2742                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2743                             "Alive VER1 ucode status 0x%04x revision 0x%01X 0x%01X flags 0x%01X\n",
2744                              le16toh(palive1->status), palive1->ver_type,
2745                              palive1->ver_subtype, palive1->flags);
2746         } else if (iwm_rx_packet_payload_len(pkt) == sizeof(*palive2)) {
2747                 palive2 = (void *)pkt->data;
2748                 sc->error_event_table =
2749                         le32toh(palive2->error_event_table_ptr);
2750                 sc->log_event_table =
2751                         le32toh(palive2->log_event_table_ptr);
2752                 alive_data->scd_base_addr = le32toh(palive2->scd_base_ptr);
2753                 sc->umac_error_event_table =
2754                         le32toh(palive2->error_info_addr);
2755
2756                 alive_data->valid = le16toh(palive2->status) ==
2757                                     IWM_ALIVE_STATUS_OK;
2758                 if (sc->umac_error_event_table)
2759                         sc->support_umac_log = TRUE;
2760
2761                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2762                             "Alive VER2 ucode status 0x%04x revision 0x%01X 0x%01X flags 0x%01X\n",
2763                             le16toh(palive2->status), palive2->ver_type,
2764                             palive2->ver_subtype, palive2->flags);
2765
2766                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2767                             "UMAC version: Major - 0x%x, Minor - 0x%x\n",
2768                             palive2->umac_major, palive2->umac_minor);
2769         } else if (iwm_rx_packet_payload_len(pkt) == sizeof(*palive)) {
2770                 palive = (void *)pkt->data;
2771
2772                 sc->error_event_table =
2773                         le32toh(palive->error_event_table_ptr);
2774                 sc->log_event_table =
2775                         le32toh(palive->log_event_table_ptr);
2776                 alive_data->scd_base_addr = le32toh(palive->scd_base_ptr);
2777                 sc->umac_error_event_table =
2778                         le32toh(palive->error_info_addr);
2779
2780                 alive_data->valid = le16toh(palive->status) ==
2781                                     IWM_ALIVE_STATUS_OK;
2782                 if (sc->umac_error_event_table)
2783                         sc->support_umac_log = TRUE;
2784
2785                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2786                             "Alive VER3 ucode status 0x%04x revision 0x%01X 0x%01X flags 0x%01X\n",
2787                             le16toh(palive->status), palive->ver_type,
2788                             palive->ver_subtype, palive->flags);
2789
2790                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2791                             "UMAC version: Major - 0x%x, Minor - 0x%x\n",
2792                             le32toh(palive->umac_major),
2793                             le32toh(palive->umac_minor));
2794         }
2795
2796         return TRUE;
2797 }
2798
2799 static int
2800 iwm_wait_phy_db_entry(struct iwm_softc *sc,
2801         struct iwm_rx_packet *pkt, void *data)
2802 {
2803         struct iwm_phy_db *phy_db = data;
2804
2805         if (pkt->hdr.code != IWM_CALIB_RES_NOTIF_PHY_DB) {
2806                 if(pkt->hdr.code != IWM_INIT_COMPLETE_NOTIF) {
2807                         device_printf(sc->sc_dev, "%s: Unexpected cmd: %d\n",
2808                             __func__, pkt->hdr.code);
2809                 }
2810                 return TRUE;
2811         }
2812
2813         if (iwm_phy_db_set_section(phy_db, pkt)) {
2814                 device_printf(sc->sc_dev,
2815                     "%s: iwm_phy_db_set_section failed\n", __func__);
2816         }
2817
2818         return FALSE;
2819 }
2820
2821 static int
2822 iwm_mvm_load_ucode_wait_alive(struct iwm_softc *sc,
2823         enum iwm_ucode_type ucode_type)
2824 {
2825         struct iwm_notification_wait alive_wait;
2826         struct iwm_mvm_alive_data alive_data;
2827         const struct iwm_fw_sects *fw;
2828         enum iwm_ucode_type old_type = sc->cur_ucode;
2829         int error;
2830         static const uint16_t alive_cmd[] = { IWM_MVM_ALIVE };
2831
2832         fw = &sc->sc_fw.fw_sects[ucode_type];
2833         sc->cur_ucode = ucode_type;
2834         sc->ucode_loaded = FALSE;
2835
2836         memset(&alive_data, 0, sizeof(alive_data));
2837         iwm_init_notification_wait(sc->sc_notif_wait, &alive_wait,
2838                                    alive_cmd, nitems(alive_cmd),
2839                                    iwm_alive_fn, &alive_data);
2840
2841         error = iwm_start_fw(sc, fw);
2842         if (error) {
2843                 device_printf(sc->sc_dev, "iwm_start_fw: failed %d\n", error);
2844                 sc->cur_ucode = old_type;
2845                 iwm_remove_notification(sc->sc_notif_wait, &alive_wait);
2846                 return error;
2847         }
2848
2849         /*
2850          * Some things may run in the background now, but we
2851          * just wait for the ALIVE notification here.
2852          */
2853         IWM_UNLOCK(sc);
2854         error = iwm_wait_notification(sc->sc_notif_wait, &alive_wait,
2855                                       IWM_MVM_UCODE_ALIVE_TIMEOUT);
2856         IWM_LOCK(sc);
2857         if (error) {
2858                 if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) {
2859                         uint32_t a = 0x5a5a5a5a, b = 0x5a5a5a5a;
2860                         if (iwm_nic_lock(sc)) {
2861                                 a = iwm_read_prph(sc, IWM_SB_CPU_1_STATUS);
2862                                 b = iwm_read_prph(sc, IWM_SB_CPU_2_STATUS);
2863                                 iwm_nic_unlock(sc);
2864                         }
2865                         device_printf(sc->sc_dev,
2866                             "SecBoot CPU1 Status: 0x%x, CPU2 Status: 0x%x\n",
2867                             a, b);
2868                 }
2869                 sc->cur_ucode = old_type;
2870                 return error;
2871         }
2872
2873         if (!alive_data.valid) {
2874                 device_printf(sc->sc_dev, "%s: Loaded ucode is not valid\n",
2875                     __func__);
2876                 sc->cur_ucode = old_type;
2877                 return EIO;
2878         }
2879
2880         iwm_trans_pcie_fw_alive(sc, alive_data.scd_base_addr);
2881
2882         /*
2883          * configure and operate fw paging mechanism.
2884          * driver configures the paging flow only once, CPU2 paging image
2885          * included in the IWM_UCODE_INIT image.
2886          */
2887         if (fw->paging_mem_size) {
2888                 error = iwm_save_fw_paging(sc, fw);
2889                 if (error) {
2890                         device_printf(sc->sc_dev,
2891                             "%s: failed to save the FW paging image\n",
2892                             __func__);
2893                         return error;
2894                 }
2895
2896                 error = iwm_send_paging_cmd(sc, fw);
2897                 if (error) {
2898                         device_printf(sc->sc_dev,
2899                             "%s: failed to send the paging cmd\n", __func__);
2900                         iwm_free_fw_paging(sc);
2901                         return error;
2902                 }
2903         }
2904
2905         if (!error)
2906                 sc->ucode_loaded = TRUE;
2907         return error;
2908 }
2909
2910 /*
2911  * mvm misc bits
2912  */
2913
2914 /*
2915  * follows iwlwifi/fw.c
2916  */
2917 static int
2918 iwm_run_init_mvm_ucode(struct iwm_softc *sc, int justnvm)
2919 {
2920         struct iwm_notification_wait calib_wait;
2921         static const uint16_t init_complete[] = {
2922                 IWM_INIT_COMPLETE_NOTIF,
2923                 IWM_CALIB_RES_NOTIF_PHY_DB
2924         };
2925         int ret;
2926
2927         /* do not operate with rfkill switch turned on */
2928         if ((sc->sc_flags & IWM_FLAG_RFKILL) && !justnvm) {
2929                 device_printf(sc->sc_dev,
2930                     "radio is disabled by hardware switch\n");
2931                 return EPERM;
2932         }
2933
2934         iwm_init_notification_wait(sc->sc_notif_wait,
2935                                    &calib_wait,
2936                                    init_complete,
2937                                    nitems(init_complete),
2938                                    iwm_wait_phy_db_entry,
2939                                    sc->sc_phy_db);
2940
2941         /* Will also start the device */
2942         ret = iwm_mvm_load_ucode_wait_alive(sc, IWM_UCODE_INIT);
2943         if (ret) {
2944                 device_printf(sc->sc_dev, "Failed to start INIT ucode: %d\n",
2945                     ret);
2946                 goto error;
2947         }
2948
2949         if (justnvm) {
2950                 /* Read nvm */
2951                 ret = iwm_nvm_init(sc);
2952                 if (ret) {
2953                         device_printf(sc->sc_dev, "failed to read nvm\n");
2954                         goto error;
2955                 }
2956                 IEEE80211_ADDR_COPY(sc->sc_ic.ic_macaddr, sc->nvm_data->hw_addr);
2957                 goto error;
2958         }
2959
2960         ret = iwm_send_bt_init_conf(sc);
2961         if (ret) {
2962                 device_printf(sc->sc_dev,
2963                     "failed to send bt coex configuration: %d\n", ret);
2964                 goto error;
2965         }
2966
2967         /* Send TX valid antennas before triggering calibrations */
2968         ret = iwm_send_tx_ant_cfg(sc, iwm_mvm_get_valid_tx_ant(sc));
2969         if (ret) {
2970                 device_printf(sc->sc_dev,
2971                     "failed to send antennas before calibration: %d\n", ret);
2972                 goto error;
2973         }
2974
2975         /*
2976          * Send phy configurations command to init uCode
2977          * to start the 16.0 uCode init image internal calibrations.
2978          */
2979         ret = iwm_send_phy_cfg_cmd(sc);
2980         if (ret) {
2981                 device_printf(sc->sc_dev,
2982                     "%s: Failed to run INIT calibrations: %d\n",
2983                     __func__, ret);
2984                 goto error;
2985         }
2986
2987         /*
2988          * Nothing to do but wait for the init complete notification
2989          * from the firmware.
2990          */
2991         IWM_UNLOCK(sc);
2992         ret = iwm_wait_notification(sc->sc_notif_wait, &calib_wait,
2993             IWM_MVM_UCODE_CALIB_TIMEOUT);
2994         IWM_LOCK(sc);
2995
2996
2997         goto out;
2998
2999 error:
3000         iwm_remove_notification(sc->sc_notif_wait, &calib_wait);
3001 out:
3002         return ret;
3003 }
3004
3005 /*
3006  * receive side
3007  */
3008
3009 /* (re)stock rx ring, called at init-time and at runtime */
3010 static int
3011 iwm_rx_addbuf(struct iwm_softc *sc, int size, int idx)
3012 {
3013         struct iwm_rx_ring *ring = &sc->rxq;
3014         struct iwm_rx_data *data = &ring->data[idx];
3015         struct mbuf *m;
3016         bus_dmamap_t dmamap;
3017         bus_dma_segment_t seg;
3018         int nsegs, error;
3019
3020         m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, IWM_RBUF_SIZE);
3021         if (m == NULL)
3022                 return ENOBUFS;
3023
3024         m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
3025         error = bus_dmamap_load_mbuf_sg(ring->data_dmat, ring->spare_map, m,
3026             &seg, &nsegs, BUS_DMA_NOWAIT);
3027         if (error != 0) {
3028                 device_printf(sc->sc_dev,
3029                     "%s: can't map mbuf, error %d\n", __func__, error);
3030                 m_freem(m);
3031                 return error;
3032         }
3033
3034         if (data->m != NULL)
3035                 bus_dmamap_unload(ring->data_dmat, data->map);
3036
3037         /* Swap ring->spare_map with data->map */
3038         dmamap = data->map;
3039         data->map = ring->spare_map;
3040         ring->spare_map = dmamap;
3041
3042         bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_PREREAD);
3043         data->m = m;
3044
3045         /* Update RX descriptor. */
3046         KASSERT((seg.ds_addr & 255) == 0, ("seg.ds_addr not aligned"));
3047         ring->desc[idx] = htole32(seg.ds_addr >> 8);
3048         bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map,
3049             BUS_DMASYNC_PREWRITE);
3050
3051         return 0;
3052 }
3053
3054 /* iwlwifi: mvm/rx.c */
3055 /*
3056  * iwm_mvm_get_signal_strength - use new rx PHY INFO API
3057  * values are reported by the fw as positive values - need to negate
3058  * to obtain their dBM.  Account for missing antennas by replacing 0
3059  * values by -256dBm: practically 0 power and a non-feasible 8 bit value.
3060  */
3061 static int
3062 iwm_mvm_get_signal_strength(struct iwm_softc *sc, struct iwm_rx_phy_info *phy_info)
3063 {
3064         int energy_a, energy_b, energy_c, max_energy;
3065         uint32_t val;
3066
3067         val = le32toh(phy_info->non_cfg_phy[IWM_RX_INFO_ENERGY_ANT_ABC_IDX]);
3068         energy_a = (val & IWM_RX_INFO_ENERGY_ANT_A_MSK) >>
3069             IWM_RX_INFO_ENERGY_ANT_A_POS;
3070         energy_a = energy_a ? -energy_a : -256;
3071         energy_b = (val & IWM_RX_INFO_ENERGY_ANT_B_MSK) >>
3072             IWM_RX_INFO_ENERGY_ANT_B_POS;
3073         energy_b = energy_b ? -energy_b : -256;
3074         energy_c = (val & IWM_RX_INFO_ENERGY_ANT_C_MSK) >>
3075             IWM_RX_INFO_ENERGY_ANT_C_POS;
3076         energy_c = energy_c ? -energy_c : -256;
3077         max_energy = MAX(energy_a, energy_b);
3078         max_energy = MAX(max_energy, energy_c);
3079
3080         IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3081             "energy In A %d B %d C %d , and max %d\n",
3082             energy_a, energy_b, energy_c, max_energy);
3083
3084         return max_energy;
3085 }
3086
3087 static void
3088 iwm_mvm_rx_rx_phy_cmd(struct iwm_softc *sc, struct iwm_rx_packet *pkt)
3089 {
3090         struct iwm_rx_phy_info *phy_info = (void *)pkt->data;
3091
3092         IWM_DPRINTF(sc, IWM_DEBUG_RECV, "received PHY stats\n");
3093
3094         memcpy(&sc->sc_last_phy_info, phy_info, sizeof(sc->sc_last_phy_info));
3095 }
3096
3097 /*
3098  * Retrieve the average noise (in dBm) among receivers.
3099  */
3100 static int
3101 iwm_get_noise(struct iwm_softc *sc,
3102     const struct iwm_mvm_statistics_rx_non_phy *stats)
3103 {
3104         int i, total, nbant, noise;
3105
3106         total = nbant = noise = 0;
3107         for (i = 0; i < 3; i++) {
3108                 noise = le32toh(stats->beacon_silence_rssi[i]) & 0xff;
3109                 IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: i=%d, noise=%d\n",
3110                     __func__,
3111                     i,
3112                     noise);
3113
3114                 if (noise) {
3115                         total += noise;
3116                         nbant++;
3117                 }
3118         }
3119
3120         IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: nbant=%d, total=%d\n",
3121             __func__, nbant, total);
3122 #if 0
3123         /* There should be at least one antenna but check anyway. */
3124         return (nbant == 0) ? -127 : (total / nbant) - 107;
3125 #else
3126         /* For now, just hard-code it to -96 to be safe */
3127         return (-96);
3128 #endif
3129 }
3130
3131 static void
3132 iwm_mvm_handle_rx_statistics(struct iwm_softc *sc, struct iwm_rx_packet *pkt)
3133 {
3134         struct iwm_notif_statistics_v10 *stats = (void *)&pkt->data;
3135
3136         memcpy(&sc->sc_stats, stats, sizeof(sc->sc_stats));
3137         sc->sc_noise = iwm_get_noise(sc, &stats->rx.general);
3138 }
3139
3140 /*
3141  * iwm_mvm_rx_rx_mpdu - IWM_REPLY_RX_MPDU_CMD handler
3142  *
3143  * Handles the actual data of the Rx packet from the fw
3144  */
3145 static boolean_t
3146 iwm_mvm_rx_rx_mpdu(struct iwm_softc *sc, struct mbuf *m, uint32_t offset,
3147         boolean_t stolen)
3148 {
3149         struct ieee80211com *ic = &sc->sc_ic;
3150         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
3151         struct ieee80211_frame *wh;
3152         struct ieee80211_node *ni;
3153         struct ieee80211_rx_stats rxs;
3154         struct iwm_rx_phy_info *phy_info;
3155         struct iwm_rx_mpdu_res_start *rx_res;
3156         struct iwm_rx_packet *pkt = mtodoff(m, struct iwm_rx_packet *, offset);
3157         uint32_t len;
3158         uint32_t rx_pkt_status;
3159         int rssi;
3160
3161         phy_info = &sc->sc_last_phy_info;
3162         rx_res = (struct iwm_rx_mpdu_res_start *)pkt->data;
3163         wh = (struct ieee80211_frame *)(pkt->data + sizeof(*rx_res));
3164         len = le16toh(rx_res->byte_count);
3165         rx_pkt_status = le32toh(*(uint32_t *)(pkt->data + sizeof(*rx_res) + len));
3166
3167         if (__predict_false(phy_info->cfg_phy_cnt > 20)) {
3168                 device_printf(sc->sc_dev,
3169                     "dsp size out of range [0,20]: %d\n",
3170                     phy_info->cfg_phy_cnt);
3171                 goto fail;
3172         }
3173
3174         if (!(rx_pkt_status & IWM_RX_MPDU_RES_STATUS_CRC_OK) ||
3175             !(rx_pkt_status & IWM_RX_MPDU_RES_STATUS_OVERRUN_OK)) {
3176                 IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3177                     "Bad CRC or FIFO: 0x%08X.\n", rx_pkt_status);
3178                 goto fail;
3179         }
3180
3181         rssi = iwm_mvm_get_signal_strength(sc, phy_info);
3182
3183         /* Map it to relative value */
3184         rssi = rssi - sc->sc_noise;
3185
3186         /* replenish ring for the buffer we're going to feed to the sharks */
3187         if (!stolen && iwm_rx_addbuf(sc, IWM_RBUF_SIZE, sc->rxq.cur) != 0) {
3188                 device_printf(sc->sc_dev, "%s: unable to add more buffers\n",
3189                     __func__);
3190                 goto fail;
3191         }
3192
3193         m->m_data = pkt->data + sizeof(*rx_res);
3194         m->m_pkthdr.len = m->m_len = len;
3195
3196         IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3197             "%s: rssi=%d, noise=%d\n", __func__, rssi, sc->sc_noise);
3198
3199         ni = ieee80211_find_rxnode(ic, (struct ieee80211_frame_min *)wh);
3200
3201         IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3202             "%s: phy_info: channel=%d, flags=0x%08x\n",
3203             __func__,
3204             le16toh(phy_info->channel),
3205             le16toh(phy_info->phy_flags));
3206
3207         /*
3208          * Populate an RX state struct with the provided information.
3209          */
3210         bzero(&rxs, sizeof(rxs));
3211         rxs.r_flags |= IEEE80211_R_IEEE | IEEE80211_R_FREQ;
3212         rxs.r_flags |= IEEE80211_R_NF | IEEE80211_R_RSSI;
3213         rxs.c_ieee = le16toh(phy_info->channel);
3214         if (le16toh(phy_info->phy_flags & IWM_RX_RES_PHY_FLAGS_BAND_24)) {
3215                 rxs.c_freq = ieee80211_ieee2mhz(rxs.c_ieee, IEEE80211_CHAN_2GHZ);
3216         } else {
3217                 rxs.c_freq = ieee80211_ieee2mhz(rxs.c_ieee, IEEE80211_CHAN_5GHZ);
3218         }
3219
3220         /* rssi is in 1/2db units */
3221         rxs.c_rssi = rssi * 2;
3222         rxs.c_nf = sc->sc_noise;
3223         if (ieee80211_add_rx_params(m, &rxs) == 0) {
3224                 if (ni)
3225                         ieee80211_free_node(ni);
3226                 goto fail;
3227         }
3228
3229         if (ieee80211_radiotap_active_vap(vap)) {
3230                 struct iwm_rx_radiotap_header *tap = &sc->sc_rxtap;
3231
3232                 tap->wr_flags = 0;
3233                 if (phy_info->phy_flags & htole16(IWM_PHY_INFO_FLAG_SHPREAMBLE))
3234                         tap->wr_flags |= IEEE80211_RADIOTAP_F_SHORTPRE;
3235                 tap->wr_chan_freq = htole16(rxs.c_freq);
3236                 /* XXX only if ic->ic_curchan->ic_ieee == rxs.c_ieee */
3237                 tap->wr_chan_flags = htole16(ic->ic_curchan->ic_flags);
3238                 tap->wr_dbm_antsignal = (int8_t)rssi;
3239                 tap->wr_dbm_antnoise = (int8_t)sc->sc_noise;
3240                 tap->wr_tsft = phy_info->system_timestamp;
3241                 switch (phy_info->rate) {
3242                 /* CCK rates. */
3243                 case  10: tap->wr_rate =   2; break;
3244                 case  20: tap->wr_rate =   4; break;
3245                 case  55: tap->wr_rate =  11; break;
3246                 case 110: tap->wr_rate =  22; break;
3247                 /* OFDM rates. */
3248                 case 0xd: tap->wr_rate =  12; break;
3249                 case 0xf: tap->wr_rate =  18; break;
3250                 case 0x5: tap->wr_rate =  24; break;
3251                 case 0x7: tap->wr_rate =  36; break;
3252                 case 0x9: tap->wr_rate =  48; break;
3253                 case 0xb: tap->wr_rate =  72; break;
3254                 case 0x1: tap->wr_rate =  96; break;
3255                 case 0x3: tap->wr_rate = 108; break;
3256                 /* Unknown rate: should not happen. */
3257                 default:  tap->wr_rate =   0;
3258                 }
3259         }
3260
3261         IWM_UNLOCK(sc);
3262         if (ni != NULL) {
3263                 IWM_DPRINTF(sc, IWM_DEBUG_RECV, "input m %p\n", m);
3264                 ieee80211_input_mimo(ni, m);
3265                 ieee80211_free_node(ni);
3266         } else {
3267                 IWM_DPRINTF(sc, IWM_DEBUG_RECV, "inputall m %p\n", m);
3268                 ieee80211_input_mimo_all(ic, m);
3269         }
3270         IWM_LOCK(sc);
3271
3272         return TRUE;
3273
3274 fail:
3275         counter_u64_add(ic->ic_ierrors, 1);
3276         return FALSE;
3277 }
3278
3279 static int
3280 iwm_mvm_rx_tx_cmd_single(struct iwm_softc *sc, struct iwm_rx_packet *pkt,
3281         struct iwm_node *in)
3282 {
3283         struct iwm_mvm_tx_resp *tx_resp = (void *)pkt->data;
3284         struct ieee80211_ratectl_tx_status *txs = &sc->sc_txs;
3285         struct ieee80211_node *ni = &in->in_ni;
3286         struct ieee80211vap *vap = ni->ni_vap;
3287         int status = le16toh(tx_resp->status.status) & IWM_TX_STATUS_MSK;
3288         int new_rate, cur_rate = vap->iv_bss->ni_txrate;
3289         boolean_t rate_matched;
3290         uint8_t tx_resp_rate;
3291
3292         KASSERT(tx_resp->frame_count == 1, ("too many frames"));
3293
3294         /* Update rate control statistics. */
3295         IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "%s: status=0x%04x, seq=%d, fc=%d, btc=%d, frts=%d, ff=%d, irate=%08x, wmt=%d\n",
3296             __func__,
3297             (int) le16toh(tx_resp->status.status),
3298             (int) le16toh(tx_resp->status.sequence),
3299             tx_resp->frame_count,
3300             tx_resp->bt_kill_count,
3301             tx_resp->failure_rts,
3302             tx_resp->failure_frame,
3303             le32toh(tx_resp->initial_rate),
3304             (int) le16toh(tx_resp->wireless_media_time));
3305
3306         tx_resp_rate = iwm_rate_from_ucode_rate(le32toh(tx_resp->initial_rate));
3307
3308         /* For rate control, ignore frames sent at different initial rate */
3309         rate_matched = (tx_resp_rate != 0 && tx_resp_rate == cur_rate);
3310
3311         if (tx_resp_rate != 0 && cur_rate != 0 && !rate_matched) {
3312                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3313                     "tx_resp_rate doesn't match ni_txrate (tx_resp_rate=%u "
3314                     "ni_txrate=%d)\n", tx_resp_rate, cur_rate);
3315         }
3316
3317         txs->flags = IEEE80211_RATECTL_STATUS_SHORT_RETRY |
3318                      IEEE80211_RATECTL_STATUS_LONG_RETRY;
3319         txs->short_retries = tx_resp->failure_rts;
3320         txs->long_retries = tx_resp->failure_frame;
3321         if (status != IWM_TX_STATUS_SUCCESS &&
3322             status != IWM_TX_STATUS_DIRECT_DONE) {
3323                 switch (status) {
3324                 case IWM_TX_STATUS_FAIL_SHORT_LIMIT:
3325                         txs->status = IEEE80211_RATECTL_TX_FAIL_SHORT;
3326                         break;
3327                 case IWM_TX_STATUS_FAIL_LONG_LIMIT:
3328                         txs->status = IEEE80211_RATECTL_TX_FAIL_LONG;
3329                         break;
3330                 case IWM_TX_STATUS_FAIL_LIFE_EXPIRE:
3331                         txs->status = IEEE80211_RATECTL_TX_FAIL_EXPIRED;
3332                         break;
3333                 default:
3334                         txs->status = IEEE80211_RATECTL_TX_FAIL_UNSPECIFIED;
3335                         break;
3336                 }
3337         } else {
3338                 txs->status = IEEE80211_RATECTL_TX_SUCCESS;
3339         }
3340
3341         if (rate_matched) {
3342                 ieee80211_ratectl_tx_complete(ni, txs);
3343
3344                 int rix = ieee80211_ratectl_rate(vap->iv_bss, NULL, 0);
3345                 new_rate = vap->iv_bss->ni_txrate;
3346                 if (new_rate != 0 && new_rate != cur_rate) {
3347                         struct iwm_node *in = IWM_NODE(vap->iv_bss);
3348                         iwm_setrates(sc, in, rix);
3349                         iwm_mvm_send_lq_cmd(sc, &in->in_lq, FALSE);
3350                 }
3351         }
3352
3353         return (txs->status != IEEE80211_RATECTL_TX_SUCCESS);
3354 }
3355
3356 static void
3357 iwm_mvm_rx_tx_cmd(struct iwm_softc *sc, struct iwm_rx_packet *pkt)
3358 {
3359         struct iwm_cmd_header *cmd_hdr = &pkt->hdr;
3360         int idx = cmd_hdr->idx;
3361         int qid = cmd_hdr->qid;
3362         struct iwm_tx_ring *ring = &sc->txq[qid];
3363         struct iwm_tx_data *txd = &ring->data[idx];
3364         struct iwm_node *in = txd->in;
3365         struct mbuf *m = txd->m;
3366         int status;
3367
3368         KASSERT(txd->done == 0, ("txd not done"));
3369         KASSERT(txd->in != NULL, ("txd without node"));
3370         KASSERT(txd->m != NULL, ("txd without mbuf"));
3371
3372         sc->sc_tx_timer = 0;
3373
3374         status = iwm_mvm_rx_tx_cmd_single(sc, pkt, in);
3375
3376         /* Unmap and free mbuf. */
3377         bus_dmamap_sync(ring->data_dmat, txd->map, BUS_DMASYNC_POSTWRITE);
3378         bus_dmamap_unload(ring->data_dmat, txd->map);
3379
3380         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3381             "free txd %p, in %p\n", txd, txd->in);
3382         txd->done = 1;
3383         txd->m = NULL;
3384         txd->in = NULL;
3385
3386         ieee80211_tx_complete(&in->in_ni, m, status);
3387
3388         if (--ring->queued < IWM_TX_RING_LOMARK) {
3389                 sc->qfullmsk &= ~(1 << ring->qid);
3390                 if (sc->qfullmsk == 0) {
3391                         iwm_start(sc);
3392                 }
3393         }
3394 }
3395
3396 /*
3397  * transmit side
3398  */
3399
3400 /*
3401  * Process a "command done" firmware notification.  This is where we wakeup
3402  * processes waiting for a synchronous command completion.
3403  * from if_iwn
3404  */
3405 static void
3406 iwm_cmd_done(struct iwm_softc *sc, struct iwm_rx_packet *pkt)
3407 {
3408         struct iwm_tx_ring *ring = &sc->txq[IWM_MVM_CMD_QUEUE];
3409         struct iwm_tx_data *data;
3410
3411         if (pkt->hdr.qid != IWM_MVM_CMD_QUEUE) {
3412                 return; /* Not a command ack. */
3413         }
3414
3415         /* XXX wide commands? */
3416         IWM_DPRINTF(sc, IWM_DEBUG_CMD,
3417             "cmd notification type 0x%x qid %d idx %d\n",
3418             pkt->hdr.code, pkt->hdr.qid, pkt->hdr.idx);
3419
3420         data = &ring->data[pkt->hdr.idx];
3421
3422         /* If the command was mapped in an mbuf, free it. */
3423         if (data->m != NULL) {
3424                 bus_dmamap_sync(ring->data_dmat, data->map,
3425                     BUS_DMASYNC_POSTWRITE);
3426                 bus_dmamap_unload(ring->data_dmat, data->map);
3427                 m_freem(data->m);
3428                 data->m = NULL;
3429         }
3430         wakeup(&ring->desc[pkt->hdr.idx]);
3431
3432         if (((pkt->hdr.idx + ring->queued) % IWM_TX_RING_COUNT) != ring->cur) {
3433                 device_printf(sc->sc_dev,
3434                     "%s: Some HCMDs skipped?: idx=%d queued=%d cur=%d\n",
3435                     __func__, pkt->hdr.idx, ring->queued, ring->cur);
3436                 /* XXX call iwm_force_nmi() */
3437         }
3438
3439         KASSERT(ring->queued > 0, ("ring->queued is empty?"));
3440         ring->queued--;
3441         if (ring->queued == 0)
3442                 iwm_pcie_clear_cmd_in_flight(sc);
3443 }
3444
3445 #if 0
3446 /*
3447  * necessary only for block ack mode
3448  */
3449 void
3450 iwm_update_sched(struct iwm_softc *sc, int qid, int idx, uint8_t sta_id,
3451         uint16_t len)
3452 {
3453         struct iwm_agn_scd_bc_tbl *scd_bc_tbl;
3454         uint16_t w_val;
3455
3456         scd_bc_tbl = sc->sched_dma.vaddr;
3457
3458         len += 8; /* magic numbers came naturally from paris */
3459         len = roundup(len, 4) / 4;
3460
3461         w_val = htole16(sta_id << 12 | len);
3462
3463         /* Update TX scheduler. */
3464         scd_bc_tbl[qid].tfd_offset[idx] = w_val;
3465         bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map,
3466             BUS_DMASYNC_PREWRITE);
3467
3468         /* I really wonder what this is ?!? */
3469         if (idx < IWM_TFD_QUEUE_SIZE_BC_DUP) {
3470                 scd_bc_tbl[qid].tfd_offset[IWM_TFD_QUEUE_SIZE_MAX + idx] = w_val;
3471                 bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map,
3472                     BUS_DMASYNC_PREWRITE);
3473         }
3474 }
3475 #endif
3476
3477 static int
3478 iwm_tx_rateidx_global_lookup(struct iwm_softc *sc, uint8_t rate)
3479 {
3480         int i;
3481
3482         for (i = 0; i < nitems(iwm_rates); i++) {
3483                 if (iwm_rates[i].rate == rate)
3484                         return (i);
3485         }
3486         /* XXX error? */
3487         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE,
3488             "%s: couldn't find an entry for rate=%d\n",
3489             __func__,
3490             rate);
3491         return (0);
3492 }
3493
3494 /*
3495  * Fill in the rate related information for a transmit command.
3496  */
3497 static const struct iwm_rate *
3498 iwm_tx_fill_cmd(struct iwm_softc *sc, struct iwm_node *in,
3499         struct mbuf *m, struct iwm_tx_cmd *tx)
3500 {
3501         struct ieee80211_node *ni = &in->in_ni;
3502         struct ieee80211_frame *wh;
3503         const struct ieee80211_txparam *tp = ni->ni_txparms;
3504         const struct iwm_rate *rinfo;
3505         int type;
3506         int ridx, rate_flags;
3507
3508         wh = mtod(m, struct ieee80211_frame *);
3509         type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK;
3510
3511         tx->rts_retry_limit = IWM_RTS_DFAULT_RETRY_LIMIT;
3512         tx->data_retry_limit = IWM_DEFAULT_TX_RETRY;
3513
3514         if (type == IEEE80211_FC0_TYPE_MGT ||
3515             type == IEEE80211_FC0_TYPE_CTL ||
3516             (m->m_flags & M_EAPOL) != 0) {
3517                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->mgmtrate);
3518                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3519                     "%s: MGT (%d)\n", __func__, tp->mgmtrate);
3520         } else if (IEEE80211_IS_MULTICAST(wh->i_addr1)) {
3521                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->mcastrate);
3522                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3523                     "%s: MCAST (%d)\n", __func__, tp->mcastrate);
3524         } else if (tp->ucastrate != IEEE80211_FIXED_RATE_NONE) {
3525                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->ucastrate);
3526                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3527                     "%s: FIXED_RATE (%d)\n", __func__, tp->ucastrate);
3528         } else {
3529                 /* for data frames, use RS table */
3530                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: DATA\n", __func__);
3531                 ridx = iwm_rate2ridx(sc, ni->ni_txrate);
3532                 if (ridx == -1)
3533                         ridx = 0;
3534
3535                 /* This is the index into the programmed table */
3536                 tx->initial_rate_index = 0;
3537                 tx->tx_flags |= htole32(IWM_TX_CMD_FLG_STA_RATE);
3538         }
3539
3540         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE,
3541             "%s: frame type=%d txrate %d\n",
3542                 __func__, type, iwm_rates[ridx].rate);
3543
3544         rinfo = &iwm_rates[ridx];
3545
3546         IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: ridx=%d; rate=%d, CCK=%d\n",
3547             __func__, ridx,
3548             rinfo->rate,
3549             !! (IWM_RIDX_IS_CCK(ridx))
3550             );
3551
3552         /* XXX TODO: hard-coded TX antenna? */
3553         rate_flags = 1 << IWM_RATE_MCS_ANT_POS;
3554         if (IWM_RIDX_IS_CCK(ridx))
3555                 rate_flags |= IWM_RATE_MCS_CCK_MSK;
3556         tx->rate_n_flags = htole32(rate_flags | rinfo->plcp);
3557
3558         return rinfo;
3559 }
3560
3561 #define TB0_SIZE 16
3562 static int
3563 iwm_tx(struct iwm_softc *sc, struct mbuf *m, struct ieee80211_node *ni, int ac)
3564 {
3565         struct ieee80211com *ic = &sc->sc_ic;
3566         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
3567         struct iwm_node *in = IWM_NODE(ni);
3568         struct iwm_tx_ring *ring;
3569         struct iwm_tx_data *data;
3570         struct iwm_tfd *desc;
3571         struct iwm_device_cmd *cmd;
3572         struct iwm_tx_cmd *tx;
3573         struct ieee80211_frame *wh;
3574         struct ieee80211_key *k = NULL;
3575         struct mbuf *m1;
3576         const struct iwm_rate *rinfo;
3577         uint32_t flags;
3578         u_int hdrlen;
3579         bus_dma_segment_t *seg, segs[IWM_MAX_SCATTER];
3580         int nsegs;
3581         uint8_t tid, type;
3582         int i, totlen, error, pad;
3583
3584         wh = mtod(m, struct ieee80211_frame *);
3585         hdrlen = ieee80211_anyhdrsize(wh);
3586         type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK;
3587         tid = 0;
3588         ring = &sc->txq[ac];
3589         desc = &ring->desc[ring->cur];
3590         memset(desc, 0, sizeof(*desc));
3591         data = &ring->data[ring->cur];
3592
3593         /* Fill out iwm_tx_cmd to send to the firmware */
3594         cmd = &ring->cmd[ring->cur];
3595         cmd->hdr.code = IWM_TX_CMD;
3596         cmd->hdr.flags = 0;
3597         cmd->hdr.qid = ring->qid;
3598         cmd->hdr.idx = ring->cur;
3599
3600         tx = (void *)cmd->data;
3601         memset(tx, 0, sizeof(*tx));
3602
3603         rinfo = iwm_tx_fill_cmd(sc, in, m, tx);
3604
3605         /* Encrypt the frame if need be. */
3606         if (wh->i_fc[1] & IEEE80211_FC1_PROTECTED) {
3607                 /* Retrieve key for TX && do software encryption. */
3608                 k = ieee80211_crypto_encap(ni, m);
3609                 if (k == NULL) {
3610                         m_freem(m);
3611                         return (ENOBUFS);
3612                 }
3613                 /* 802.11 header may have moved. */
3614                 wh = mtod(m, struct ieee80211_frame *);
3615         }
3616
3617         if (ieee80211_radiotap_active_vap(vap)) {
3618                 struct iwm_tx_radiotap_header *tap = &sc->sc_txtap;
3619
3620                 tap->wt_flags = 0;
3621                 tap->wt_chan_freq = htole16(ni->ni_chan->ic_freq);
3622                 tap->wt_chan_flags = htole16(ni->ni_chan->ic_flags);
3623                 tap->wt_rate = rinfo->rate;
3624                 if (k != NULL)
3625                         tap->wt_flags |= IEEE80211_RADIOTAP_F_WEP;
3626                 ieee80211_radiotap_tx(vap, m);
3627         }
3628
3629
3630         totlen = m->m_pkthdr.len;
3631
3632         flags = 0;
3633         if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) {
3634                 flags |= IWM_TX_CMD_FLG_ACK;
3635         }
3636
3637         if (type == IEEE80211_FC0_TYPE_DATA
3638             && (totlen + IEEE80211_CRC_LEN > vap->iv_rtsthreshold)
3639             && !IEEE80211_IS_MULTICAST(wh->i_addr1)) {
3640                 flags |= IWM_TX_CMD_FLG_PROT_REQUIRE;
3641         }
3642
3643         if (IEEE80211_IS_MULTICAST(wh->i_addr1) ||
3644             type != IEEE80211_FC0_TYPE_DATA)
3645                 tx->sta_id = sc->sc_aux_sta.sta_id;
3646         else
3647                 tx->sta_id = IWM_STATION_ID;
3648
3649         if (type == IEEE80211_FC0_TYPE_MGT) {
3650                 uint8_t subtype = wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK;
3651
3652                 if (subtype == IEEE80211_FC0_SUBTYPE_ASSOC_REQ ||
3653                     subtype == IEEE80211_FC0_SUBTYPE_REASSOC_REQ) {
3654                         tx->pm_frame_timeout = htole16(IWM_PM_FRAME_ASSOC);
3655                 } else if (subtype == IEEE80211_FC0_SUBTYPE_ACTION) {
3656                         tx->pm_frame_timeout = htole16(IWM_PM_FRAME_NONE);
3657                 } else {
3658                         tx->pm_frame_timeout = htole16(IWM_PM_FRAME_MGMT);
3659                 }
3660         } else {
3661                 tx->pm_frame_timeout = htole16(IWM_PM_FRAME_NONE);
3662         }
3663
3664         if (hdrlen & 3) {
3665                 /* First segment length must be a multiple of 4. */
3666                 flags |= IWM_TX_CMD_FLG_MH_PAD;
3667                 pad = 4 - (hdrlen & 3);
3668         } else
3669                 pad = 0;
3670
3671         tx->driver_txop = 0;
3672         tx->next_frame_len = 0;
3673
3674         tx->len = htole16(totlen);
3675         tx->tid_tspec = tid;
3676         tx->life_time = htole32(IWM_TX_CMD_LIFE_TIME_INFINITE);
3677
3678         /* Set physical address of "scratch area". */
3679         tx->dram_lsb_ptr = htole32(data->scratch_paddr);
3680         tx->dram_msb_ptr = iwm_get_dma_hi_addr(data->scratch_paddr);
3681
3682         /* Copy 802.11 header in TX command. */
3683         memcpy(((uint8_t *)tx) + sizeof(*tx), wh, hdrlen);
3684
3685         flags |= IWM_TX_CMD_FLG_BT_DIS | IWM_TX_CMD_FLG_SEQ_CTL;
3686
3687         tx->sec_ctl = 0;
3688         tx->tx_flags |= htole32(flags);
3689
3690         /* Trim 802.11 header. */
3691         m_adj(m, hdrlen);
3692         error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m,
3693             segs, &nsegs, BUS_DMA_NOWAIT);
3694         if (error != 0) {
3695                 if (error != EFBIG) {
3696                         device_printf(sc->sc_dev, "can't map mbuf (error %d)\n",
3697                             error);
3698                         m_freem(m);
3699                         return error;
3700                 }
3701                 /* Too many DMA segments, linearize mbuf. */
3702                 m1 = m_collapse(m, M_NOWAIT, IWM_MAX_SCATTER - 2);
3703                 if (m1 == NULL) {
3704                         device_printf(sc->sc_dev,
3705                             "%s: could not defrag mbuf\n", __func__);
3706                         m_freem(m);
3707                         return (ENOBUFS);
3708                 }
3709                 m = m1;
3710
3711                 error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m,
3712                     segs, &nsegs, BUS_DMA_NOWAIT);
3713                 if (error != 0) {
3714                         device_printf(sc->sc_dev, "can't map mbuf (error %d)\n",
3715                             error);
3716                         m_freem(m);
3717                         return error;
3718                 }
3719         }
3720         data->m = m;
3721         data->in = in;
3722         data->done = 0;
3723
3724         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3725             "sending txd %p, in %p\n", data, data->in);
3726         KASSERT(data->in != NULL, ("node is NULL"));
3727
3728         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3729             "sending data: qid=%d idx=%d len=%d nsegs=%d txflags=0x%08x rate_n_flags=0x%08x rateidx=%u\n",
3730             ring->qid, ring->cur, totlen, nsegs,
3731             le32toh(tx->tx_flags),
3732             le32toh(tx->rate_n_flags),
3733             tx->initial_rate_index
3734             );
3735
3736         /* Fill TX descriptor. */
3737         desc->num_tbs = 2 + nsegs;
3738
3739         desc->tbs[0].lo = htole32(data->cmd_paddr);
3740         desc->tbs[0].hi_n_len = htole16(iwm_get_dma_hi_addr(data->cmd_paddr)) |
3741             (TB0_SIZE << 4);
3742         desc->tbs[1].lo = htole32(data->cmd_paddr + TB0_SIZE);
3743         desc->tbs[1].hi_n_len = htole16(iwm_get_dma_hi_addr(data->cmd_paddr)) |
3744             ((sizeof(struct iwm_cmd_header) + sizeof(*tx)
3745               + hdrlen + pad - TB0_SIZE) << 4);
3746
3747         /* Other DMA segments are for data payload. */
3748         for (i = 0; i < nsegs; i++) {
3749                 seg = &segs[i];
3750                 desc->tbs[i+2].lo = htole32(seg->ds_addr);
3751                 desc->tbs[i+2].hi_n_len = \
3752                     htole16(iwm_get_dma_hi_addr(seg->ds_addr))
3753                     | ((seg->ds_len) << 4);
3754         }
3755
3756         bus_dmamap_sync(ring->data_dmat, data->map,
3757             BUS_DMASYNC_PREWRITE);
3758         bus_dmamap_sync(ring->cmd_dma.tag, ring->cmd_dma.map,
3759             BUS_DMASYNC_PREWRITE);
3760         bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map,
3761             BUS_DMASYNC_PREWRITE);
3762
3763 #if 0
3764         iwm_update_sched(sc, ring->qid, ring->cur, tx->sta_id, le16toh(tx->len));
3765 #endif
3766
3767         /* Kick TX ring. */
3768         ring->cur = (ring->cur + 1) % IWM_TX_RING_COUNT;
3769         IWM_WRITE(sc, IWM_HBUS_TARG_WRPTR, ring->qid << 8 | ring->cur);
3770
3771         /* Mark TX ring as full if we reach a certain threshold. */
3772         if (++ring->queued > IWM_TX_RING_HIMARK) {
3773                 sc->qfullmsk |= 1 << ring->qid;
3774         }
3775
3776         return 0;
3777 }
3778
3779 static int
3780 iwm_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
3781     const struct ieee80211_bpf_params *params)
3782 {
3783         struct ieee80211com *ic = ni->ni_ic;
3784         struct iwm_softc *sc = ic->ic_softc;
3785         int error = 0;
3786
3787         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3788             "->%s begin\n", __func__);
3789
3790         if ((sc->sc_flags & IWM_FLAG_HW_INITED) == 0) {
3791                 m_freem(m);
3792                 IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3793                     "<-%s not RUNNING\n", __func__);
3794                 return (ENETDOWN);
3795         }
3796
3797         IWM_LOCK(sc);
3798         /* XXX fix this */
3799         if (params == NULL) {
3800                 error = iwm_tx(sc, m, ni, 0);
3801         } else {
3802                 error = iwm_tx(sc, m, ni, 0);
3803         }
3804         sc->sc_tx_timer = 5;
3805         IWM_UNLOCK(sc);
3806
3807         return (error);
3808 }
3809
3810 /*
3811  * mvm/tx.c
3812  */
3813
3814 /*
3815  * Note that there are transports that buffer frames before they reach
3816  * the firmware. This means that after flush_tx_path is called, the
3817  * queue might not be empty. The race-free way to handle this is to:
3818  * 1) set the station as draining
3819  * 2) flush the Tx path
3820  * 3) wait for the transport queues to be empty
3821  */
3822 int
3823 iwm_mvm_flush_tx_path(struct iwm_softc *sc, uint32_t tfd_msk, uint32_t flags)
3824 {
3825         int ret;
3826         struct iwm_tx_path_flush_cmd flush_cmd = {
3827                 .queues_ctl = htole32(tfd_msk),
3828                 .flush_ctl = htole16(IWM_DUMP_TX_FIFO_FLUSH),
3829         };
3830
3831         ret = iwm_mvm_send_cmd_pdu(sc, IWM_TXPATH_FLUSH, flags,
3832             sizeof(flush_cmd), &flush_cmd);
3833         if (ret)
3834                 device_printf(sc->sc_dev,
3835                     "Flushing tx queue failed: %d\n", ret);
3836         return ret;
3837 }
3838
3839 /*
3840  * BEGIN mvm/quota.c
3841  */
3842
3843 static int
3844 iwm_mvm_update_quotas(struct iwm_softc *sc, struct iwm_vap *ivp)
3845 {
3846         struct iwm_time_quota_cmd cmd;
3847         int i, idx, ret, num_active_macs, quota, quota_rem;
3848         int colors[IWM_MAX_BINDINGS] = { -1, -1, -1, -1, };
3849         int n_ifs[IWM_MAX_BINDINGS] = {0, };
3850         uint16_t id;
3851
3852         memset(&cmd, 0, sizeof(cmd));
3853
3854         /* currently, PHY ID == binding ID */
3855         if (ivp) {
3856                 id = ivp->phy_ctxt->id;
3857                 KASSERT(id < IWM_MAX_BINDINGS, ("invalid id"));
3858                 colors[id] = ivp->phy_ctxt->color;
3859
3860                 if (1)
3861                         n_ifs[id] = 1;
3862         }
3863
3864         /*
3865          * The FW's scheduling session consists of
3866          * IWM_MVM_MAX_QUOTA fragments. Divide these fragments
3867          * equally between all the bindings that require quota
3868          */
3869         num_active_macs = 0;
3870         for (i = 0; i < IWM_MAX_BINDINGS; i++) {
3871                 cmd.quotas[i].id_and_color = htole32(IWM_FW_CTXT_INVALID);
3872                 num_active_macs += n_ifs[i];
3873         }
3874
3875         quota = 0;
3876         quota_rem = 0;
3877         if (num_active_macs) {
3878                 quota = IWM_MVM_MAX_QUOTA / num_active_macs;
3879                 quota_rem = IWM_MVM_MAX_QUOTA % num_active_macs;
3880         }
3881
3882         for (idx = 0, i = 0; i < IWM_MAX_BINDINGS; i++) {
3883                 if (colors[i] < 0)
3884                         continue;
3885
3886                 cmd.quotas[idx].id_and_color =
3887                         htole32(IWM_FW_CMD_ID_AND_COLOR(i, colors[i]));
3888
3889                 if (n_ifs[i] <= 0) {
3890                         cmd.quotas[idx].quota = htole32(0);
3891                         cmd.quotas[idx].max_duration = htole32(0);
3892                 } else {
3893                         cmd.quotas[idx].quota = htole32(quota * n_ifs[i]);
3894                         cmd.quotas[idx].max_duration = htole32(0);
3895                 }
3896                 idx++;
3897         }
3898
3899         /* Give the remainder of the session to the first binding */
3900         cmd.quotas[0].quota = htole32(le32toh(cmd.quotas[0].quota) + quota_rem);
3901
3902         ret = iwm_mvm_send_cmd_pdu(sc, IWM_TIME_QUOTA_CMD, IWM_CMD_SYNC,
3903             sizeof(cmd), &cmd);
3904         if (ret)
3905                 device_printf(sc->sc_dev,
3906                     "%s: Failed to send quota: %d\n", __func__, ret);
3907         return ret;
3908 }
3909
3910 /*
3911  * END mvm/quota.c
3912  */
3913
3914 /*
3915  * ieee80211 routines
3916  */
3917
3918 /*
3919  * Change to AUTH state in 80211 state machine.  Roughly matches what
3920  * Linux does in bss_info_changed().
3921  */
3922 static int
3923 iwm_auth(struct ieee80211vap *vap, struct iwm_softc *sc)
3924 {
3925         struct ieee80211_node *ni;
3926         struct iwm_node *in;
3927         struct iwm_vap *iv = IWM_VAP(vap);
3928         uint32_t duration;
3929         int error;
3930
3931         /*
3932          * XXX i have a feeling that the vap node is being
3933          * freed from underneath us. Grr.
3934          */
3935         ni = ieee80211_ref_node(vap->iv_bss);
3936         in = IWM_NODE(ni);
3937         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_STATE,
3938             "%s: called; vap=%p, bss ni=%p\n",
3939             __func__,
3940             vap,
3941             ni);
3942
3943         in->in_assoc = 0;
3944
3945         /*
3946          * Firmware bug - it'll crash if the beacon interval is less
3947          * than 16. We can't avoid connecting at all, so refuse the
3948          * station state change, this will cause net80211 to abandon
3949          * attempts to connect to this AP, and eventually wpa_s will
3950          * blacklist the AP...
3951          */
3952         if (ni->ni_intval < 16) {
3953                 device_printf(sc->sc_dev,
3954                     "AP %s beacon interval is %d, refusing due to firmware bug!\n",
3955                     ether_sprintf(ni->ni_bssid), ni->ni_intval);
3956                 error = EINVAL;
3957                 goto out;
3958         }
3959
3960         error = iwm_allow_mcast(vap, sc);
3961         if (error) {
3962                 device_printf(sc->sc_dev,
3963                     "%s: failed to set multicast\n", __func__);
3964                 goto out;
3965         }
3966
3967         /*
3968          * This is where it deviates from what Linux does.
3969          *
3970          * Linux iwlwifi doesn't reset the nic each time, nor does it
3971          * call ctxt_add() here.  Instead, it adds it during vap creation,
3972          * and always does a mac_ctx_changed().
3973          *
3974          * The openbsd port doesn't attempt to do that - it reset things
3975          * at odd states and does the add here.
3976          *
3977          * So, until the state handling is fixed (ie, we never reset
3978          * the NIC except for a firmware failure, which should drag
3979          * the NIC back to IDLE, re-setup and re-add all the mac/phy
3980          * contexts that are required), let's do a dirty hack here.
3981          */
3982         if (iv->is_uploaded) {
3983                 if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) {
3984                         device_printf(sc->sc_dev,
3985                             "%s: failed to update MAC\n", __func__);
3986                         goto out;
3987                 }
3988         } else {
3989                 if ((error = iwm_mvm_mac_ctxt_add(sc, vap)) != 0) {
3990                         device_printf(sc->sc_dev,
3991                             "%s: failed to add MAC\n", __func__);
3992                         goto out;
3993                 }
3994         }
3995
3996         if ((error = iwm_mvm_phy_ctxt_changed(sc, &sc->sc_phyctxt[0],
3997             in->in_ni.ni_chan, 1, 1)) != 0) {
3998                 device_printf(sc->sc_dev,
3999                     "%s: failed update phy ctxt\n", __func__);
4000                 goto out;
4001         }
4002         iv->phy_ctxt = &sc->sc_phyctxt[0];
4003
4004         if ((error = iwm_mvm_binding_add_vif(sc, iv)) != 0) {
4005                 device_printf(sc->sc_dev,
4006                     "%s: binding update cmd\n", __func__);
4007                 goto out;
4008         }
4009         /*
4010          * Authentication becomes unreliable when powersaving is left enabled
4011          * here. Powersaving will be activated again when association has
4012          * finished or is aborted.
4013          */
4014         iv->ps_disabled = TRUE;
4015         error = iwm_mvm_power_update_mac(sc);
4016         iv->ps_disabled = FALSE;
4017         if (error != 0) {
4018                 device_printf(sc->sc_dev,
4019                     "%s: failed to update power management\n",
4020                     __func__);
4021                 goto out;
4022         }
4023         if ((error = iwm_mvm_add_sta(sc, in)) != 0) {
4024                 device_printf(sc->sc_dev,
4025                     "%s: failed to add sta\n", __func__);
4026                 goto out;
4027         }
4028
4029         /*
4030          * Prevent the FW from wandering off channel during association
4031          * by "protecting" the session with a time event.
4032          */
4033         /* XXX duration is in units of TU, not MS */
4034         duration = IWM_MVM_TE_SESSION_PROTECTION_MAX_TIME_MS;
4035         iwm_mvm_protect_session(sc, iv, duration, 500 /* XXX magic number */);
4036         DELAY(100);
4037
4038         error = 0;
4039 out:
4040         ieee80211_free_node(ni);
4041         return (error);
4042 }
4043
4044 static int
4045 iwm_release(struct iwm_softc *sc, struct iwm_node *in)
4046 {
4047         uint32_t tfd_msk;
4048
4049         /*
4050          * Ok, so *technically* the proper set of calls for going
4051          * from RUN back to SCAN is:
4052          *
4053          * iwm_mvm_power_mac_disable(sc, in);
4054          * iwm_mvm_mac_ctxt_changed(sc, vap);
4055          * iwm_mvm_rm_sta(sc, in);
4056          * iwm_mvm_update_quotas(sc, NULL);
4057          * iwm_mvm_mac_ctxt_changed(sc, in);
4058          * iwm_mvm_binding_remove_vif(sc, IWM_VAP(in->in_ni.ni_vap));
4059          * iwm_mvm_mac_ctxt_remove(sc, in);
4060          *
4061          * However, that freezes the device not matter which permutations
4062          * and modifications are attempted.  Obviously, this driver is missing
4063          * something since it works in the Linux driver, but figuring out what
4064          * is missing is a little more complicated.  Now, since we're going
4065          * back to nothing anyway, we'll just do a complete device reset.
4066          * Up your's, device!
4067          */
4068         /*
4069          * Just using 0xf for the queues mask is fine as long as we only
4070          * get here from RUN state.
4071          */
4072         tfd_msk = 0xf;
4073         iwm_xmit_queue_drain(sc);
4074         iwm_mvm_flush_tx_path(sc, tfd_msk, IWM_CMD_SYNC);
4075         /*
4076          * We seem to get away with just synchronously sending the
4077          * IWM_TXPATH_FLUSH command.
4078          */
4079 //      iwm_trans_wait_tx_queue_empty(sc, tfd_msk);
4080         iwm_stop_device(sc);
4081         iwm_init_hw(sc);
4082         if (in)
4083                 in->in_assoc = 0;
4084         return 0;
4085
4086 #if 0
4087         int error;
4088
4089         iwm_mvm_power_mac_disable(sc, in);
4090
4091         if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) {
4092                 device_printf(sc->sc_dev, "mac ctxt change fail 1 %d\n", error);
4093                 return error;
4094         }
4095
4096         if ((error = iwm_mvm_rm_sta(sc, in)) != 0) {
4097                 device_printf(sc->sc_dev, "sta remove fail %d\n", error);
4098                 return error;
4099         }
4100         error = iwm_mvm_rm_sta(sc, in);
4101         in->in_assoc = 0;
4102         iwm_mvm_update_quotas(sc, NULL);
4103         if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) {
4104                 device_printf(sc->sc_dev, "mac ctxt change fail 2 %d\n", error);
4105                 return error;
4106         }
4107         iwm_mvm_binding_remove_vif(sc, IWM_VAP(in->in_ni.ni_vap));
4108
4109         iwm_mvm_mac_ctxt_remove(sc, in);
4110
4111         return error;
4112 #endif
4113 }
4114
4115 static struct ieee80211_node *
4116 iwm_node_alloc(struct ieee80211vap *vap, const uint8_t mac[IEEE80211_ADDR_LEN])
4117 {
4118         return malloc(sizeof (struct iwm_node), M_80211_NODE,
4119             M_NOWAIT | M_ZERO);
4120 }
4121
4122 static uint8_t
4123 iwm_rate_from_ucode_rate(uint32_t rate_n_flags)
4124 {
4125         uint8_t plcp = rate_n_flags & 0xff;
4126         int i;
4127
4128         for (i = 0; i <= IWM_RIDX_MAX; i++) {
4129                 if (iwm_rates[i].plcp == plcp)
4130                         return iwm_rates[i].rate;
4131         }
4132         return 0;
4133 }
4134
4135 uint8_t
4136 iwm_ridx2rate(struct ieee80211_rateset *rs, int ridx)
4137 {
4138         int i;
4139         uint8_t rval;
4140
4141         for (i = 0; i < rs->rs_nrates; i++) {
4142                 rval = (rs->rs_rates[i] & IEEE80211_RATE_VAL);
4143                 if (rval == iwm_rates[ridx].rate)
4144                         return rs->rs_rates[i];
4145         }
4146
4147         return 0;
4148 }
4149
4150 static int
4151 iwm_rate2ridx(struct iwm_softc *sc, uint8_t rate)
4152 {
4153         int i;
4154
4155         for (i = 0; i <= IWM_RIDX_MAX; i++) {
4156                 if (iwm_rates[i].rate == rate)
4157                         return i;
4158         }
4159
4160         device_printf(sc->sc_dev,
4161             "%s: WARNING: device rate for %u not found!\n",
4162             __func__, rate);
4163
4164         return -1;
4165 }
4166
4167
4168 static void
4169 iwm_setrates(struct iwm_softc *sc, struct iwm_node *in, int rix)
4170 {
4171         struct ieee80211_node *ni = &in->in_ni;
4172         struct iwm_lq_cmd *lq = &in->in_lq;
4173         struct ieee80211_rateset *rs = &ni->ni_rates;
4174         int nrates = rs->rs_nrates;
4175         int i, ridx, tab = 0;
4176 //      int txant = 0;
4177
4178         KASSERT(rix >= 0 && rix < nrates, ("invalid rix"));
4179
4180         if (nrates > nitems(lq->rs_table)) {
4181                 device_printf(sc->sc_dev,
4182                     "%s: node supports %d rates, driver handles "
4183                     "only %zu\n", __func__, nrates, nitems(lq->rs_table));
4184                 return;
4185         }
4186         if (nrates == 0) {
4187                 device_printf(sc->sc_dev,
4188                     "%s: node supports 0 rates, odd!\n", __func__);
4189                 return;
4190         }
4191         nrates = imin(rix + 1, nrates);
4192
4193         IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
4194             "%s: nrates=%d\n", __func__, nrates);
4195
4196         /* then construct a lq_cmd based on those */
4197         memset(lq, 0, sizeof(*lq));
4198         lq->sta_id = IWM_STATION_ID;
4199
4200         /* For HT, always enable RTS/CTS to avoid excessive retries. */
4201         if (ni->ni_flags & IEEE80211_NODE_HT)
4202                 lq->flags |= IWM_LQ_FLAG_USE_RTS_MSK;
4203
4204         /*
4205          * are these used? (we don't do SISO or MIMO)
4206          * need to set them to non-zero, though, or we get an error.
4207          */
4208         lq->single_stream_ant_msk = 1;
4209         lq->dual_stream_ant_msk = 1;
4210
4211         /*
4212          * Build the actual rate selection table.
4213          * The lowest bits are the rates.  Additionally,
4214          * CCK needs bit 9 to be set.  The rest of the bits
4215          * we add to the table select the tx antenna
4216          * Note that we add the rates in the highest rate first
4217          * (opposite of ni_rates).
4218          */
4219         for (i = 0; i < nrates; i++) {
4220                 int rate = rs->rs_rates[rix - i] & IEEE80211_RATE_VAL;
4221                 int nextant;
4222
4223                 /* Map 802.11 rate to HW rate index. */
4224                 ridx = iwm_rate2ridx(sc, rate);
4225                 if (ridx == -1)
4226                         continue;
4227
4228 #if 0
4229                 if (txant == 0)
4230                         txant = iwm_mvm_get_valid_tx_ant(sc);
4231                 nextant = 1<<(ffs(txant)-1);
4232                 txant &= ~nextant;
4233 #else
4234                 nextant = iwm_mvm_get_valid_tx_ant(sc);
4235 #endif
4236                 tab = iwm_rates[ridx].plcp;
4237                 tab |= nextant << IWM_RATE_MCS_ANT_POS;
4238                 if (IWM_RIDX_IS_CCK(ridx))
4239                         tab |= IWM_RATE_MCS_CCK_MSK;
4240                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
4241                     "station rate i=%d, rate=%d, hw=%x\n",
4242                     i, iwm_rates[ridx].rate, tab);
4243                 lq->rs_table[i] = htole32(tab);
4244         }
4245         /* then fill the rest with the lowest possible rate */
4246         for (i = nrates; i < nitems(lq->rs_table); i++) {
4247                 KASSERT(tab != 0, ("invalid tab"));
4248                 lq->rs_table[i] = htole32(tab);
4249         }
4250 }
4251
4252 static int
4253 iwm_media_change(struct ifnet *ifp)
4254 {
4255         struct ieee80211vap *vap = ifp->if_softc;
4256         struct ieee80211com *ic = vap->iv_ic;
4257         struct iwm_softc *sc = ic->ic_softc;
4258         int error;
4259
4260         error = ieee80211_media_change(ifp);
4261         if (error != ENETRESET)
4262                 return error;
4263
4264         IWM_LOCK(sc);
4265         if (ic->ic_nrunning > 0) {
4266                 iwm_stop(sc);
4267                 iwm_init(sc);
4268         }
4269         IWM_UNLOCK(sc);
4270         return error;
4271 }
4272
4273
4274 static int
4275 iwm_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
4276 {
4277         struct iwm_vap *ivp = IWM_VAP(vap);
4278         struct ieee80211com *ic = vap->iv_ic;
4279         struct iwm_softc *sc = ic->ic_softc;
4280         struct iwm_node *in;
4281         int error;
4282
4283         IWM_DPRINTF(sc, IWM_DEBUG_STATE,
4284             "switching state %s -> %s\n",
4285             ieee80211_state_name[vap->iv_state],
4286             ieee80211_state_name[nstate]);
4287         IEEE80211_UNLOCK(ic);
4288         IWM_LOCK(sc);
4289
4290         if (vap->iv_state == IEEE80211_S_SCAN && nstate != vap->iv_state)
4291                 iwm_led_blink_stop(sc);
4292
4293         /* disable beacon filtering if we're hopping out of RUN */
4294         if (vap->iv_state == IEEE80211_S_RUN && nstate != vap->iv_state) {
4295                 iwm_mvm_disable_beacon_filter(sc);
4296
4297                 if (((in = IWM_NODE(vap->iv_bss)) != NULL))
4298                         in->in_assoc = 0;
4299
4300                 if (nstate == IEEE80211_S_INIT) {
4301                         IWM_UNLOCK(sc);
4302                         IEEE80211_LOCK(ic);
4303                         error = ivp->iv_newstate(vap, nstate, arg);
4304                         IEEE80211_UNLOCK(ic);
4305                         IWM_LOCK(sc);
4306                         iwm_release(sc, NULL);
4307                         IWM_UNLOCK(sc);
4308                         IEEE80211_LOCK(ic);
4309                         return error;
4310                 }
4311
4312                 /*
4313                  * It's impossible to directly go RUN->SCAN. If we iwm_release()
4314                  * above then the card will be completely reinitialized,
4315                  * so the driver must do everything necessary to bring the card
4316                  * from INIT to SCAN.
4317                  *
4318                  * Additionally, upon receiving deauth frame from AP,
4319                  * OpenBSD 802.11 stack puts the driver in IEEE80211_S_AUTH
4320                  * state. This will also fail with this driver, so bring the FSM
4321                  * from IEEE80211_S_RUN to IEEE80211_S_SCAN in this case as well.
4322                  *
4323                  * XXX TODO: fix this for FreeBSD!
4324                  */
4325                 if (nstate == IEEE80211_S_SCAN ||
4326                     nstate == IEEE80211_S_AUTH ||
4327                     nstate == IEEE80211_S_ASSOC) {
4328                         IWM_DPRINTF(sc, IWM_DEBUG_STATE,
4329                             "Force transition to INIT; MGT=%d\n", arg);
4330                         IWM_UNLOCK(sc);
4331                         IEEE80211_LOCK(ic);
4332                         /* Always pass arg as -1 since we can't Tx right now. */
4333                         /*
4334                          * XXX arg is just ignored anyway when transitioning
4335                          *     to IEEE80211_S_INIT.
4336                          */
4337                         vap->iv_newstate(vap, IEEE80211_S_INIT, -1);
4338                         IWM_DPRINTF(sc, IWM_DEBUG_STATE,
4339                             "Going INIT->SCAN\n");
4340                         nstate = IEEE80211_S_SCAN;
4341                         IEEE80211_UNLOCK(ic);
4342                         IWM_LOCK(sc);
4343                 }
4344         }
4345
4346         switch (nstate) {
4347         case IEEE80211_S_INIT:
4348         case IEEE80211_S_SCAN:
4349                 if (vap->iv_state == IEEE80211_S_AUTH ||
4350                     vap->iv_state == IEEE80211_S_ASSOC) {
4351                         int myerr;
4352                         IWM_UNLOCK(sc);
4353                         IEEE80211_LOCK(ic);
4354                         myerr = ivp->iv_newstate(vap, nstate, arg);
4355                         IEEE80211_UNLOCK(ic);
4356                         IWM_LOCK(sc);
4357                         error = iwm_mvm_rm_sta(sc, vap, FALSE);
4358                         if (error) {
4359                                 device_printf(sc->sc_dev,
4360                                     "%s: Failed to remove station: %d\n",
4361                                     __func__, error);
4362                         }
4363                         error = iwm_mvm_mac_ctxt_changed(sc, vap);
4364                         if (error) {
4365                                 device_printf(sc->sc_dev,
4366                                     "%s: Failed to change mac context: %d\n",
4367                                     __func__, error);
4368                         }
4369                         error = iwm_mvm_binding_remove_vif(sc, ivp);
4370                         if (error) {
4371                                 device_printf(sc->sc_dev,
4372                                     "%s: Failed to remove channel ctx: %d\n",
4373                                     __func__, error);
4374                         }
4375                         ivp->phy_ctxt = NULL;
4376                         error = iwm_mvm_power_update_mac(sc);
4377                         if (error != 0) {
4378                                 device_printf(sc->sc_dev,
4379                                     "%s: failed to update power management\n",
4380                                     __func__);
4381                         }
4382                         IWM_UNLOCK(sc);
4383                         IEEE80211_LOCK(ic);
4384                         return myerr;
4385                 }
4386                 break;
4387
4388         case IEEE80211_S_AUTH:
4389                 if ((error = iwm_auth(vap, sc)) != 0) {
4390                         device_printf(sc->sc_dev,
4391                             "%s: could not move to auth state: %d\n",
4392                             __func__, error);
4393                 }
4394                 break;
4395
4396         case IEEE80211_S_ASSOC:
4397                 /*
4398                  * EBS may be disabled due to previous failures reported by FW.
4399                  * Reset EBS status here assuming environment has been changed.
4400                  */
4401                 sc->last_ebs_successful = TRUE;
4402                 break;
4403
4404         case IEEE80211_S_RUN:
4405                 in = IWM_NODE(vap->iv_bss);
4406                 /* Update the association state, now we have it all */
4407                 /* (eg associd comes in at this point */
4408                 error = iwm_mvm_update_sta(sc, in);
4409                 if (error != 0) {
4410                         device_printf(sc->sc_dev,
4411                             "%s: failed to update STA\n", __func__);
4412                         IWM_UNLOCK(sc);
4413                         IEEE80211_LOCK(ic);
4414                         return error;
4415                 }
4416                 in->in_assoc = 1;
4417                 error = iwm_mvm_mac_ctxt_changed(sc, vap);
4418                 if (error != 0) {
4419                         device_printf(sc->sc_dev,
4420                             "%s: failed to update MAC: %d\n", __func__, error);
4421                 }
4422
4423                 iwm_mvm_sf_update(sc, vap, FALSE);
4424                 iwm_mvm_enable_beacon_filter(sc, ivp);
4425                 iwm_mvm_power_update_mac(sc);
4426                 iwm_mvm_update_quotas(sc, ivp);
4427                 int rix = ieee80211_ratectl_rate(&in->in_ni, NULL, 0);
4428                 iwm_setrates(sc, in, rix);
4429
4430                 if ((error = iwm_mvm_send_lq_cmd(sc, &in->in_lq, TRUE)) != 0) {
4431                         device_printf(sc->sc_dev,
4432                             "%s: IWM_LQ_CMD failed: %d\n", __func__, error);
4433                 }
4434
4435                 iwm_mvm_led_enable(sc);
4436                 break;
4437
4438         default:
4439                 break;
4440         }
4441         IWM_UNLOCK(sc);
4442         IEEE80211_LOCK(ic);
4443
4444         return (ivp->iv_newstate(vap, nstate, arg));
4445 }
4446
4447 void
4448 iwm_endscan_cb(void *arg, int pending)
4449 {
4450         struct iwm_softc *sc = arg;
4451         struct ieee80211com *ic = &sc->sc_ic;
4452
4453         IWM_DPRINTF(sc, IWM_DEBUG_SCAN | IWM_DEBUG_TRACE,
4454             "%s: scan ended\n",
4455             __func__);
4456
4457         ieee80211_scan_done(TAILQ_FIRST(&ic->ic_vaps));
4458 }
4459
4460 static int
4461 iwm_send_bt_init_conf(struct iwm_softc *sc)
4462 {
4463         struct iwm_bt_coex_cmd bt_cmd;
4464
4465         bt_cmd.mode = htole32(IWM_BT_COEX_WIFI);
4466         bt_cmd.enabled_modules = htole32(IWM_BT_COEX_HIGH_BAND_RET);
4467
4468         return iwm_mvm_send_cmd_pdu(sc, IWM_BT_CONFIG, 0, sizeof(bt_cmd),
4469             &bt_cmd);
4470 }
4471
4472 static boolean_t
4473 iwm_mvm_is_lar_supported(struct iwm_softc *sc)
4474 {
4475         boolean_t nvm_lar = sc->nvm_data->lar_enabled;
4476         boolean_t tlv_lar = fw_has_capa(&sc->ucode_capa,
4477                                         IWM_UCODE_TLV_CAPA_LAR_SUPPORT);
4478
4479         if (iwm_lar_disable)
4480                 return FALSE;
4481
4482         /*
4483          * Enable LAR only if it is supported by the FW (TLV) &&
4484          * enabled in the NVM
4485          */
4486         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000)
4487                 return nvm_lar && tlv_lar;
4488         else
4489                 return tlv_lar;
4490 }
4491
4492 static boolean_t
4493 iwm_mvm_is_wifi_mcc_supported(struct iwm_softc *sc)
4494 {
4495         return fw_has_api(&sc->ucode_capa,
4496                           IWM_UCODE_TLV_API_WIFI_MCC_UPDATE) ||
4497                fw_has_capa(&sc->ucode_capa,
4498                            IWM_UCODE_TLV_CAPA_LAR_MULTI_MCC);
4499 }
4500
4501 static int
4502 iwm_send_update_mcc_cmd(struct iwm_softc *sc, const char *alpha2)
4503 {
4504         struct iwm_mcc_update_cmd mcc_cmd;
4505         struct iwm_host_cmd hcmd = {
4506                 .id = IWM_MCC_UPDATE_CMD,
4507                 .flags = (IWM_CMD_SYNC | IWM_CMD_WANT_SKB),
4508                 .data = { &mcc_cmd },
4509         };
4510         int ret;
4511 #ifdef IWM_DEBUG
4512         struct iwm_rx_packet *pkt;
4513         struct iwm_mcc_update_resp_v1 *mcc_resp_v1 = NULL;
4514         struct iwm_mcc_update_resp *mcc_resp;
4515         int n_channels;
4516         uint16_t mcc;
4517 #endif
4518         int resp_v2 = fw_has_capa(&sc->ucode_capa,
4519             IWM_UCODE_TLV_CAPA_LAR_SUPPORT_V2);
4520
4521         if (!iwm_mvm_is_lar_supported(sc)) {
4522                 IWM_DPRINTF(sc, IWM_DEBUG_LAR, "%s: no LAR support\n",
4523                     __func__);
4524                 return 0;
4525         }
4526
4527         memset(&mcc_cmd, 0, sizeof(mcc_cmd));
4528         mcc_cmd.mcc = htole16(alpha2[0] << 8 | alpha2[1]);
4529         if (iwm_mvm_is_wifi_mcc_supported(sc))
4530                 mcc_cmd.source_id = IWM_MCC_SOURCE_GET_CURRENT;
4531         else
4532                 mcc_cmd.source_id = IWM_MCC_SOURCE_OLD_FW;
4533
4534         if (resp_v2)
4535                 hcmd.len[0] = sizeof(struct iwm_mcc_update_cmd);
4536         else
4537                 hcmd.len[0] = sizeof(struct iwm_mcc_update_cmd_v1);
4538
4539         IWM_DPRINTF(sc, IWM_DEBUG_LAR,
4540             "send MCC update to FW with '%c%c' src = %d\n",
4541             alpha2[0], alpha2[1], mcc_cmd.source_id);
4542
4543         ret = iwm_send_cmd(sc, &hcmd);
4544         if (ret)
4545                 return ret;
4546
4547 #ifdef IWM_DEBUG
4548         pkt = hcmd.resp_pkt;
4549
4550         /* Extract MCC response */
4551         if (resp_v2) {
4552                 mcc_resp = (void *)pkt->data;
4553                 mcc = mcc_resp->mcc;
4554                 n_channels =  le32toh(mcc_resp->n_channels);
4555         } else {
4556                 mcc_resp_v1 = (void *)pkt->data;
4557                 mcc = mcc_resp_v1->mcc;
4558                 n_channels =  le32toh(mcc_resp_v1->n_channels);
4559         }
4560
4561         /* W/A for a FW/NVM issue - returns 0x00 for the world domain */
4562         if (mcc == 0)
4563                 mcc = 0x3030;  /* "00" - world */
4564
4565         IWM_DPRINTF(sc, IWM_DEBUG_LAR,
4566             "regulatory domain '%c%c' (%d channels available)\n",
4567             mcc >> 8, mcc & 0xff, n_channels);
4568 #endif
4569         iwm_free_resp(sc, &hcmd);
4570
4571         return 0;
4572 }
4573
4574 static void
4575 iwm_mvm_tt_tx_backoff(struct iwm_softc *sc, uint32_t backoff)
4576 {
4577         struct iwm_host_cmd cmd = {
4578                 .id = IWM_REPLY_THERMAL_MNG_BACKOFF,
4579                 .len = { sizeof(uint32_t), },
4580                 .data = { &backoff, },
4581         };
4582
4583         if (iwm_send_cmd(sc, &cmd) != 0) {
4584                 device_printf(sc->sc_dev,
4585                     "failed to change thermal tx backoff\n");
4586         }
4587 }
4588
4589 static int
4590 iwm_init_hw(struct iwm_softc *sc)
4591 {
4592         struct ieee80211com *ic = &sc->sc_ic;
4593         int error, i, ac;
4594
4595         sc->sf_state = IWM_SF_UNINIT;
4596
4597         if ((error = iwm_start_hw(sc)) != 0) {
4598                 printf("iwm_start_hw: failed %d\n", error);
4599                 return error;
4600         }
4601
4602         if ((error = iwm_run_init_mvm_ucode(sc, 0)) != 0) {
4603                 printf("iwm_run_init_mvm_ucode: failed %d\n", error);
4604                 return error;
4605         }
4606
4607         /*
4608          * should stop and start HW since that INIT
4609          * image just loaded
4610          */
4611         iwm_stop_device(sc);
4612         sc->sc_ps_disabled = FALSE;
4613         if ((error = iwm_start_hw(sc)) != 0) {
4614                 device_printf(sc->sc_dev, "could not initialize hardware\n");
4615                 return error;
4616         }
4617
4618         /* omstart, this time with the regular firmware */
4619         error = iwm_mvm_load_ucode_wait_alive(sc, IWM_UCODE_REGULAR);
4620         if (error) {
4621                 device_printf(sc->sc_dev, "could not load firmware\n");
4622                 goto error;
4623         }
4624
4625         error = iwm_mvm_sf_update(sc, NULL, FALSE);
4626         if (error)
4627                 device_printf(sc->sc_dev, "Failed to initialize Smart Fifo\n");
4628
4629         if ((error = iwm_send_bt_init_conf(sc)) != 0) {
4630                 device_printf(sc->sc_dev, "bt init conf failed\n");
4631                 goto error;
4632         }
4633
4634         error = iwm_send_tx_ant_cfg(sc, iwm_mvm_get_valid_tx_ant(sc));
4635         if (error != 0) {
4636                 device_printf(sc->sc_dev, "antenna config failed\n");
4637                 goto error;
4638         }
4639
4640         /* Send phy db control command and then phy db calibration */
4641         if ((error = iwm_send_phy_db_data(sc->sc_phy_db)) != 0)
4642                 goto error;
4643
4644         if ((error = iwm_send_phy_cfg_cmd(sc)) != 0) {
4645                 device_printf(sc->sc_dev, "phy_cfg_cmd failed\n");
4646                 goto error;
4647         }
4648
4649         /* Add auxiliary station for scanning */
4650         if ((error = iwm_mvm_add_aux_sta(sc)) != 0) {
4651                 device_printf(sc->sc_dev, "add_aux_sta failed\n");
4652                 goto error;
4653         }
4654
4655         for (i = 0; i < IWM_NUM_PHY_CTX; i++) {
4656                 /*
4657                  * The channel used here isn't relevant as it's
4658                  * going to be overwritten in the other flows.
4659                  * For now use the first channel we have.
4660                  */
4661                 if ((error = iwm_mvm_phy_ctxt_add(sc,
4662                     &sc->sc_phyctxt[i], &ic->ic_channels[1], 1, 1)) != 0)
4663                         goto error;
4664         }
4665
4666         /* Initialize tx backoffs to the minimum. */
4667         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
4668                 iwm_mvm_tt_tx_backoff(sc, 0);
4669
4670         error = iwm_mvm_power_update_device(sc);
4671         if (error)
4672                 goto error;
4673
4674         if ((error = iwm_send_update_mcc_cmd(sc, "ZZ")) != 0)
4675                 goto error;
4676
4677         if (fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_UMAC_SCAN)) {
4678                 if ((error = iwm_mvm_config_umac_scan(sc)) != 0)
4679                         goto error;
4680         }
4681
4682         /* Enable Tx queues. */
4683         for (ac = 0; ac < WME_NUM_AC; ac++) {
4684                 error = iwm_enable_txq(sc, IWM_STATION_ID, ac,
4685                     iwm_mvm_ac_to_tx_fifo[ac]);
4686                 if (error)
4687                         goto error;
4688         }
4689
4690         if ((error = iwm_mvm_disable_beacon_filter(sc)) != 0) {
4691                 device_printf(sc->sc_dev, "failed to disable beacon filter\n");
4692                 goto error;
4693         }
4694
4695         return 0;
4696
4697  error:
4698         iwm_stop_device(sc);
4699         return error;
4700 }
4701
4702 /* Allow multicast from our BSSID. */
4703 static int
4704 iwm_allow_mcast(struct ieee80211vap *vap, struct iwm_softc *sc)
4705 {
4706         struct ieee80211_node *ni = vap->iv_bss;
4707         struct iwm_mcast_filter_cmd *cmd;
4708         size_t size;
4709         int error;
4710
4711         size = roundup(sizeof(*cmd), 4);
4712         cmd = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
4713         if (cmd == NULL)
4714                 return ENOMEM;
4715         cmd->filter_own = 1;
4716         cmd->port_id = 0;
4717         cmd->count = 0;
4718         cmd->pass_all = 1;
4719         IEEE80211_ADDR_COPY(cmd->bssid, ni->ni_bssid);
4720
4721         error = iwm_mvm_send_cmd_pdu(sc, IWM_MCAST_FILTER_CMD,
4722             IWM_CMD_SYNC, size, cmd);
4723         free(cmd, M_DEVBUF);
4724
4725         return (error);
4726 }
4727
4728 /*
4729  * ifnet interfaces
4730  */
4731
4732 static void
4733 iwm_init(struct iwm_softc *sc)
4734 {
4735         int error;
4736
4737         if (sc->sc_flags & IWM_FLAG_HW_INITED) {
4738                 return;
4739         }
4740         sc->sc_generation++;
4741         sc->sc_flags &= ~IWM_FLAG_STOPPED;
4742
4743         if ((error = iwm_init_hw(sc)) != 0) {
4744                 printf("iwm_init_hw failed %d\n", error);
4745                 iwm_stop(sc);
4746                 return;
4747         }
4748
4749         /*
4750          * Ok, firmware loaded and we are jogging
4751          */
4752         sc->sc_flags |= IWM_FLAG_HW_INITED;
4753         callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog, sc);
4754 }
4755
4756 static int
4757 iwm_transmit(struct ieee80211com *ic, struct mbuf *m)
4758 {
4759         struct iwm_softc *sc;
4760         int error;
4761
4762         sc = ic->ic_softc;
4763
4764         IWM_LOCK(sc);
4765         if ((sc->sc_flags & IWM_FLAG_HW_INITED) == 0) {
4766                 IWM_UNLOCK(sc);
4767                 return (ENXIO);
4768         }
4769         error = mbufq_enqueue(&sc->sc_snd, m);
4770         if (error) {
4771                 IWM_UNLOCK(sc);
4772                 return (error);
4773         }
4774         iwm_start(sc);
4775         IWM_UNLOCK(sc);
4776         return (0);
4777 }
4778
4779 /*
4780  * Dequeue packets from sendq and call send.
4781  */
4782 static void
4783 iwm_start(struct iwm_softc *sc)
4784 {
4785         struct ieee80211_node *ni;
4786         struct mbuf *m;
4787         int ac = 0;
4788
4789         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TRACE, "->%s\n", __func__);
4790         while (sc->qfullmsk == 0 &&
4791                 (m = mbufq_dequeue(&sc->sc_snd)) != NULL) {
4792                 ni = (struct ieee80211_node *)m->m_pkthdr.rcvif;
4793                 if (iwm_tx(sc, m, ni, ac) != 0) {
4794                         if_inc_counter(ni->ni_vap->iv_ifp,
4795                             IFCOUNTER_OERRORS, 1);
4796                         ieee80211_free_node(ni);
4797                         continue;
4798                 }
4799                 sc->sc_tx_timer = 15;
4800         }
4801         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TRACE, "<-%s\n", __func__);
4802 }
4803
4804 static void
4805 iwm_stop(struct iwm_softc *sc)
4806 {
4807
4808         sc->sc_flags &= ~IWM_FLAG_HW_INITED;
4809         sc->sc_flags |= IWM_FLAG_STOPPED;
4810         sc->sc_generation++;
4811         iwm_led_blink_stop(sc);
4812         sc->sc_tx_timer = 0;
4813         iwm_stop_device(sc);
4814         sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
4815 }
4816
4817 static void
4818 iwm_watchdog(void *arg)
4819 {
4820         struct iwm_softc *sc = arg;
4821         struct ieee80211com *ic = &sc->sc_ic;
4822
4823         if (sc->sc_tx_timer > 0) {
4824                 if (--sc->sc_tx_timer == 0) {
4825                         device_printf(sc->sc_dev, "device timeout\n");
4826 #ifdef IWM_DEBUG
4827                         iwm_nic_error(sc);
4828 #endif
4829                         ieee80211_restart_all(ic);
4830                         counter_u64_add(sc->sc_ic.ic_oerrors, 1);
4831                         return;
4832                 }
4833         }
4834         callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog, sc);
4835 }
4836
4837 static void
4838 iwm_parent(struct ieee80211com *ic)
4839 {
4840         struct iwm_softc *sc = ic->ic_softc;
4841         int startall = 0;
4842
4843         IWM_LOCK(sc);
4844         if (ic->ic_nrunning > 0) {
4845                 if (!(sc->sc_flags & IWM_FLAG_HW_INITED)) {
4846                         iwm_init(sc);
4847                         startall = 1;
4848                 }
4849         } else if (sc->sc_flags & IWM_FLAG_HW_INITED)
4850                 iwm_stop(sc);
4851         IWM_UNLOCK(sc);
4852         if (startall)
4853                 ieee80211_start_all(ic);
4854 }
4855
4856 /*
4857  * The interrupt side of things
4858  */
4859
4860 /*
4861  * error dumping routines are from iwlwifi/mvm/utils.c
4862  */
4863
4864 /*
4865  * Note: This structure is read from the device with IO accesses,
4866  * and the reading already does the endian conversion. As it is
4867  * read with uint32_t-sized accesses, any members with a different size
4868  * need to be ordered correctly though!
4869  */
4870 struct iwm_error_event_table {
4871         uint32_t valid;         /* (nonzero) valid, (0) log is empty */
4872         uint32_t error_id;              /* type of error */
4873         uint32_t trm_hw_status0;        /* TRM HW status */
4874         uint32_t trm_hw_status1;        /* TRM HW status */
4875         uint32_t blink2;                /* branch link */
4876         uint32_t ilink1;                /* interrupt link */
4877         uint32_t ilink2;                /* interrupt link */
4878         uint32_t data1;         /* error-specific data */
4879         uint32_t data2;         /* error-specific data */
4880         uint32_t data3;         /* error-specific data */
4881         uint32_t bcon_time;             /* beacon timer */
4882         uint32_t tsf_low;               /* network timestamp function timer */
4883         uint32_t tsf_hi;                /* network timestamp function timer */
4884         uint32_t gp1;           /* GP1 timer register */
4885         uint32_t gp2;           /* GP2 timer register */
4886         uint32_t fw_rev_type;   /* firmware revision type */
4887         uint32_t major;         /* uCode version major */
4888         uint32_t minor;         /* uCode version minor */
4889         uint32_t hw_ver;                /* HW Silicon version */
4890         uint32_t brd_ver;               /* HW board version */
4891         uint32_t log_pc;                /* log program counter */
4892         uint32_t frame_ptr;             /* frame pointer */
4893         uint32_t stack_ptr;             /* stack pointer */
4894         uint32_t hcmd;          /* last host command header */
4895         uint32_t isr0;          /* isr status register LMPM_NIC_ISR0:
4896                                  * rxtx_flag */
4897         uint32_t isr1;          /* isr status register LMPM_NIC_ISR1:
4898                                  * host_flag */
4899         uint32_t isr2;          /* isr status register LMPM_NIC_ISR2:
4900                                  * enc_flag */
4901         uint32_t isr3;          /* isr status register LMPM_NIC_ISR3:
4902                                  * time_flag */
4903         uint32_t isr4;          /* isr status register LMPM_NIC_ISR4:
4904                                  * wico interrupt */
4905         uint32_t last_cmd_id;   /* last HCMD id handled by the firmware */
4906         uint32_t wait_event;            /* wait event() caller address */
4907         uint32_t l2p_control;   /* L2pControlField */
4908         uint32_t l2p_duration;  /* L2pDurationField */
4909         uint32_t l2p_mhvalid;   /* L2pMhValidBits */
4910         uint32_t l2p_addr_match;        /* L2pAddrMatchStat */
4911         uint32_t lmpm_pmg_sel;  /* indicate which clocks are turned on
4912                                  * (LMPM_PMG_SEL) */
4913         uint32_t u_timestamp;   /* indicate when the date and time of the
4914                                  * compilation */
4915         uint32_t flow_handler;  /* FH read/write pointers, RX credit */
4916 } __packed /* LOG_ERROR_TABLE_API_S_VER_3 */;
4917
4918 /*
4919  * UMAC error struct - relevant starting from family 8000 chip.
4920  * Note: This structure is read from the device with IO accesses,
4921  * and the reading already does the endian conversion. As it is
4922  * read with u32-sized accesses, any members with a different size
4923  * need to be ordered correctly though!
4924  */
4925 struct iwm_umac_error_event_table {
4926         uint32_t valid;         /* (nonzero) valid, (0) log is empty */
4927         uint32_t error_id;      /* type of error */
4928         uint32_t blink1;        /* branch link */
4929         uint32_t blink2;        /* branch link */
4930         uint32_t ilink1;        /* interrupt link */
4931         uint32_t ilink2;        /* interrupt link */
4932         uint32_t data1;         /* error-specific data */
4933         uint32_t data2;         /* error-specific data */
4934         uint32_t data3;         /* error-specific data */
4935         uint32_t umac_major;
4936         uint32_t umac_minor;
4937         uint32_t frame_pointer; /* core register 27*/
4938         uint32_t stack_pointer; /* core register 28 */
4939         uint32_t cmd_header;    /* latest host cmd sent to UMAC */
4940         uint32_t nic_isr_pref;  /* ISR status register */
4941 } __packed;
4942
4943 #define ERROR_START_OFFSET  (1 * sizeof(uint32_t))
4944 #define ERROR_ELEM_SIZE     (7 * sizeof(uint32_t))
4945
4946 #ifdef IWM_DEBUG
4947 struct {
4948         const char *name;
4949         uint8_t num;
4950 } advanced_lookup[] = {
4951         { "NMI_INTERRUPT_WDG", 0x34 },
4952         { "SYSASSERT", 0x35 },
4953         { "UCODE_VERSION_MISMATCH", 0x37 },
4954         { "BAD_COMMAND", 0x38 },
4955         { "NMI_INTERRUPT_DATA_ACTION_PT", 0x3C },
4956         { "FATAL_ERROR", 0x3D },
4957         { "NMI_TRM_HW_ERR", 0x46 },
4958         { "NMI_INTERRUPT_TRM", 0x4C },
4959         { "NMI_INTERRUPT_BREAK_POINT", 0x54 },
4960         { "NMI_INTERRUPT_WDG_RXF_FULL", 0x5C },
4961         { "NMI_INTERRUPT_WDG_NO_RBD_RXF_FULL", 0x64 },
4962         { "NMI_INTERRUPT_HOST", 0x66 },
4963         { "NMI_INTERRUPT_ACTION_PT", 0x7C },
4964         { "NMI_INTERRUPT_UNKNOWN", 0x84 },
4965         { "NMI_INTERRUPT_INST_ACTION_PT", 0x86 },
4966         { "ADVANCED_SYSASSERT", 0 },
4967 };
4968
4969 static const char *
4970 iwm_desc_lookup(uint32_t num)
4971 {
4972         int i;
4973
4974         for (i = 0; i < nitems(advanced_lookup) - 1; i++)
4975                 if (advanced_lookup[i].num == num)
4976                         return advanced_lookup[i].name;
4977
4978         /* No entry matches 'num', so it is the last: ADVANCED_SYSASSERT */
4979         return advanced_lookup[i].name;
4980 }
4981
4982 static void
4983 iwm_nic_umac_error(struct iwm_softc *sc)
4984 {
4985         struct iwm_umac_error_event_table table;
4986         uint32_t base;
4987
4988         base = sc->umac_error_event_table;
4989
4990         if (base < 0x800000) {
4991                 device_printf(sc->sc_dev, "Invalid error log pointer 0x%08x\n",
4992                     base);
4993                 return;
4994         }
4995
4996         if (iwm_read_mem(sc, base, &table, sizeof(table)/sizeof(uint32_t))) {
4997                 device_printf(sc->sc_dev, "reading errlog failed\n");
4998                 return;
4999         }
5000
5001         if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) {
5002                 device_printf(sc->sc_dev, "Start UMAC Error Log Dump:\n");
5003                 device_printf(sc->sc_dev, "Status: 0x%x, count: %d\n",
5004                     sc->sc_flags, table.valid);
5005         }
5006
5007         device_printf(sc->sc_dev, "0x%08X | %s\n", table.error_id,
5008                 iwm_desc_lookup(table.error_id));
5009         device_printf(sc->sc_dev, "0x%08X | umac branchlink1\n", table.blink1);
5010         device_printf(sc->sc_dev, "0x%08X | umac branchlink2\n", table.blink2);
5011         device_printf(sc->sc_dev, "0x%08X | umac interruptlink1\n",
5012             table.ilink1);
5013         device_printf(sc->sc_dev, "0x%08X | umac interruptlink2\n",
5014             table.ilink2);
5015         device_printf(sc->sc_dev, "0x%08X | umac data1\n", table.data1);
5016         device_printf(sc->sc_dev, "0x%08X | umac data2\n", table.data2);
5017         device_printf(sc->sc_dev, "0x%08X | umac data3\n", table.data3);
5018         device_printf(sc->sc_dev, "0x%08X | umac major\n", table.umac_major);
5019         device_printf(sc->sc_dev, "0x%08X | umac minor\n", table.umac_minor);
5020         device_printf(sc->sc_dev, "0x%08X | frame pointer\n",
5021             table.frame_pointer);
5022         device_printf(sc->sc_dev, "0x%08X | stack pointer\n",
5023             table.stack_pointer);
5024         device_printf(sc->sc_dev, "0x%08X | last host cmd\n", table.cmd_header);
5025         device_printf(sc->sc_dev, "0x%08X | isr status reg\n",
5026             table.nic_isr_pref);
5027 }
5028
5029 /*
5030  * Support for dumping the error log seemed like a good idea ...
5031  * but it's mostly hex junk and the only sensible thing is the
5032  * hw/ucode revision (which we know anyway).  Since it's here,
5033  * I'll just leave it in, just in case e.g. the Intel guys want to
5034  * help us decipher some "ADVANCED_SYSASSERT" later.
5035  */
5036 static void
5037 iwm_nic_error(struct iwm_softc *sc)
5038 {
5039         struct iwm_error_event_table table;
5040         uint32_t base;
5041
5042         device_printf(sc->sc_dev, "dumping device error log\n");
5043         base = sc->error_event_table;
5044         if (base < 0x800000) {
5045                 device_printf(sc->sc_dev,
5046                     "Invalid error log pointer 0x%08x\n", base);
5047                 return;
5048         }
5049
5050         if (iwm_read_mem(sc, base, &table, sizeof(table)/sizeof(uint32_t))) {
5051                 device_printf(sc->sc_dev, "reading errlog failed\n");
5052                 return;
5053         }
5054
5055         if (!table.valid) {
5056                 device_printf(sc->sc_dev, "errlog not found, skipping\n");
5057                 return;
5058         }
5059
5060         if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) {
5061                 device_printf(sc->sc_dev, "Start Error Log Dump:\n");
5062                 device_printf(sc->sc_dev, "Status: 0x%x, count: %d\n",
5063                     sc->sc_flags, table.valid);
5064         }
5065
5066         device_printf(sc->sc_dev, "0x%08X | %-28s\n", table.error_id,
5067             iwm_desc_lookup(table.error_id));
5068         device_printf(sc->sc_dev, "%08X | trm_hw_status0\n",
5069             table.trm_hw_status0);
5070         device_printf(sc->sc_dev, "%08X | trm_hw_status1\n",
5071             table.trm_hw_status1);
5072         device_printf(sc->sc_dev, "%08X | branchlink2\n", table.blink2);
5073         device_printf(sc->sc_dev, "%08X | interruptlink1\n", table.ilink1);
5074         device_printf(sc->sc_dev, "%08X | interruptlink2\n", table.ilink2);
5075         device_printf(sc->sc_dev, "%08X | data1\n", table.data1);
5076         device_printf(sc->sc_dev, "%08X | data2\n", table.data2);
5077         device_printf(sc->sc_dev, "%08X | data3\n", table.data3);
5078         device_printf(sc->sc_dev, "%08X | beacon time\n", table.bcon_time);
5079         device_printf(sc->sc_dev, "%08X | tsf low\n", table.tsf_low);
5080         device_printf(sc->sc_dev, "%08X | tsf hi\n", table.tsf_hi);
5081         device_printf(sc->sc_dev, "%08X | time gp1\n", table.gp1);
5082         device_printf(sc->sc_dev, "%08X | time gp2\n", table.gp2);
5083         device_printf(sc->sc_dev, "%08X | uCode revision type\n",
5084             table.fw_rev_type);
5085         device_printf(sc->sc_dev, "%08X | uCode version major\n", table.major);
5086         device_printf(sc->sc_dev, "%08X | uCode version minor\n", table.minor);
5087         device_printf(sc->sc_dev, "%08X | hw version\n", table.hw_ver);
5088         device_printf(sc->sc_dev, "%08X | board version\n", table.brd_ver);
5089         device_printf(sc->sc_dev, "%08X | hcmd\n", table.hcmd);
5090         device_printf(sc->sc_dev, "%08X | isr0\n", table.isr0);
5091         device_printf(sc->sc_dev, "%08X | isr1\n", table.isr1);
5092         device_printf(sc->sc_dev, "%08X | isr2\n", table.isr2);
5093         device_printf(sc->sc_dev, "%08X | isr3\n", table.isr3);
5094         device_printf(sc->sc_dev, "%08X | isr4\n", table.isr4);
5095         device_printf(sc->sc_dev, "%08X | last cmd Id\n", table.last_cmd_id);
5096         device_printf(sc->sc_dev, "%08X | wait_event\n", table.wait_event);
5097         device_printf(sc->sc_dev, "%08X | l2p_control\n", table.l2p_control);
5098         device_printf(sc->sc_dev, "%08X | l2p_duration\n", table.l2p_duration);
5099         device_printf(sc->sc_dev, "%08X | l2p_mhvalid\n", table.l2p_mhvalid);
5100         device_printf(sc->sc_dev, "%08X | l2p_addr_match\n", table.l2p_addr_match);
5101         device_printf(sc->sc_dev, "%08X | lmpm_pmg_sel\n", table.lmpm_pmg_sel);
5102         device_printf(sc->sc_dev, "%08X | timestamp\n", table.u_timestamp);
5103         device_printf(sc->sc_dev, "%08X | flow_handler\n", table.flow_handler);
5104
5105         if (sc->umac_error_event_table)
5106                 iwm_nic_umac_error(sc);
5107 }
5108 #endif
5109
5110 static void
5111 iwm_handle_rxb(struct iwm_softc *sc, struct mbuf *m)
5112 {
5113         struct ieee80211com *ic = &sc->sc_ic;
5114         struct iwm_cmd_response *cresp;
5115         struct mbuf *m1;
5116         uint32_t offset = 0;
5117         uint32_t maxoff = IWM_RBUF_SIZE;
5118         uint32_t nextoff;
5119         boolean_t stolen = FALSE;
5120
5121 #define HAVEROOM(a)     \
5122     ((a) + sizeof(uint32_t) + sizeof(struct iwm_cmd_header) < maxoff)
5123
5124         while (HAVEROOM(offset)) {
5125                 struct iwm_rx_packet *pkt = mtodoff(m, struct iwm_rx_packet *,
5126                     offset);
5127                 int qid, idx, code, len;
5128
5129                 qid = pkt->hdr.qid;
5130                 idx = pkt->hdr.idx;
5131
5132                 code = IWM_WIDE_ID(pkt->hdr.flags, pkt->hdr.code);
5133
5134                 /*
5135                  * randomly get these from the firmware, no idea why.
5136                  * they at least seem harmless, so just ignore them for now
5137                  */
5138                 if ((pkt->hdr.code == 0 && (qid & ~0x80) == 0 && idx == 0) ||
5139                     pkt->len_n_flags == htole32(IWM_FH_RSCSR_FRAME_INVALID)) {
5140                         break;
5141                 }
5142
5143                 IWM_DPRINTF(sc, IWM_DEBUG_INTR,
5144                     "rx packet qid=%d idx=%d type=%x\n",
5145                     qid & ~0x80, pkt->hdr.idx, code);
5146
5147                 len = iwm_rx_packet_len(pkt);
5148                 len += sizeof(uint32_t); /* account for status word */
5149                 nextoff = offset + roundup2(len, IWM_FH_RSCSR_FRAME_ALIGN);
5150
5151                 iwm_notification_wait_notify(sc->sc_notif_wait, code, pkt);
5152
5153                 switch (code) {
5154                 case IWM_REPLY_RX_PHY_CMD:
5155                         iwm_mvm_rx_rx_phy_cmd(sc, pkt);
5156                         break;
5157
5158                 case IWM_REPLY_RX_MPDU_CMD: {
5159                         /*
5160                          * If this is the last frame in the RX buffer, we
5161                          * can directly feed the mbuf to the sharks here.
5162                          */
5163                         struct iwm_rx_packet *nextpkt = mtodoff(m,
5164                             struct iwm_rx_packet *, nextoff);
5165                         if (!HAVEROOM(nextoff) ||
5166                             (nextpkt->hdr.code == 0 &&
5167                              (nextpkt->hdr.qid & ~0x80) == 0 &&
5168                              nextpkt->hdr.idx == 0) ||
5169                             (nextpkt->len_n_flags ==
5170                              htole32(IWM_FH_RSCSR_FRAME_INVALID))) {
5171                                 if (iwm_mvm_rx_rx_mpdu(sc, m, offset, stolen)) {
5172                                         stolen = FALSE;
5173                                         /* Make sure we abort the loop */
5174                                         nextoff = maxoff;
5175                                 }
5176                                 break;
5177                         }
5178
5179                         /*
5180                          * Use m_copym instead of m_split, because that
5181                          * makes it easier to keep a valid rx buffer in
5182                          * the ring, when iwm_mvm_rx_rx_mpdu() fails.
5183                          *
5184                          * We need to start m_copym() at offset 0, to get the
5185                          * M_PKTHDR flag preserved.
5186                          */
5187                         m1 = m_copym(m, 0, M_COPYALL, M_NOWAIT);
5188                         if (m1) {
5189                                 if (iwm_mvm_rx_rx_mpdu(sc, m1, offset, stolen))
5190                                         stolen = TRUE;
5191                                 else
5192                                         m_freem(m1);
5193                         }
5194                         break;
5195                 }
5196
5197                 case IWM_TX_CMD:
5198                         iwm_mvm_rx_tx_cmd(sc, pkt);
5199                         break;
5200
5201                 case IWM_MISSED_BEACONS_NOTIFICATION: {
5202                         struct iwm_missed_beacons_notif *resp;
5203                         int missed;
5204
5205                         /* XXX look at mac_id to determine interface ID */
5206                         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
5207
5208                         resp = (void *)pkt->data;
5209                         missed = le32toh(resp->consec_missed_beacons);
5210
5211                         IWM_DPRINTF(sc, IWM_DEBUG_BEACON | IWM_DEBUG_STATE,
5212                             "%s: MISSED_BEACON: mac_id=%d, "
5213                             "consec_since_last_rx=%d, consec=%d, num_expect=%d "
5214                             "num_rx=%d\n",
5215                             __func__,
5216                             le32toh(resp->mac_id),
5217                             le32toh(resp->consec_missed_beacons_since_last_rx),
5218                             le32toh(resp->consec_missed_beacons),
5219                             le32toh(resp->num_expected_beacons),
5220                             le32toh(resp->num_recvd_beacons));
5221
5222                         /* Be paranoid */
5223                         if (vap == NULL)
5224                                 break;
5225
5226                         /* XXX no net80211 locking? */
5227                         if (vap->iv_state == IEEE80211_S_RUN &&
5228                             (ic->ic_flags & IEEE80211_F_SCAN) == 0) {
5229                                 if (missed > vap->iv_bmissthreshold) {
5230                                         /* XXX bad locking; turn into task */
5231                                         IWM_UNLOCK(sc);
5232                                         ieee80211_beacon_miss(ic);
5233                                         IWM_LOCK(sc);
5234                                 }
5235                         }
5236
5237                         break;
5238                 }
5239
5240                 case IWM_MFUART_LOAD_NOTIFICATION:
5241                         break;
5242
5243                 case IWM_MVM_ALIVE:
5244                         break;
5245
5246                 case IWM_CALIB_RES_NOTIF_PHY_DB:
5247                         break;
5248
5249                 case IWM_STATISTICS_NOTIFICATION:
5250                         iwm_mvm_handle_rx_statistics(sc, pkt);
5251                         break;
5252
5253                 case IWM_NVM_ACCESS_CMD:
5254                 case IWM_MCC_UPDATE_CMD:
5255                         if (sc->sc_wantresp == (((qid & ~0x80) << 16) | idx)) {
5256                                 memcpy(sc->sc_cmd_resp,
5257                                     pkt, sizeof(sc->sc_cmd_resp));
5258                         }
5259                         break;
5260
5261                 case IWM_MCC_CHUB_UPDATE_CMD: {
5262                         struct iwm_mcc_chub_notif *notif;
5263                         notif = (void *)pkt->data;
5264
5265                         sc->sc_fw_mcc[0] = (notif->mcc & 0xff00) >> 8;
5266                         sc->sc_fw_mcc[1] = notif->mcc & 0xff;
5267                         sc->sc_fw_mcc[2] = '\0';
5268                         IWM_DPRINTF(sc, IWM_DEBUG_LAR,
5269                             "fw source %d sent CC '%s'\n",
5270                             notif->source_id, sc->sc_fw_mcc);
5271                         break;
5272                 }
5273
5274                 case IWM_DTS_MEASUREMENT_NOTIFICATION:
5275                 case IWM_WIDE_ID(IWM_PHY_OPS_GROUP,
5276                                  IWM_DTS_MEASUREMENT_NOTIF_WIDE): {
5277                         struct iwm_dts_measurement_notif_v1 *notif;
5278
5279                         if (iwm_rx_packet_payload_len(pkt) < sizeof(*notif)) {
5280                                 device_printf(sc->sc_dev,
5281                                     "Invalid DTS_MEASUREMENT_NOTIFICATION\n");
5282                                 break;
5283                         }
5284                         notif = (void *)pkt->data;
5285                         IWM_DPRINTF(sc, IWM_DEBUG_TEMP,
5286                             "IWM_DTS_MEASUREMENT_NOTIFICATION - %d\n",
5287                             notif->temp);
5288                         break;
5289                 }
5290
5291                 case IWM_PHY_CONFIGURATION_CMD:
5292                 case IWM_TX_ANT_CONFIGURATION_CMD:
5293                 case IWM_ADD_STA:
5294                 case IWM_MAC_CONTEXT_CMD:
5295                 case IWM_REPLY_SF_CFG_CMD:
5296                 case IWM_POWER_TABLE_CMD:
5297                 case IWM_PHY_CONTEXT_CMD:
5298                 case IWM_BINDING_CONTEXT_CMD:
5299                 case IWM_TIME_EVENT_CMD:
5300                 case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_CFG_CMD):
5301                 case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_REQ_UMAC):
5302                 case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_ABORT_UMAC):
5303                 case IWM_SCAN_OFFLOAD_REQUEST_CMD:
5304                 case IWM_SCAN_OFFLOAD_ABORT_CMD:
5305                 case IWM_REPLY_BEACON_FILTERING_CMD:
5306                 case IWM_MAC_PM_POWER_TABLE:
5307                 case IWM_TIME_QUOTA_CMD:
5308                 case IWM_REMOVE_STA:
5309                 case IWM_TXPATH_FLUSH:
5310                 case IWM_LQ_CMD:
5311                 case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP,
5312                                  IWM_FW_PAGING_BLOCK_CMD):
5313                 case IWM_BT_CONFIG:
5314                 case IWM_REPLY_THERMAL_MNG_BACKOFF:
5315                         cresp = (void *)pkt->data;
5316                         if (sc->sc_wantresp == (((qid & ~0x80) << 16) | idx)) {
5317                                 memcpy(sc->sc_cmd_resp,
5318                                     pkt, sizeof(*pkt)+sizeof(*cresp));
5319                         }
5320                         break;
5321
5322                 /* ignore */
5323                 case IWM_PHY_DB_CMD:
5324                         break;
5325
5326                 case IWM_INIT_COMPLETE_NOTIF:
5327                         break;
5328
5329                 case IWM_SCAN_OFFLOAD_COMPLETE:
5330                         iwm_mvm_rx_lmac_scan_complete_notif(sc, pkt);
5331                         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
5332                                 sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
5333                                 ieee80211_runtask(ic, &sc->sc_es_task);
5334                         }
5335                         break;
5336
5337                 case IWM_SCAN_ITERATION_COMPLETE: {
5338                         struct iwm_lmac_scan_complete_notif *notif;
5339                         notif = (void *)pkt->data;
5340                         break;
5341                 }
5342
5343                 case IWM_SCAN_COMPLETE_UMAC:
5344                         iwm_mvm_rx_umac_scan_complete_notif(sc, pkt);
5345                         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
5346                                 sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
5347                                 ieee80211_runtask(ic, &sc->sc_es_task);
5348                         }
5349                         break;
5350
5351                 case IWM_SCAN_ITERATION_COMPLETE_UMAC: {
5352                         struct iwm_umac_scan_iter_complete_notif *notif;
5353                         notif = (void *)pkt->data;
5354
5355                         IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "UMAC scan iteration "
5356                             "complete, status=0x%x, %d channels scanned\n",
5357                             notif->status, notif->scanned_channels);
5358                         break;
5359                 }
5360
5361                 case IWM_REPLY_ERROR: {
5362                         struct iwm_error_resp *resp;
5363                         resp = (void *)pkt->data;
5364
5365                         device_printf(sc->sc_dev,
5366                             "firmware error 0x%x, cmd 0x%x\n",
5367                             le32toh(resp->error_type),
5368                             resp->cmd_id);
5369                         break;
5370                 }
5371
5372                 case IWM_TIME_EVENT_NOTIFICATION: {
5373                         struct iwm_time_event_notif *notif;
5374                         notif = (void *)pkt->data;
5375
5376                         IWM_DPRINTF(sc, IWM_DEBUG_INTR,
5377                             "TE notif status = 0x%x action = 0x%x\n",
5378                             notif->status, notif->action);
5379                         break;
5380                 }
5381
5382                 /*
5383                  * Firmware versions 21 and 22 generate some DEBUG_LOG_MSG
5384                  * messages. Just ignore them for now.
5385                  */
5386                 case IWM_DEBUG_LOG_MSG:
5387                         break;
5388
5389                 case IWM_MCAST_FILTER_CMD:
5390                         break;
5391
5392                 case IWM_SCD_QUEUE_CFG: {
5393                         struct iwm_scd_txq_cfg_rsp *rsp;
5394                         rsp = (void *)pkt->data;
5395
5396                         IWM_DPRINTF(sc, IWM_DEBUG_CMD,
5397                             "queue cfg token=0x%x sta_id=%d "
5398                             "tid=%d scd_queue=%d\n",
5399                             rsp->token, rsp->sta_id, rsp->tid,
5400                             rsp->scd_queue);
5401                         break;
5402                 }
5403
5404                 default:
5405                         device_printf(sc->sc_dev,
5406                             "frame %d/%d %x UNHANDLED (this should "
5407                             "not happen)\n", qid & ~0x80, idx,
5408                             pkt->len_n_flags);
5409                         break;
5410                 }
5411
5412                 /*
5413                  * Why test bit 0x80?  The Linux driver:
5414                  *
5415                  * There is one exception:  uCode sets bit 15 when it
5416                  * originates the response/notification, i.e. when the
5417                  * response/notification is not a direct response to a
5418                  * command sent by the driver.  For example, uCode issues
5419                  * IWM_REPLY_RX when it sends a received frame to the driver;
5420                  * it is not a direct response to any driver command.
5421                  *
5422                  * Ok, so since when is 7 == 15?  Well, the Linux driver
5423                  * uses a slightly different format for pkt->hdr, and "qid"
5424                  * is actually the upper byte of a two-byte field.
5425                  */
5426                 if (!(qid & (1 << 7)))
5427                         iwm_cmd_done(sc, pkt);
5428
5429                 offset = nextoff;
5430         }
5431         if (stolen)
5432                 m_freem(m);
5433 #undef HAVEROOM
5434 }
5435
5436 /*
5437  * Process an IWM_CSR_INT_BIT_FH_RX or IWM_CSR_INT_BIT_SW_RX interrupt.
5438  * Basic structure from if_iwn
5439  */
5440 static void
5441 iwm_notif_intr(struct iwm_softc *sc)
5442 {
5443         uint16_t hw;
5444
5445         bus_dmamap_sync(sc->rxq.stat_dma.tag, sc->rxq.stat_dma.map,
5446             BUS_DMASYNC_POSTREAD);
5447
5448         hw = le16toh(sc->rxq.stat->closed_rb_num) & 0xfff;
5449
5450         /*
5451          * Process responses
5452          */
5453         while (sc->rxq.cur != hw) {
5454                 struct iwm_rx_ring *ring = &sc->rxq;
5455                 struct iwm_rx_data *data = &ring->data[ring->cur];
5456
5457                 bus_dmamap_sync(ring->data_dmat, data->map,
5458                     BUS_DMASYNC_POSTREAD);
5459
5460                 IWM_DPRINTF(sc, IWM_DEBUG_INTR,
5461                     "%s: hw = %d cur = %d\n", __func__, hw, ring->cur);
5462                 iwm_handle_rxb(sc, data->m);
5463
5464                 ring->cur = (ring->cur + 1) % IWM_RX_RING_COUNT;
5465         }
5466
5467         /*
5468          * Tell the firmware that it can reuse the ring entries that
5469          * we have just processed.
5470          * Seems like the hardware gets upset unless we align
5471          * the write by 8??
5472          */
5473         hw = (hw == 0) ? IWM_RX_RING_COUNT - 1 : hw - 1;
5474         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_WPTR, rounddown2(hw, 8));
5475 }
5476
5477 static void
5478 iwm_intr(void *arg)
5479 {
5480         struct iwm_softc *sc = arg;
5481         int handled = 0;
5482         int r1, r2, rv = 0;
5483         int isperiodic = 0;
5484
5485         IWM_LOCK(sc);
5486         IWM_WRITE(sc, IWM_CSR_INT_MASK, 0);
5487
5488         if (sc->sc_flags & IWM_FLAG_USE_ICT) {
5489                 uint32_t *ict = sc->ict_dma.vaddr;
5490                 int tmp;
5491
5492                 tmp = htole32(ict[sc->ict_cur]);
5493                 if (!tmp)
5494                         goto out_ena;
5495
5496                 /*
5497                  * ok, there was something.  keep plowing until we have all.
5498                  */
5499                 r1 = r2 = 0;
5500                 while (tmp) {
5501                         r1 |= tmp;
5502                         ict[sc->ict_cur] = 0;
5503                         sc->ict_cur = (sc->ict_cur+1) % IWM_ICT_COUNT;
5504                         tmp = htole32(ict[sc->ict_cur]);
5505                 }
5506
5507                 /* this is where the fun begins.  don't ask */
5508                 if (r1 == 0xffffffff)
5509                         r1 = 0;
5510
5511                 /* i am not expected to understand this */
5512                 if (r1 & 0xc0000)
5513                         r1 |= 0x8000;
5514                 r1 = (0xff & r1) | ((0xff00 & r1) << 16);
5515         } else {
5516                 r1 = IWM_READ(sc, IWM_CSR_INT);
5517                 /* "hardware gone" (where, fishing?) */
5518                 if (r1 == 0xffffffff || (r1 & 0xfffffff0) == 0xa5a5a5a0)
5519                         goto out;
5520                 r2 = IWM_READ(sc, IWM_CSR_FH_INT_STATUS);
5521         }
5522         if (r1 == 0 && r2 == 0) {
5523                 goto out_ena;
5524         }
5525
5526         IWM_WRITE(sc, IWM_CSR_INT, r1 | ~sc->sc_intmask);
5527
5528         /* Safely ignore these bits for debug checks below */
5529         r1 &= ~(IWM_CSR_INT_BIT_ALIVE | IWM_CSR_INT_BIT_SCD);
5530
5531         if (r1 & IWM_CSR_INT_BIT_SW_ERR) {
5532                 int i;
5533                 struct ieee80211com *ic = &sc->sc_ic;
5534                 struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
5535
5536 #ifdef IWM_DEBUG
5537                 iwm_nic_error(sc);
5538 #endif
5539                 /* Dump driver status (TX and RX rings) while we're here. */
5540                 device_printf(sc->sc_dev, "driver status:\n");
5541                 for (i = 0; i < IWM_MVM_MAX_QUEUES; i++) {
5542                         struct iwm_tx_ring *ring = &sc->txq[i];
5543                         device_printf(sc->sc_dev,
5544                             "  tx ring %2d: qid=%-2d cur=%-3d "
5545                             "queued=%-3d\n",
5546                             i, ring->qid, ring->cur, ring->queued);
5547                 }
5548                 device_printf(sc->sc_dev,
5549                     "  rx ring: cur=%d\n", sc->rxq.cur);
5550                 device_printf(sc->sc_dev,
5551                     "  802.11 state %d\n", (vap == NULL) ? -1 : vap->iv_state);
5552
5553                 /* Don't stop the device; just do a VAP restart */
5554                 IWM_UNLOCK(sc);
5555
5556                 if (vap == NULL) {
5557                         printf("%s: null vap\n", __func__);
5558                         return;
5559                 }
5560
5561                 device_printf(sc->sc_dev, "%s: controller panicked, iv_state = %d; "
5562                     "restarting\n", __func__, vap->iv_state);
5563
5564                 ieee80211_restart_all(ic);
5565                 return;
5566         }
5567
5568         if (r1 & IWM_CSR_INT_BIT_HW_ERR) {
5569                 handled |= IWM_CSR_INT_BIT_HW_ERR;
5570                 device_printf(sc->sc_dev, "hardware error, stopping device\n");
5571                 iwm_stop(sc);
5572                 rv = 1;
5573                 goto out;
5574         }
5575
5576         /* firmware chunk loaded */
5577         if (r1 & IWM_CSR_INT_BIT_FH_TX) {
5578                 IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, IWM_CSR_FH_INT_TX_MASK);
5579                 handled |= IWM_CSR_INT_BIT_FH_TX;
5580                 sc->sc_fw_chunk_done = 1;
5581                 wakeup(&sc->sc_fw);
5582         }
5583
5584         if (r1 & IWM_CSR_INT_BIT_RF_KILL) {
5585                 handled |= IWM_CSR_INT_BIT_RF_KILL;
5586                 if (iwm_check_rfkill(sc)) {
5587                         device_printf(sc->sc_dev,
5588                             "%s: rfkill switch, disabling interface\n",
5589                             __func__);
5590                         iwm_stop(sc);
5591                 }
5592         }
5593
5594         /*
5595          * The Linux driver uses periodic interrupts to avoid races.
5596          * We cargo-cult like it's going out of fashion.
5597          */
5598         if (r1 & IWM_CSR_INT_BIT_RX_PERIODIC) {
5599                 handled |= IWM_CSR_INT_BIT_RX_PERIODIC;
5600                 IWM_WRITE(sc, IWM_CSR_INT, IWM_CSR_INT_BIT_RX_PERIODIC);
5601                 if ((r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX)) == 0)
5602                         IWM_WRITE_1(sc,
5603                             IWM_CSR_INT_PERIODIC_REG, IWM_CSR_INT_PERIODIC_DIS);
5604                 isperiodic = 1;
5605         }
5606
5607         if ((r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX)) || isperiodic) {
5608                 handled |= (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX);
5609                 IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, IWM_CSR_FH_INT_RX_MASK);
5610
5611                 iwm_notif_intr(sc);
5612
5613                 /* enable periodic interrupt, see above */
5614                 if (r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX) && !isperiodic)
5615                         IWM_WRITE_1(sc, IWM_CSR_INT_PERIODIC_REG,
5616                             IWM_CSR_INT_PERIODIC_ENA);
5617         }
5618
5619         if (__predict_false(r1 & ~handled))
5620                 IWM_DPRINTF(sc, IWM_DEBUG_INTR,
5621                     "%s: unhandled interrupts: %x\n", __func__, r1);
5622         rv = 1;
5623
5624  out_ena:
5625         iwm_restore_interrupts(sc);
5626  out:
5627         IWM_UNLOCK(sc);
5628         return;
5629 }
5630
5631 /*
5632  * Autoconf glue-sniffing
5633  */
5634 #define PCI_VENDOR_INTEL                0x8086
5635 #define PCI_PRODUCT_INTEL_WL_3160_1     0x08b3
5636 #define PCI_PRODUCT_INTEL_WL_3160_2     0x08b4
5637 #define PCI_PRODUCT_INTEL_WL_3165_1     0x3165
5638 #define PCI_PRODUCT_INTEL_WL_3165_2     0x3166
5639 #define PCI_PRODUCT_INTEL_WL_3168_1     0x24fb
5640 #define PCI_PRODUCT_INTEL_WL_7260_1     0x08b1
5641 #define PCI_PRODUCT_INTEL_WL_7260_2     0x08b2
5642 #define PCI_PRODUCT_INTEL_WL_7265_1     0x095a
5643 #define PCI_PRODUCT_INTEL_WL_7265_2     0x095b
5644 #define PCI_PRODUCT_INTEL_WL_8260_1     0x24f3
5645 #define PCI_PRODUCT_INTEL_WL_8260_2     0x24f4
5646 #define PCI_PRODUCT_INTEL_WL_8265_1     0x24fd
5647
5648 static const struct iwm_devices {
5649         uint16_t                device;
5650         const struct iwm_cfg    *cfg;
5651 } iwm_devices[] = {
5652         { PCI_PRODUCT_INTEL_WL_3160_1, &iwm3160_cfg },
5653         { PCI_PRODUCT_INTEL_WL_3160_2, &iwm3160_cfg },
5654         { PCI_PRODUCT_INTEL_WL_3165_1, &iwm3165_cfg },
5655         { PCI_PRODUCT_INTEL_WL_3165_2, &iwm3165_cfg },
5656         { PCI_PRODUCT_INTEL_WL_3168_1, &iwm3168_cfg },
5657         { PCI_PRODUCT_INTEL_WL_7260_1, &iwm7260_cfg },
5658         { PCI_PRODUCT_INTEL_WL_7260_2, &iwm7260_cfg },
5659         { PCI_PRODUCT_INTEL_WL_7265_1, &iwm7265_cfg },
5660         { PCI_PRODUCT_INTEL_WL_7265_2, &iwm7265_cfg },
5661         { PCI_PRODUCT_INTEL_WL_8260_1, &iwm8260_cfg },
5662         { PCI_PRODUCT_INTEL_WL_8260_2, &iwm8260_cfg },
5663         { PCI_PRODUCT_INTEL_WL_8265_1, &iwm8265_cfg },
5664 };
5665
5666 static int
5667 iwm_probe(device_t dev)
5668 {
5669         int i;
5670
5671         for (i = 0; i < nitems(iwm_devices); i++) {
5672                 if (pci_get_vendor(dev) == PCI_VENDOR_INTEL &&
5673                     pci_get_device(dev) == iwm_devices[i].device) {
5674                         device_set_desc(dev, iwm_devices[i].cfg->name);
5675                         return (BUS_PROBE_DEFAULT);
5676                 }
5677         }
5678
5679         return (ENXIO);
5680 }
5681
5682 static int
5683 iwm_dev_check(device_t dev)
5684 {
5685         struct iwm_softc *sc;
5686         uint16_t devid;
5687         int i;
5688
5689         sc = device_get_softc(dev);
5690
5691         devid = pci_get_device(dev);
5692         for (i = 0; i < nitems(iwm_devices); i++) {
5693                 if (iwm_devices[i].device == devid) {
5694                         sc->cfg = iwm_devices[i].cfg;
5695                         return (0);
5696                 }
5697         }
5698         device_printf(dev, "unknown adapter type\n");
5699         return ENXIO;
5700 }
5701
5702 /* PCI registers */
5703 #define PCI_CFG_RETRY_TIMEOUT   0x041
5704
5705 static int
5706 iwm_pci_attach(device_t dev)
5707 {
5708         struct iwm_softc *sc;
5709         int count, error, rid;
5710         uint16_t reg;
5711
5712         sc = device_get_softc(dev);
5713
5714         /* We disable the RETRY_TIMEOUT register (0x41) to keep
5715          * PCI Tx retries from interfering with C3 CPU state */
5716         pci_write_config(dev, PCI_CFG_RETRY_TIMEOUT, 0x00, 1);
5717
5718         /* Enable bus-mastering and hardware bug workaround. */
5719         pci_enable_busmaster(dev);
5720         reg = pci_read_config(dev, PCIR_STATUS, sizeof(reg));
5721         /* if !MSI */
5722         if (reg & PCIM_STATUS_INTxSTATE) {
5723                 reg &= ~PCIM_STATUS_INTxSTATE;
5724         }
5725         pci_write_config(dev, PCIR_STATUS, reg, sizeof(reg));
5726
5727         rid = PCIR_BAR(0);
5728         sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
5729             RF_ACTIVE);
5730         if (sc->sc_mem == NULL) {
5731                 device_printf(sc->sc_dev, "can't map mem space\n");
5732                 return (ENXIO);
5733         }
5734         sc->sc_st = rman_get_bustag(sc->sc_mem);
5735         sc->sc_sh = rman_get_bushandle(sc->sc_mem);
5736
5737         /* Install interrupt handler. */
5738         count = 1;
5739         rid = 0;
5740         if (pci_alloc_msi(dev, &count) == 0)
5741                 rid = 1;
5742         sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE |
5743             (rid != 0 ? 0 : RF_SHAREABLE));
5744         if (sc->sc_irq == NULL) {
5745                 device_printf(dev, "can't map interrupt\n");
5746                         return (ENXIO);
5747         }
5748         error = bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
5749             NULL, iwm_intr, sc, &sc->sc_ih);
5750         if (sc->sc_ih == NULL) {
5751                 device_printf(dev, "can't establish interrupt");
5752                         return (ENXIO);
5753         }
5754         sc->sc_dmat = bus_get_dma_tag(sc->sc_dev);
5755
5756         return (0);
5757 }
5758
5759 static void
5760 iwm_pci_detach(device_t dev)
5761 {
5762         struct iwm_softc *sc = device_get_softc(dev);
5763
5764         if (sc->sc_irq != NULL) {
5765                 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
5766                 bus_release_resource(dev, SYS_RES_IRQ,
5767                     rman_get_rid(sc->sc_irq), sc->sc_irq);
5768                 pci_release_msi(dev);
5769         }
5770         if (sc->sc_mem != NULL)
5771                 bus_release_resource(dev, SYS_RES_MEMORY,
5772                     rman_get_rid(sc->sc_mem), sc->sc_mem);
5773 }
5774
5775
5776
5777 static int
5778 iwm_attach(device_t dev)
5779 {
5780         struct iwm_softc *sc = device_get_softc(dev);
5781         struct ieee80211com *ic = &sc->sc_ic;
5782         int error;
5783         int txq_i, i;
5784
5785         sc->sc_dev = dev;
5786         sc->sc_attached = 1;
5787         IWM_LOCK_INIT(sc);
5788         mbufq_init(&sc->sc_snd, ifqmaxlen);
5789         callout_init_mtx(&sc->sc_watchdog_to, &sc->sc_mtx, 0);
5790         callout_init_mtx(&sc->sc_led_blink_to, &sc->sc_mtx, 0);
5791         TASK_INIT(&sc->sc_es_task, 0, iwm_endscan_cb, sc);
5792
5793         sc->sc_notif_wait = iwm_notification_wait_init(sc);
5794         if (sc->sc_notif_wait == NULL) {
5795                 device_printf(dev, "failed to init notification wait struct\n");
5796                 goto fail;
5797         }
5798
5799         sc->sf_state = IWM_SF_UNINIT;
5800
5801         /* Init phy db */
5802         sc->sc_phy_db = iwm_phy_db_init(sc);
5803         if (!sc->sc_phy_db) {
5804                 device_printf(dev, "Cannot init phy_db\n");
5805                 goto fail;
5806         }
5807
5808         /* Set EBS as successful as long as not stated otherwise by the FW. */
5809         sc->last_ebs_successful = TRUE;
5810
5811         /* PCI attach */
5812         error = iwm_pci_attach(dev);
5813         if (error != 0)
5814                 goto fail;
5815
5816         sc->sc_wantresp = -1;
5817
5818         /* Match device id */
5819         error = iwm_dev_check(dev);
5820         if (error != 0)
5821                 goto fail;
5822
5823         sc->sc_hw_rev = IWM_READ(sc, IWM_CSR_HW_REV);
5824         /*
5825          * In the 8000 HW family the format of the 4 bytes of CSR_HW_REV have
5826          * changed, and now the revision step also includes bit 0-1 (no more
5827          * "dash" value). To keep hw_rev backwards compatible - we'll store it
5828          * in the old format.
5829          */
5830         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) {
5831                 int ret;
5832                 uint32_t hw_step;
5833
5834                 sc->sc_hw_rev = (sc->sc_hw_rev & 0xfff0) |
5835                                 (IWM_CSR_HW_REV_STEP(sc->sc_hw_rev << 2) << 2);
5836
5837                 if (iwm_prepare_card_hw(sc) != 0) {
5838                         device_printf(dev, "could not initialize hardware\n");
5839                         goto fail;
5840                 }
5841
5842                 /*
5843                  * In order to recognize C step the driver should read the
5844                  * chip version id located at the AUX bus MISC address.
5845                  */
5846                 IWM_SETBITS(sc, IWM_CSR_GP_CNTRL,
5847                             IWM_CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
5848                 DELAY(2);
5849
5850                 ret = iwm_poll_bit(sc, IWM_CSR_GP_CNTRL,
5851                                    IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
5852                                    IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
5853                                    25000);
5854                 if (!ret) {
5855                         device_printf(sc->sc_dev,
5856                             "Failed to wake up the nic\n");
5857                         goto fail;
5858                 }
5859
5860                 if (iwm_nic_lock(sc)) {
5861                         hw_step = iwm_read_prph(sc, IWM_WFPM_CTRL_REG);
5862                         hw_step |= IWM_ENABLE_WFPM;
5863                         iwm_write_prph(sc, IWM_WFPM_CTRL_REG, hw_step);
5864                         hw_step = iwm_read_prph(sc, IWM_AUX_MISC_REG);
5865                         hw_step = (hw_step >> IWM_HW_STEP_LOCATION_BITS) & 0xF;
5866                         if (hw_step == 0x3)
5867                                 sc->sc_hw_rev = (sc->sc_hw_rev & 0xFFFFFFF3) |
5868                                                 (IWM_SILICON_C_STEP << 2);
5869                         iwm_nic_unlock(sc);
5870                 } else {
5871                         device_printf(sc->sc_dev, "Failed to lock the nic\n");
5872                         goto fail;
5873                 }
5874         }
5875
5876         /* special-case 7265D, it has the same PCI IDs. */
5877         if (sc->cfg == &iwm7265_cfg &&
5878             (sc->sc_hw_rev & IWM_CSR_HW_REV_TYPE_MSK) == IWM_CSR_HW_REV_TYPE_7265D) {
5879                 sc->cfg = &iwm7265d_cfg;
5880         }
5881
5882         /* Allocate DMA memory for firmware transfers. */
5883         if ((error = iwm_alloc_fwmem(sc)) != 0) {
5884                 device_printf(dev, "could not allocate memory for firmware\n");
5885                 goto fail;
5886         }
5887
5888         /* Allocate "Keep Warm" page. */
5889         if ((error = iwm_alloc_kw(sc)) != 0) {
5890                 device_printf(dev, "could not allocate keep warm page\n");
5891                 goto fail;
5892         }
5893
5894         /* We use ICT interrupts */
5895         if ((error = iwm_alloc_ict(sc)) != 0) {
5896                 device_printf(dev, "could not allocate ICT table\n");
5897                 goto fail;
5898         }
5899
5900         /* Allocate TX scheduler "rings". */
5901         if ((error = iwm_alloc_sched(sc)) != 0) {
5902                 device_printf(dev, "could not allocate TX scheduler rings\n");
5903                 goto fail;
5904         }
5905
5906         /* Allocate TX rings */
5907         for (txq_i = 0; txq_i < nitems(sc->txq); txq_i++) {
5908                 if ((error = iwm_alloc_tx_ring(sc,
5909                     &sc->txq[txq_i], txq_i)) != 0) {
5910                         device_printf(dev,
5911                             "could not allocate TX ring %d\n",
5912                             txq_i);
5913                         goto fail;
5914                 }
5915         }
5916
5917         /* Allocate RX ring. */
5918         if ((error = iwm_alloc_rx_ring(sc, &sc->rxq)) != 0) {
5919                 device_printf(dev, "could not allocate RX ring\n");
5920                 goto fail;
5921         }
5922
5923         /* Clear pending interrupts. */
5924         IWM_WRITE(sc, IWM_CSR_INT, 0xffffffff);
5925
5926         ic->ic_softc = sc;
5927         ic->ic_name = device_get_nameunit(sc->sc_dev);
5928         ic->ic_phytype = IEEE80211_T_OFDM;      /* not only, but not used */
5929         ic->ic_opmode = IEEE80211_M_STA;        /* default to BSS mode */
5930
5931         /* Set device capabilities. */
5932         ic->ic_caps =
5933             IEEE80211_C_STA |
5934             IEEE80211_C_WPA |           /* WPA/RSN */
5935             IEEE80211_C_WME |
5936             IEEE80211_C_PMGT |
5937             IEEE80211_C_SHSLOT |        /* short slot time supported */
5938             IEEE80211_C_SHPREAMBLE      /* short preamble supported */
5939 //          IEEE80211_C_BGSCAN          /* capable of bg scanning */
5940             ;
5941         /* Advertise full-offload scanning */
5942         ic->ic_flags_ext = IEEE80211_FEXT_SCAN_OFFLOAD;
5943         for (i = 0; i < nitems(sc->sc_phyctxt); i++) {
5944                 sc->sc_phyctxt[i].id = i;
5945                 sc->sc_phyctxt[i].color = 0;
5946                 sc->sc_phyctxt[i].ref = 0;
5947                 sc->sc_phyctxt[i].channel = NULL;
5948         }
5949
5950         /* Default noise floor */
5951         sc->sc_noise = -96;
5952
5953         /* Max RSSI */
5954         sc->sc_max_rssi = IWM_MAX_DBM - IWM_MIN_DBM;
5955
5956 #ifdef IWM_DEBUG
5957         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5958             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug",
5959             CTLFLAG_RW, &sc->sc_debug, 0, "control debugging");
5960 #endif
5961
5962         error = iwm_read_firmware(sc);
5963         if (error) {
5964                 goto fail;
5965         } else if (sc->sc_fw.fw_fp == NULL) {
5966                 /*
5967                  * XXX Add a solution for properly deferring firmware load
5968                  *     during bootup.
5969                  */
5970                 goto fail;
5971         } else {
5972                 sc->sc_preinit_hook.ich_func = iwm_preinit;
5973                 sc->sc_preinit_hook.ich_arg = sc;
5974                 if (config_intrhook_establish(&sc->sc_preinit_hook) != 0) {
5975                         device_printf(dev,
5976                             "config_intrhook_establish failed\n");
5977                         goto fail;
5978                 }
5979         }
5980
5981         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
5982             "<-%s\n", __func__);
5983
5984         return 0;
5985
5986         /* Free allocated memory if something failed during attachment. */
5987 fail:
5988         iwm_detach_local(sc, 0);
5989
5990         return ENXIO;
5991 }
5992
5993 static int
5994 iwm_is_valid_ether_addr(uint8_t *addr)
5995 {
5996         char zero_addr[IEEE80211_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 };
5997
5998         if ((addr[0] & 1) || IEEE80211_ADDR_EQ(zero_addr, addr))
5999                 return (FALSE);
6000
6001         return (TRUE);
6002 }
6003
6004 static int
6005 iwm_wme_update(struct ieee80211com *ic)
6006 {
6007 #define IWM_EXP2(x)     ((1 << (x)) - 1)        /* CWmin = 2^ECWmin - 1 */
6008         struct iwm_softc *sc = ic->ic_softc;
6009         struct chanAccParams chp;
6010         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
6011         struct iwm_vap *ivp = IWM_VAP(vap);
6012         struct iwm_node *in;
6013         struct wmeParams tmp[WME_NUM_AC];
6014         int aci, error;
6015
6016         if (vap == NULL)
6017                 return (0);
6018
6019         ieee80211_wme_ic_getparams(ic, &chp);
6020
6021         IEEE80211_LOCK(ic);
6022         for (aci = 0; aci < WME_NUM_AC; aci++)
6023                 tmp[aci] = chp.cap_wmeParams[aci];
6024         IEEE80211_UNLOCK(ic);
6025
6026         IWM_LOCK(sc);
6027         for (aci = 0; aci < WME_NUM_AC; aci++) {
6028                 const struct wmeParams *ac = &tmp[aci];
6029                 ivp->queue_params[aci].aifsn = ac->wmep_aifsn;
6030                 ivp->queue_params[aci].cw_min = IWM_EXP2(ac->wmep_logcwmin);
6031                 ivp->queue_params[aci].cw_max = IWM_EXP2(ac->wmep_logcwmax);
6032                 ivp->queue_params[aci].edca_txop =
6033                     IEEE80211_TXOP_TO_US(ac->wmep_txopLimit);
6034         }
6035         ivp->have_wme = TRUE;
6036         if (ivp->is_uploaded && vap->iv_bss != NULL) {
6037                 in = IWM_NODE(vap->iv_bss);
6038                 if (in->in_assoc) {
6039                         if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) {
6040                                 device_printf(sc->sc_dev,
6041                                     "%s: failed to update MAC\n", __func__);
6042                         }
6043                 }
6044         }
6045         IWM_UNLOCK(sc);
6046
6047         return (0);
6048 #undef IWM_EXP2
6049 }
6050
6051 static void
6052 iwm_preinit(void *arg)
6053 {
6054         struct iwm_softc *sc = arg;
6055         device_t dev = sc->sc_dev;
6056         struct ieee80211com *ic = &sc->sc_ic;
6057         int error;
6058
6059         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6060             "->%s\n", __func__);
6061
6062         IWM_LOCK(sc);
6063         if ((error = iwm_start_hw(sc)) != 0) {
6064                 device_printf(dev, "could not initialize hardware\n");
6065                 IWM_UNLOCK(sc);
6066                 goto fail;
6067         }
6068
6069         error = iwm_run_init_mvm_ucode(sc, 1);
6070         iwm_stop_device(sc);
6071         if (error) {
6072                 IWM_UNLOCK(sc);
6073                 goto fail;
6074         }
6075         device_printf(dev,
6076             "hw rev 0x%x, fw ver %s, address %s\n",
6077             sc->sc_hw_rev & IWM_CSR_HW_REV_TYPE_MSK,
6078             sc->sc_fwver, ether_sprintf(sc->nvm_data->hw_addr));
6079
6080         /* not all hardware can do 5GHz band */
6081         if (!sc->nvm_data->sku_cap_band_52GHz_enable)
6082                 memset(&ic->ic_sup_rates[IEEE80211_MODE_11A], 0,
6083                     sizeof(ic->ic_sup_rates[IEEE80211_MODE_11A]));
6084         IWM_UNLOCK(sc);
6085
6086         iwm_init_channel_map(ic, IEEE80211_CHAN_MAX, &ic->ic_nchans,
6087             ic->ic_channels);
6088
6089         /*
6090          * At this point we've committed - if we fail to do setup,
6091          * we now also have to tear down the net80211 state.
6092          */
6093         ieee80211_ifattach(ic);
6094         ic->ic_vap_create = iwm_vap_create;
6095         ic->ic_vap_delete = iwm_vap_delete;
6096         ic->ic_raw_xmit = iwm_raw_xmit;
6097         ic->ic_node_alloc = iwm_node_alloc;
6098         ic->ic_scan_start = iwm_scan_start;
6099         ic->ic_scan_end = iwm_scan_end;
6100         ic->ic_update_mcast = iwm_update_mcast;
6101         ic->ic_getradiocaps = iwm_init_channel_map;
6102         ic->ic_set_channel = iwm_set_channel;
6103         ic->ic_scan_curchan = iwm_scan_curchan;
6104         ic->ic_scan_mindwell = iwm_scan_mindwell;
6105         ic->ic_wme.wme_update = iwm_wme_update;
6106         ic->ic_parent = iwm_parent;
6107         ic->ic_transmit = iwm_transmit;
6108         iwm_radiotap_attach(sc);
6109         if (bootverbose)
6110                 ieee80211_announce(ic);
6111
6112         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6113             "<-%s\n", __func__);
6114         config_intrhook_disestablish(&sc->sc_preinit_hook);
6115
6116         return;
6117 fail:
6118         config_intrhook_disestablish(&sc->sc_preinit_hook);
6119         iwm_detach_local(sc, 0);
6120 }
6121
6122 /*
6123  * Attach the interface to 802.11 radiotap.
6124  */
6125 static void
6126 iwm_radiotap_attach(struct iwm_softc *sc)
6127 {
6128         struct ieee80211com *ic = &sc->sc_ic;
6129
6130         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6131             "->%s begin\n", __func__);
6132         ieee80211_radiotap_attach(ic,
6133             &sc->sc_txtap.wt_ihdr, sizeof(sc->sc_txtap),
6134                 IWM_TX_RADIOTAP_PRESENT,
6135             &sc->sc_rxtap.wr_ihdr, sizeof(sc->sc_rxtap),
6136                 IWM_RX_RADIOTAP_PRESENT);
6137         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6138             "->%s end\n", __func__);
6139 }
6140
6141 static struct ieee80211vap *
6142 iwm_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], int unit,
6143     enum ieee80211_opmode opmode, int flags,
6144     const uint8_t bssid[IEEE80211_ADDR_LEN],
6145     const uint8_t mac[IEEE80211_ADDR_LEN])
6146 {
6147         struct iwm_vap *ivp;
6148         struct ieee80211vap *vap;
6149
6150         if (!TAILQ_EMPTY(&ic->ic_vaps))         /* only one at a time */
6151                 return NULL;
6152         ivp = malloc(sizeof(struct iwm_vap), M_80211_VAP, M_WAITOK | M_ZERO);
6153         vap = &ivp->iv_vap;
6154         ieee80211_vap_setup(ic, vap, name, unit, opmode, flags, bssid);
6155         vap->iv_bmissthreshold = 10;            /* override default */
6156         /* Override with driver methods. */
6157         ivp->iv_newstate = vap->iv_newstate;
6158         vap->iv_newstate = iwm_newstate;
6159
6160         ivp->id = IWM_DEFAULT_MACID;
6161         ivp->color = IWM_DEFAULT_COLOR;
6162
6163         ivp->have_wme = FALSE;
6164         ivp->ps_disabled = FALSE;
6165
6166         ieee80211_ratectl_init(vap);
6167         /* Complete setup. */
6168         ieee80211_vap_attach(vap, iwm_media_change, ieee80211_media_status,
6169             mac);
6170         ic->ic_opmode = opmode;
6171
6172         return vap;
6173 }
6174
6175 static void
6176 iwm_vap_delete(struct ieee80211vap *vap)
6177 {
6178         struct iwm_vap *ivp = IWM_VAP(vap);
6179
6180         ieee80211_ratectl_deinit(vap);
6181         ieee80211_vap_detach(vap);
6182         free(ivp, M_80211_VAP);
6183 }
6184
6185 static void
6186 iwm_xmit_queue_drain(struct iwm_softc *sc)
6187 {
6188         struct mbuf *m;
6189         struct ieee80211_node *ni;
6190
6191         while ((m = mbufq_dequeue(&sc->sc_snd)) != NULL) {
6192                 ni = (struct ieee80211_node *)m->m_pkthdr.rcvif;
6193                 ieee80211_free_node(ni);
6194                 m_freem(m);
6195         }
6196 }
6197
6198 static void
6199 iwm_scan_start(struct ieee80211com *ic)
6200 {
6201         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
6202         struct iwm_softc *sc = ic->ic_softc;
6203         int error;
6204
6205         IWM_LOCK(sc);
6206         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
6207                 /* This should not be possible */
6208                 device_printf(sc->sc_dev,
6209                     "%s: Previous scan not completed yet\n", __func__);
6210         }
6211         if (fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_UMAC_SCAN))
6212                 error = iwm_mvm_umac_scan(sc);
6213         else
6214                 error = iwm_mvm_lmac_scan(sc);
6215         if (error != 0) {
6216                 device_printf(sc->sc_dev, "could not initiate scan\n");
6217                 IWM_UNLOCK(sc);
6218                 ieee80211_cancel_scan(vap);
6219         } else {
6220                 sc->sc_flags |= IWM_FLAG_SCAN_RUNNING;
6221                 iwm_led_blink_start(sc);
6222                 IWM_UNLOCK(sc);
6223         }
6224 }
6225
6226 static void
6227 iwm_scan_end(struct ieee80211com *ic)
6228 {
6229         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
6230         struct iwm_softc *sc = ic->ic_softc;
6231
6232         IWM_LOCK(sc);
6233         iwm_led_blink_stop(sc);
6234         if (vap->iv_state == IEEE80211_S_RUN)
6235                 iwm_mvm_led_enable(sc);
6236         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
6237                 /*
6238                  * Removing IWM_FLAG_SCAN_RUNNING now, is fine because
6239                  * both iwm_scan_end and iwm_scan_start run in the ic->ic_tq
6240                  * taskqueue.
6241                  */
6242                 sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
6243                 iwm_mvm_scan_stop_wait(sc);
6244         }
6245         IWM_UNLOCK(sc);
6246
6247         /*
6248          * Make sure we don't race, if sc_es_task is still enqueued here.
6249          * This is to make sure that it won't call ieee80211_scan_done
6250          * when we have already started the next scan.
6251          */
6252         taskqueue_cancel(ic->ic_tq, &sc->sc_es_task, NULL);
6253 }
6254
6255 static void
6256 iwm_update_mcast(struct ieee80211com *ic)
6257 {
6258 }
6259
6260 static void
6261 iwm_set_channel(struct ieee80211com *ic)
6262 {
6263 }
6264
6265 static void
6266 iwm_scan_curchan(struct ieee80211_scan_state *ss, unsigned long maxdwell)
6267 {
6268 }
6269
6270 static void
6271 iwm_scan_mindwell(struct ieee80211_scan_state *ss)
6272 {
6273         return;
6274 }
6275
6276 void
6277 iwm_init_task(void *arg1)
6278 {
6279         struct iwm_softc *sc = arg1;
6280
6281         IWM_LOCK(sc);
6282         while (sc->sc_flags & IWM_FLAG_BUSY)
6283                 msleep(&sc->sc_flags, &sc->sc_mtx, 0, "iwmpwr", 0);
6284         sc->sc_flags |= IWM_FLAG_BUSY;
6285         iwm_stop(sc);
6286         if (sc->sc_ic.ic_nrunning > 0)
6287                 iwm_init(sc);
6288         sc->sc_flags &= ~IWM_FLAG_BUSY;
6289         wakeup(&sc->sc_flags);
6290         IWM_UNLOCK(sc);
6291 }
6292
6293 static int
6294 iwm_resume(device_t dev)
6295 {
6296         struct iwm_softc *sc = device_get_softc(dev);
6297         int do_reinit = 0;
6298
6299         /*
6300          * We disable the RETRY_TIMEOUT register (0x41) to keep
6301          * PCI Tx retries from interfering with C3 CPU state.
6302          */
6303         pci_write_config(dev, PCI_CFG_RETRY_TIMEOUT, 0x00, 1);
6304         iwm_init_task(device_get_softc(dev));
6305
6306         IWM_LOCK(sc);
6307         if (sc->sc_flags & IWM_FLAG_SCANNING) {
6308                 sc->sc_flags &= ~IWM_FLAG_SCANNING;
6309                 do_reinit = 1;
6310         }
6311         IWM_UNLOCK(sc);
6312
6313         if (do_reinit)
6314                 ieee80211_resume_all(&sc->sc_ic);
6315
6316         return 0;
6317 }
6318
6319 static int
6320 iwm_suspend(device_t dev)
6321 {
6322         int do_stop = 0;
6323         struct iwm_softc *sc = device_get_softc(dev);
6324
6325         do_stop = !! (sc->sc_ic.ic_nrunning > 0);
6326
6327         ieee80211_suspend_all(&sc->sc_ic);
6328
6329         if (do_stop) {
6330                 IWM_LOCK(sc);
6331                 iwm_stop(sc);
6332                 sc->sc_flags |= IWM_FLAG_SCANNING;
6333                 IWM_UNLOCK(sc);
6334         }
6335
6336         return (0);
6337 }
6338
6339 static int
6340 iwm_detach_local(struct iwm_softc *sc, int do_net80211)
6341 {
6342         struct iwm_fw_info *fw = &sc->sc_fw;
6343         device_t dev = sc->sc_dev;
6344         int i;
6345
6346         if (!sc->sc_attached)
6347                 return 0;
6348         sc->sc_attached = 0;
6349
6350         if (do_net80211)
6351                 ieee80211_draintask(&sc->sc_ic, &sc->sc_es_task);
6352
6353         callout_drain(&sc->sc_led_blink_to);
6354         callout_drain(&sc->sc_watchdog_to);
6355         iwm_stop_device(sc);
6356         if (do_net80211) {
6357                 IWM_LOCK(sc);
6358                 iwm_xmit_queue_drain(sc);
6359                 IWM_UNLOCK(sc);
6360                 ieee80211_ifdetach(&sc->sc_ic);
6361         }
6362
6363         iwm_phy_db_free(sc->sc_phy_db);
6364         sc->sc_phy_db = NULL;
6365
6366         iwm_free_nvm_data(sc->nvm_data);
6367
6368         /* Free descriptor rings */
6369         iwm_free_rx_ring(sc, &sc->rxq);
6370         for (i = 0; i < nitems(sc->txq); i++)
6371                 iwm_free_tx_ring(sc, &sc->txq[i]);
6372
6373         /* Free firmware */
6374         if (fw->fw_fp != NULL)
6375                 iwm_fw_info_free(fw);
6376
6377         /* Free scheduler */
6378         iwm_dma_contig_free(&sc->sched_dma);
6379         iwm_dma_contig_free(&sc->ict_dma);
6380         iwm_dma_contig_free(&sc->kw_dma);
6381         iwm_dma_contig_free(&sc->fw_dma);
6382
6383         iwm_free_fw_paging(sc);
6384
6385         /* Finished with the hardware - detach things */
6386         iwm_pci_detach(dev);
6387
6388         if (sc->sc_notif_wait != NULL) {
6389                 iwm_notification_wait_free(sc->sc_notif_wait);
6390                 sc->sc_notif_wait = NULL;
6391         }
6392
6393         IWM_LOCK_DESTROY(sc);
6394
6395         return (0);
6396 }
6397
6398 static int
6399 iwm_detach(device_t dev)
6400 {
6401         struct iwm_softc *sc = device_get_softc(dev);
6402
6403         return (iwm_detach_local(sc, 1));
6404 }
6405
6406 static device_method_t iwm_pci_methods[] = {
6407         /* Device interface */
6408         DEVMETHOD(device_probe,         iwm_probe),
6409         DEVMETHOD(device_attach,        iwm_attach),
6410         DEVMETHOD(device_detach,        iwm_detach),
6411         DEVMETHOD(device_suspend,       iwm_suspend),
6412         DEVMETHOD(device_resume,        iwm_resume),
6413
6414         DEVMETHOD_END
6415 };
6416
6417 static driver_t iwm_pci_driver = {
6418         "iwm",
6419         iwm_pci_methods,
6420         sizeof (struct iwm_softc)
6421 };
6422
6423 static devclass_t iwm_devclass;
6424
6425 DRIVER_MODULE(iwm, pci, iwm_pci_driver, iwm_devclass, NULL, NULL);
6426 MODULE_PNP_INFO("U16:device;P:#;T:vendor=0x8086", pci, iwm_pci_driver,
6427     iwm_devices, nitems(iwm_devices));
6428 MODULE_DEPEND(iwm, firmware, 1, 1, 1);
6429 MODULE_DEPEND(iwm, pci, 1, 1, 1);
6430 MODULE_DEPEND(iwm, wlan, 1, 1, 1);