sys/dev/iwm/if_iwm.c

   1 /*      $OpenBSD: if_iwm.c,v 1.42 2015/05/30 02:49:23 deraadt Exp $     */
   2
   3 /*
   4  * Copyright (c) 2014 genua mbh <info@genua.de>
   5  * Copyright (c) 2014 Fixup Software Ltd.
   6  *
   7  * Permission to use, copy, modify, and distribute this software for any
   8  * purpose with or without fee is hereby granted, provided that the above
   9  * copyright notice and this permission notice appear in all copies.
  10  *
  11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  18  */
  19
  20 /*-
  21  * Based on BSD-licensed source modules in the Linux iwlwifi driver,
  22  * which were used as the reference documentation for this implementation.
  23  *
  24  * Driver version we are currently based off of is
  25  * Linux 3.14.3 (tag id a2df521e42b1d9a23f620ac79dbfe8655a8391dd)
  26  *
  27  ***********************************************************************
  28  *
  29  * This file is provided under a dual BSD/GPLv2 license.  When using or
  30  * redistributing this file, you may do so under either license.
  31  *
  32  * GPL LICENSE SUMMARY
  33  *
  34  * Copyright(c) 2007 - 2013 Intel Corporation. All rights reserved.
  35  *
  36  * This program is free software; you can redistribute it and/or modify
  37  * it under the terms of version 2 of the GNU General Public License as
  38  * published by the Free Software Foundation.
  39  *
  40  * This program is distributed in the hope that it will be useful, but
  41  * WITHOUT ANY WARRANTY; without even the implied warranty of
  42  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  43  * General Public License for more details.
  44  *
  45  * You should have received a copy of the GNU General Public License
  46  * along with this program; if not, write to the Free Software
  47  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
  48  * USA
  49  *
  50  * The full GNU General Public License is included in this distribution
  51  * in the file called COPYING.
  52  *
  53  * Contact Information:
  54  *  Intel Linux Wireless <ilw@linux.intel.com>
  55  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  56  *
  57  *
  58  * BSD LICENSE
  59  *
  60  * Copyright(c) 2005 - 2013 Intel Corporation. All rights reserved.
  61  * All rights reserved.
  62  *
  63  * Redistribution and use in source and binary forms, with or without
  64  * modification, are permitted provided that the following conditions
  65  * are met:
  66  *
  67  *  * Redistributions of source code must retain the above copyright
  68  *    notice, this list of conditions and the following disclaimer.
  69  *  * Redistributions in binary form must reproduce the above copyright
  70  *    notice, this list of conditions and the following disclaimer in
  71  *    the documentation and/or other materials provided with the
  72  *    distribution.
  73  *  * Neither the name Intel Corporation nor the names of its
  74  *    contributors may be used to endorse or promote products derived
  75  *    from this software without specific prior written permission.
  76  *
  77  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  78  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  79  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  80  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  81  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  82  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  83  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  84  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  85  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  86  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  87  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  88  */
  89
  90 /*-
  91  * Copyright (c) 2007-2010 Damien Bergamini <damien.bergamini@free.fr>
  92  *
  93  * Permission to use, copy, modify, and distribute this software for any
  94  * purpose with or without fee is hereby granted, provided that the above
  95  * copyright notice and this permission notice appear in all copies.
  96  *
  97  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  98  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  99  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 100  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 101  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 102  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 103  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 104  */
 105 #include <sys/cdefs.h>
 106 __FBSDID("$FreeBSD$");
 107
 108 #include "opt_wlan.h"
 109
 110 #include <sys/param.h>
 111 #include <sys/bus.h>
 112 #include <sys/conf.h>
 113 #include <sys/endian.h>
 114 #include <sys/firmware.h>
 115 #include <sys/kernel.h>
 116 #include <sys/malloc.h>
 117 #include <sys/mbuf.h>
 118 #include <sys/mutex.h>
 119 #include <sys/module.h>
 120 #include <sys/proc.h>
 121 #include <sys/rman.h>
 122 #include <sys/socket.h>
 123 #include <sys/sockio.h>
 124 #include <sys/sysctl.h>
 125 #include <sys/linker.h>
 126
 127 #include <machine/bus.h>
 128 #include <machine/endian.h>
 129 #include <machine/resource.h>
 130
 131 #include <dev/pci/pcivar.h>
 132 #include <dev/pci/pcireg.h>
 133
 134 #include <net/bpf.h>
 135
 136 #include <net/if.h>
 137 #include <net/if_var.h>
 138 #include <net/if_arp.h>
 139 #include <net/if_dl.h>
 140 #include <net/if_media.h>
 141 #include <net/if_types.h>
 142
 143 #include <netinet/in.h>
 144 #include <netinet/in_systm.h>
 145 #include <netinet/if_ether.h>
 146 #include <netinet/ip.h>
 147
 148 #include <net80211/ieee80211_var.h>
 149 #include <net80211/ieee80211_regdomain.h>
 150 #include <net80211/ieee80211_ratectl.h>
 151 #include <net80211/ieee80211_radiotap.h>
 152
 153 #include <dev/iwm/if_iwmreg.h>
 154 #include <dev/iwm/if_iwmvar.h>
 155 #include <dev/iwm/if_iwm_config.h>
 156 #include <dev/iwm/if_iwm_debug.h>
 157 #include <dev/iwm/if_iwm_notif_wait.h>
 158 #include <dev/iwm/if_iwm_util.h>
 159 #include <dev/iwm/if_iwm_binding.h>
 160 #include <dev/iwm/if_iwm_phy_db.h>
 161 #include <dev/iwm/if_iwm_mac_ctxt.h>
 162 #include <dev/iwm/if_iwm_phy_ctxt.h>
 163 #include <dev/iwm/if_iwm_time_event.h>
 164 #include <dev/iwm/if_iwm_power.h>
 165 #include <dev/iwm/if_iwm_scan.h>
 166
 167 #include <dev/iwm/if_iwm_pcie_trans.h>
 168 #include <dev/iwm/if_iwm_led.h>
 169 #include <dev/iwm/if_iwm_fw.h>
 170
 171 const uint8_t iwm_nvm_channels[] = {
 172         /* 2.4 GHz */
 173         1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
 174         /* 5 GHz */
 175         36, 40, 44, 48, 52, 56, 60, 64,
 176         100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144,
 177         149, 153, 157, 161, 165
 178 };
 179 _Static_assert(nitems(iwm_nvm_channels) <= IWM_NUM_CHANNELS,
 180     "IWM_NUM_CHANNELS is too small");
 181
 182 const uint8_t iwm_nvm_channels_8000[] = {
 183         /* 2.4 GHz */
 184         1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
 185         /* 5 GHz */
 186         36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92,
 187         96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144,
 188         149, 153, 157, 161, 165, 169, 173, 177, 181
 189 };
 190 _Static_assert(nitems(iwm_nvm_channels_8000) <= IWM_NUM_CHANNELS_8000,
 191     "IWM_NUM_CHANNELS_8000 is too small");
 192
 193 #define IWM_NUM_2GHZ_CHANNELS   14
 194 #define IWM_N_HW_ADDR_MASK      0xF
 195
 196 /*
 197  * XXX For now, there's simply a fixed set of rate table entries
 198  * that are populated.
 199  */
 200 const struct iwm_rate {
 201         uint8_t rate;
 202         uint8_t plcp;
 203 } iwm_rates[] = {
 204         {   2,  IWM_RATE_1M_PLCP  },
 205         {   4,  IWM_RATE_2M_PLCP  },
 206         {  11,  IWM_RATE_5M_PLCP  },
 207         {  22,  IWM_RATE_11M_PLCP },
 208         {  12,  IWM_RATE_6M_PLCP  },
 209         {  18,  IWM_RATE_9M_PLCP  },
 210         {  24,  IWM_RATE_12M_PLCP },
 211         {  36,  IWM_RATE_18M_PLCP },
 212         {  48,  IWM_RATE_24M_PLCP },
 213         {  72,  IWM_RATE_36M_PLCP },
 214         {  96,  IWM_RATE_48M_PLCP },
 215         { 108,  IWM_RATE_54M_PLCP },
 216 };
 217 #define IWM_RIDX_CCK    0
 218 #define IWM_RIDX_OFDM   4
 219 #define IWM_RIDX_MAX    (nitems(iwm_rates)-1)
 220 #define IWM_RIDX_IS_CCK(_i_) ((_i_) < IWM_RIDX_OFDM)
 221 #define IWM_RIDX_IS_OFDM(_i_) ((_i_) >= IWM_RIDX_OFDM)
 222
 223 struct iwm_nvm_section {
 224         uint16_t length;
 225         uint8_t *data;
 226 };
 227
 228 #define IWM_MVM_UCODE_ALIVE_TIMEOUT     hz
 229 #define IWM_MVM_UCODE_CALIB_TIMEOUT     (2*hz)
 230
 231 struct iwm_mvm_alive_data {
 232         int valid;
 233         uint32_t scd_base_addr;
 234 };
 235
 236 static int      iwm_store_cscheme(struct iwm_softc *, const uint8_t *, size_t);
 237 static int      iwm_firmware_store_section(struct iwm_softc *,
 238                                            enum iwm_ucode_type,
 239                                            const uint8_t *, size_t);
 240 static int      iwm_set_default_calib(struct iwm_softc *, const void *);
 241 static void     iwm_fw_info_free(struct iwm_fw_info *);
 242 static int      iwm_read_firmware(struct iwm_softc *, enum iwm_ucode_type);
 243 static int      iwm_alloc_fwmem(struct iwm_softc *);
 244 static int      iwm_alloc_sched(struct iwm_softc *);
 245 static int      iwm_alloc_kw(struct iwm_softc *);
 246 static int      iwm_alloc_ict(struct iwm_softc *);
 247 static int      iwm_alloc_rx_ring(struct iwm_softc *, struct iwm_rx_ring *);
 248 static void     iwm_reset_rx_ring(struct iwm_softc *, struct iwm_rx_ring *);
 249 static void     iwm_free_rx_ring(struct iwm_softc *, struct iwm_rx_ring *);
 250 static int      iwm_alloc_tx_ring(struct iwm_softc *, struct iwm_tx_ring *,
 251                                   int);
 252 static void     iwm_reset_tx_ring(struct iwm_softc *, struct iwm_tx_ring *);
 253 static void     iwm_free_tx_ring(struct iwm_softc *, struct iwm_tx_ring *);
 254 static void     iwm_enable_interrupts(struct iwm_softc *);
 255 static void     iwm_restore_interrupts(struct iwm_softc *);
 256 static void     iwm_disable_interrupts(struct iwm_softc *);
 257 static void     iwm_ict_reset(struct iwm_softc *);
 258 static int      iwm_allow_mcast(struct ieee80211vap *, struct iwm_softc *);
 259 static void     iwm_stop_device(struct iwm_softc *);
 260 static void     iwm_mvm_nic_config(struct iwm_softc *);
 261 static int      iwm_nic_rx_init(struct iwm_softc *);
 262 static int      iwm_nic_tx_init(struct iwm_softc *);
 263 static int      iwm_nic_init(struct iwm_softc *);
 264 static int      iwm_enable_txq(struct iwm_softc *, int, int, int);
 265 static int      iwm_trans_pcie_fw_alive(struct iwm_softc *, uint32_t);
 266 static int      iwm_nvm_read_chunk(struct iwm_softc *, uint16_t, uint16_t,
 267                                    uint16_t, uint8_t *, uint16_t *);
 268 static int      iwm_nvm_read_section(struct iwm_softc *, uint16_t, uint8_t *,
 269                                      uint16_t *, uint32_t);
 270 static uint32_t iwm_eeprom_channel_flags(uint16_t);
 271 static void     iwm_add_channel_band(struct iwm_softc *,
 272                     struct ieee80211_channel[], int, int *, int, size_t,
 273                     const uint8_t[]);
 274 static void     iwm_init_channel_map(struct ieee80211com *, int, int *,
 275                     struct ieee80211_channel[]);
 276 static struct iwm_nvm_data *
 277         iwm_parse_nvm_data(struct iwm_softc *, const uint16_t *,
 278                            const uint16_t *, const uint16_t *,
 279                            const uint16_t *, const uint16_t *,
 280                            const uint16_t *);
 281 static void     iwm_free_nvm_data(struct iwm_nvm_data *);
 282 static void     iwm_set_hw_address_family_8000(struct iwm_softc *,
 283                                                struct iwm_nvm_data *,
 284                                                const uint16_t *,
 285                                                const uint16_t *);
 286 static int      iwm_get_sku(const struct iwm_softc *, const uint16_t *,
 287                             const uint16_t *);
 288 static int      iwm_get_nvm_version(const struct iwm_softc *, const uint16_t *);
 289 static int      iwm_get_radio_cfg(const struct iwm_softc *, const uint16_t *,
 290                                   const uint16_t *);
 291 static int      iwm_get_n_hw_addrs(const struct iwm_softc *,
 292                                    const uint16_t *);
 293 static void     iwm_set_radio_cfg(const struct iwm_softc *,
 294                                   struct iwm_nvm_data *, uint32_t);
 295 static struct iwm_nvm_data *
 296         iwm_parse_nvm_sections(struct iwm_softc *, struct iwm_nvm_section *);
 297 static int      iwm_nvm_init(struct iwm_softc *);
 298 static int      iwm_pcie_load_section(struct iwm_softc *, uint8_t,
 299                                       const struct iwm_fw_desc *);
 300 static int      iwm_pcie_load_firmware_chunk(struct iwm_softc *, uint32_t,
 301                                              bus_addr_t, uint32_t);
 302 static int      iwm_pcie_load_cpu_sections_8000(struct iwm_softc *sc,
 303                                                 const struct iwm_fw_sects *,
 304                                                 int, int *);
 305 static int      iwm_pcie_load_cpu_sections(struct iwm_softc *,
 306                                            const struct iwm_fw_sects *,
 307                                            int, int *);
 308 static int      iwm_pcie_load_given_ucode_8000(struct iwm_softc *,
 309                                                const struct iwm_fw_sects *);
 310 static int      iwm_pcie_load_given_ucode(struct iwm_softc *,
 311                                           const struct iwm_fw_sects *);
 312 static int      iwm_start_fw(struct iwm_softc *, const struct iwm_fw_sects *);
 313 static int      iwm_send_tx_ant_cfg(struct iwm_softc *, uint8_t);
 314 static int      iwm_send_phy_cfg_cmd(struct iwm_softc *);
 315 static int      iwm_mvm_load_ucode_wait_alive(struct iwm_softc *,
 316                                               enum iwm_ucode_type);
 317 static int      iwm_run_init_mvm_ucode(struct iwm_softc *, int);
 318 static int      iwm_rx_addbuf(struct iwm_softc *, int, int);
 319 static int      iwm_mvm_get_signal_strength(struct iwm_softc *,
 320                                             struct iwm_rx_phy_info *);
 321 static void     iwm_mvm_rx_rx_phy_cmd(struct iwm_softc *,
 322                                       struct iwm_rx_packet *,
 323                                       struct iwm_rx_data *);
 324 static int      iwm_get_noise(struct iwm_softc *sc,
 325                     const struct iwm_mvm_statistics_rx_non_phy *);
 326 static void     iwm_mvm_rx_rx_mpdu(struct iwm_softc *, struct mbuf *);
 327 static int      iwm_mvm_rx_tx_cmd_single(struct iwm_softc *,
 328                                          struct iwm_rx_packet *,
 329                                          struct iwm_node *);
 330 static void     iwm_mvm_rx_tx_cmd(struct iwm_softc *, struct iwm_rx_packet *,
 331                                   struct iwm_rx_data *);
 332 static void     iwm_cmd_done(struct iwm_softc *, struct iwm_rx_packet *);
 333 #if 0
 334 static void     iwm_update_sched(struct iwm_softc *, int, int, uint8_t,
 335                                  uint16_t);
 336 #endif
 337 static const struct iwm_rate *
 338         iwm_tx_fill_cmd(struct iwm_softc *, struct iwm_node *,
 339                         struct mbuf *, struct iwm_tx_cmd *);
 340 static int      iwm_tx(struct iwm_softc *, struct mbuf *,
 341                        struct ieee80211_node *, int);
 342 static int      iwm_raw_xmit(struct ieee80211_node *, struct mbuf *,
 343                              const struct ieee80211_bpf_params *);
 344 static int      iwm_mvm_flush_tx_path(struct iwm_softc *sc,
 345                                       uint32_t tfd_msk, uint32_t flags);
 346 static int      iwm_mvm_send_add_sta_cmd_status(struct iwm_softc *,
 347                                                 struct iwm_mvm_add_sta_cmd *,
 348                                                 int *);
 349 static int      iwm_mvm_sta_send_to_fw(struct iwm_softc *, struct iwm_node *,
 350                                        int);
 351 static int      iwm_mvm_add_sta(struct iwm_softc *, struct iwm_node *);
 352 static int      iwm_mvm_update_sta(struct iwm_softc *, struct iwm_node *);
 353 static int      iwm_mvm_add_int_sta_common(struct iwm_softc *,
 354                                            struct iwm_int_sta *,
 355                                            const uint8_t *, uint16_t, uint16_t);
 356 static int      iwm_mvm_add_aux_sta(struct iwm_softc *);
 357 static int      iwm_mvm_update_quotas(struct iwm_softc *, struct iwm_node *);
 358 static int      iwm_auth(struct ieee80211vap *, struct iwm_softc *);
 359 static int      iwm_assoc(struct ieee80211vap *, struct iwm_softc *);
 360 static int      iwm_release(struct iwm_softc *, struct iwm_node *);
 361 static struct ieee80211_node *
 362                 iwm_node_alloc(struct ieee80211vap *,
 363                                const uint8_t[IEEE80211_ADDR_LEN]);
 364 static void     iwm_setrates(struct iwm_softc *, struct iwm_node *);
 365 static int      iwm_media_change(struct ifnet *);
 366 static int      iwm_newstate(struct ieee80211vap *, enum ieee80211_state, int);
 367 static void     iwm_endscan_cb(void *, int);
 368 static void     iwm_mvm_fill_sf_command(struct iwm_softc *,
 369                                         struct iwm_sf_cfg_cmd *,
 370                                         struct ieee80211_node *);
 371 static int      iwm_mvm_sf_config(struct iwm_softc *, enum iwm_sf_state);
 372 static int      iwm_send_bt_init_conf(struct iwm_softc *);
 373 static int      iwm_send_update_mcc_cmd(struct iwm_softc *, const char *);
 374 static void     iwm_mvm_tt_tx_backoff(struct iwm_softc *, uint32_t);
 375 static int      iwm_init_hw(struct iwm_softc *);
 376 static void     iwm_init(struct iwm_softc *);
 377 static void     iwm_start(struct iwm_softc *);
 378 static void     iwm_stop(struct iwm_softc *);
 379 static void     iwm_watchdog(void *);
 380 static void     iwm_parent(struct ieee80211com *);
 381 #ifdef IWM_DEBUG
 382 static const char *
 383                 iwm_desc_lookup(uint32_t);
 384 static void     iwm_nic_error(struct iwm_softc *);
 385 static void     iwm_nic_umac_error(struct iwm_softc *);
 386 #endif
 387 static void     iwm_notif_intr(struct iwm_softc *);
 388 static void     iwm_intr(void *);
 389 static int      iwm_attach(device_t);
 390 static int      iwm_is_valid_ether_addr(uint8_t *);
 391 static void     iwm_preinit(void *);
 392 static int      iwm_detach_local(struct iwm_softc *sc, int);
 393 static void     iwm_init_task(void *);
 394 static void     iwm_radiotap_attach(struct iwm_softc *);
 395 static struct ieee80211vap *
 396                 iwm_vap_create(struct ieee80211com *,
 397                                const char [IFNAMSIZ], int,
 398                                enum ieee80211_opmode, int,
 399                                const uint8_t [IEEE80211_ADDR_LEN],
 400                                const uint8_t [IEEE80211_ADDR_LEN]);
 401 static void     iwm_vap_delete(struct ieee80211vap *);
 402 static void     iwm_scan_start(struct ieee80211com *);
 403 static void     iwm_scan_end(struct ieee80211com *);
 404 static void     iwm_update_mcast(struct ieee80211com *);
 405 static void     iwm_set_channel(struct ieee80211com *);
 406 static void     iwm_scan_curchan(struct ieee80211_scan_state *, unsigned long);
 407 static void     iwm_scan_mindwell(struct ieee80211_scan_state *);
 408 static int      iwm_detach(device_t);
 409
 410 /*
 411  * Firmware parser.
 412  */
 413
 414 static int
 415 iwm_store_cscheme(struct iwm_softc *sc, const uint8_t *data, size_t dlen)
 416 {
 417         const struct iwm_fw_cscheme_list *l = (const void *)data;
 418
 419         if (dlen < sizeof(*l) ||
 420             dlen < sizeof(l->size) + l->size * sizeof(*l->cs))
 421                 return EINVAL;
 422
 423         /* we don't actually store anything for now, always use s/w crypto */
 424
 425         return 0;
 426 }
 427
 428 static int
 429 iwm_firmware_store_section(struct iwm_softc *sc,
 430     enum iwm_ucode_type type, const uint8_t *data, size_t dlen)
 431 {
 432         struct iwm_fw_sects *fws;
 433         struct iwm_fw_desc *fwone;
 434
 435         if (type >= IWM_UCODE_TYPE_MAX)
 436                 return EINVAL;
 437         if (dlen < sizeof(uint32_t))
 438                 return EINVAL;
 439
 440         fws = &sc->sc_fw.fw_sects[type];
 441         if (fws->fw_count >= IWM_UCODE_SECTION_MAX)
 442                 return EINVAL;
 443
 444         fwone = &fws->fw_sect[fws->fw_count];
 445
 446         /* first 32bit are device load offset */
 447         memcpy(&fwone->offset, data, sizeof(uint32_t));
 448
 449         /* rest is data */
 450         fwone->data = data + sizeof(uint32_t);
 451         fwone->len = dlen - sizeof(uint32_t);
 452
 453         fws->fw_count++;
 454
 455         return 0;
 456 }
 457
 458 #define IWM_DEFAULT_SCAN_CHANNELS 40
 459
 460 /* iwlwifi: iwl-drv.c */
 461 struct iwm_tlv_calib_data {
 462         uint32_t ucode_type;
 463         struct iwm_tlv_calib_ctrl calib;
 464 } __packed;
 465
 466 static int
 467 iwm_set_default_calib(struct iwm_softc *sc, const void *data)
 468 {
 469         const struct iwm_tlv_calib_data *def_calib = data;
 470         uint32_t ucode_type = le32toh(def_calib->ucode_type);
 471
 472         if (ucode_type >= IWM_UCODE_TYPE_MAX) {
 473                 device_printf(sc->sc_dev,
 474                     "Wrong ucode_type %u for default "
 475                     "calibration.\n", ucode_type);
 476                 return EINVAL;
 477         }
 478
 479         sc->sc_default_calib[ucode_type].flow_trigger =
 480             def_calib->calib.flow_trigger;
 481         sc->sc_default_calib[ucode_type].event_trigger =
 482             def_calib->calib.event_trigger;
 483
 484         return 0;
 485 }
 486
 487 static int
 488 iwm_set_ucode_api_flags(struct iwm_softc *sc, const uint8_t *data,
 489                         struct iwm_ucode_capabilities *capa)
 490 {
 491         const struct iwm_ucode_api *ucode_api = (const void *)data;
 492         uint32_t api_index = le32toh(ucode_api->api_index);
 493         uint32_t api_flags = le32toh(ucode_api->api_flags);
 494         int i;
 495
 496         if (api_index >= howmany(IWM_NUM_UCODE_TLV_API, 32)) {
 497                 device_printf(sc->sc_dev,
 498                     "api flags index %d larger than supported by driver\n",
 499                     api_index);
 500                 /* don't return an error so we can load FW that has more bits */
 501                 return 0;
 502         }
 503
 504         for (i = 0; i < 32; i++) {
 505                 if (api_flags & (1U << i))
 506                         setbit(capa->enabled_api, i + 32 * api_index);
 507         }
 508
 509         return 0;
 510 }
 511
 512 static int
 513 iwm_set_ucode_capabilities(struct iwm_softc *sc, const uint8_t *data,
 514                            struct iwm_ucode_capabilities *capa)
 515 {
 516         const struct iwm_ucode_capa *ucode_capa = (const void *)data;
 517         uint32_t api_index = le32toh(ucode_capa->api_index);
 518         uint32_t api_flags = le32toh(ucode_capa->api_capa);
 519         int i;
 520
 521         if (api_index >= howmany(IWM_NUM_UCODE_TLV_CAPA, 32)) {
 522                 device_printf(sc->sc_dev,
 523                     "capa flags index %d larger than supported by driver\n",
 524                     api_index);
 525                 /* don't return an error so we can load FW that has more bits */
 526                 return 0;
 527         }
 528
 529         for (i = 0; i < 32; i++) {
 530                 if (api_flags & (1U << i))
 531                         setbit(capa->enabled_capa, i + 32 * api_index);
 532         }
 533
 534         return 0;
 535 }
 536
 537 static void
 538 iwm_fw_info_free(struct iwm_fw_info *fw)
 539 {
 540         firmware_put(fw->fw_fp, FIRMWARE_UNLOAD);
 541         fw->fw_fp = NULL;
 542         /* don't touch fw->fw_status */
 543         memset(fw->fw_sects, 0, sizeof(fw->fw_sects));
 544 }
 545
 546 static int
 547 iwm_read_firmware(struct iwm_softc *sc, enum iwm_ucode_type ucode_type)
 548 {
 549         struct iwm_fw_info *fw = &sc->sc_fw;
 550         const struct iwm_tlv_ucode_header *uhdr;
 551         struct iwm_ucode_tlv tlv;
 552         struct iwm_ucode_capabilities *capa = &sc->ucode_capa;
 553         enum iwm_ucode_tlv_type tlv_type;
 554         const struct firmware *fwp;
 555         const uint8_t *data;
 556         uint32_t usniffer_img;
 557         uint32_t paging_mem_size;
 558         int num_of_cpus;
 559         int error = 0;
 560         size_t len;
 561
 562         if (fw->fw_status == IWM_FW_STATUS_DONE &&
 563             ucode_type != IWM_UCODE_INIT)
 564                 return 0;
 565
 566         while (fw->fw_status == IWM_FW_STATUS_INPROGRESS)
 567                 msleep(&sc->sc_fw, &sc->sc_mtx, 0, "iwmfwp", 0);
 568         fw->fw_status = IWM_FW_STATUS_INPROGRESS;
 569
 570         if (fw->fw_fp != NULL)
 571                 iwm_fw_info_free(fw);
 572
 573         /*
 574          * Load firmware into driver memory.
 575          * fw_fp will be set.
 576          */
 577         IWM_UNLOCK(sc);
 578         fwp = firmware_get(sc->cfg->fw_name);
 579         IWM_LOCK(sc);
 580         if (fwp == NULL) {
 581                 device_printf(sc->sc_dev,
 582                     "could not read firmware %s (error %d)\n",
 583                     sc->cfg->fw_name, error);
 584                 goto out;
 585         }
 586         fw->fw_fp = fwp;
 587
 588         /* (Re-)Initialize default values. */
 589         capa->flags = 0;
 590         capa->max_probe_length = IWM_DEFAULT_MAX_PROBE_LENGTH;
 591         capa->n_scan_channels = IWM_DEFAULT_SCAN_CHANNELS;
 592         memset(capa->enabled_capa, 0, sizeof(capa->enabled_capa));
 593         memset(capa->enabled_api, 0, sizeof(capa->enabled_api));
 594         memset(sc->sc_fw_mcc, 0, sizeof(sc->sc_fw_mcc));
 595
 596         /*
 597          * Parse firmware contents
 598          */
 599
 600         uhdr = (const void *)fw->fw_fp->data;
 601         if (*(const uint32_t *)fw->fw_fp->data != 0
 602             || le32toh(uhdr->magic) != IWM_TLV_UCODE_MAGIC) {
 603                 device_printf(sc->sc_dev, "invalid firmware %s\n",
 604                     sc->cfg->fw_name);
 605                 error = EINVAL;
 606                 goto out;
 607         }
 608
 609         snprintf(sc->sc_fwver, sizeof(sc->sc_fwver), "%d.%d (API ver %d)",
 610             IWM_UCODE_MAJOR(le32toh(uhdr->ver)),
 611             IWM_UCODE_MINOR(le32toh(uhdr->ver)),
 612             IWM_UCODE_API(le32toh(uhdr->ver)));
 613         data = uhdr->data;
 614         len = fw->fw_fp->datasize - sizeof(*uhdr);
 615
 616         while (len >= sizeof(tlv)) {
 617                 size_t tlv_len;
 618                 const void *tlv_data;
 619
 620                 memcpy(&tlv, data, sizeof(tlv));
 621                 tlv_len = le32toh(tlv.length);
 622                 tlv_type = le32toh(tlv.type);
 623
 624                 len -= sizeof(tlv);
 625                 data += sizeof(tlv);
 626                 tlv_data = data;
 627
 628                 if (len < tlv_len) {
 629                         device_printf(sc->sc_dev,
 630                             "firmware too short: %zu bytes\n",
 631                             len);
 632                         error = EINVAL;
 633                         goto parse_out;
 634                 }
 635
 636                 switch ((int)tlv_type) {
 637                 case IWM_UCODE_TLV_PROBE_MAX_LEN:
 638                         if (tlv_len < sizeof(uint32_t)) {
 639                                 device_printf(sc->sc_dev,
 640                                     "%s: PROBE_MAX_LEN (%d) < sizeof(uint32_t)\n",
 641                                     __func__,
 642                                     (int) tlv_len);
 643                                 error = EINVAL;
 644                                 goto parse_out;
 645                         }
 646                         capa->max_probe_length =
 647                             le32toh(*(const uint32_t *)tlv_data);
 648                         /* limit it to something sensible */
 649                         if (capa->max_probe_length >
 650                             IWM_SCAN_OFFLOAD_PROBE_REQ_SIZE) {
 651                                 IWM_DPRINTF(sc, IWM_DEBUG_FIRMWARE_TLV,
 652                                     "%s: IWM_UCODE_TLV_PROBE_MAX_LEN "
 653                                     "ridiculous\n", __func__);
 654                                 error = EINVAL;
 655                                 goto parse_out;
 656                         }
 657                         break;
 658                 case IWM_UCODE_TLV_PAN:
 659                         if (tlv_len) {
 660                                 device_printf(sc->sc_dev,
 661                                     "%s: IWM_UCODE_TLV_PAN: tlv_len (%d) > 0\n",
 662                                     __func__,
 663                                     (int) tlv_len);
 664                                 error = EINVAL;
 665                                 goto parse_out;
 666                         }
 667                         capa->flags |= IWM_UCODE_TLV_FLAGS_PAN;
 668                         break;
 669                 case IWM_UCODE_TLV_FLAGS:
 670                         if (tlv_len < sizeof(uint32_t)) {
 671                                 device_printf(sc->sc_dev,
 672                                     "%s: IWM_UCODE_TLV_FLAGS: tlv_len (%d) < sizeof(uint32_t)\n",
 673                                     __func__,
 674                                     (int) tlv_len);
 675                                 error = EINVAL;
 676                                 goto parse_out;
 677                         }
 678                         /*
 679                          * Apparently there can be many flags, but Linux driver
 680                          * parses only the first one, and so do we.
 681                          *
 682                          * XXX: why does this override IWM_UCODE_TLV_PAN?
 683                          * Intentional or a bug?  Observations from
 684                          * current firmware file:
 685                          *  1) TLV_PAN is parsed first
 686                          *  2) TLV_FLAGS contains TLV_FLAGS_PAN
 687                          * ==> this resets TLV_PAN to itself... hnnnk
 688                          */
 689                         capa->flags = le32toh(*(const uint32_t *)tlv_data);
 690                         break;
 691                 case IWM_UCODE_TLV_CSCHEME:
 692                         if ((error = iwm_store_cscheme(sc,
 693                             tlv_data, tlv_len)) != 0) {
 694                                 device_printf(sc->sc_dev,
 695                                     "%s: iwm_store_cscheme(): returned %d\n",
 696                                     __func__,
 697                                     error);
 698                                 goto parse_out;
 699                         }
 700                         break;
 701                 case IWM_UCODE_TLV_NUM_OF_CPU:
 702                         if (tlv_len != sizeof(uint32_t)) {
 703                                 device_printf(sc->sc_dev,
 704                                     "%s: IWM_UCODE_TLV_NUM_OF_CPU: tlv_len (%d) != sizeof(uint32_t)\n",
 705                                     __func__,
 706                                     (int) tlv_len);
 707                                 error = EINVAL;
 708                                 goto parse_out;
 709                         }
 710                         num_of_cpus = le32toh(*(const uint32_t *)tlv_data);
 711                         if (num_of_cpus == 2) {
 712                                 fw->fw_sects[IWM_UCODE_REGULAR].is_dual_cpus =
 713                                         TRUE;
 714                                 fw->fw_sects[IWM_UCODE_INIT].is_dual_cpus =
 715                                         TRUE;
 716                                 fw->fw_sects[IWM_UCODE_WOWLAN].is_dual_cpus =
 717                                         TRUE;
 718                         } else if ((num_of_cpus > 2) || (num_of_cpus < 1)) {
 719                                 device_printf(sc->sc_dev,
 720                                     "%s: Driver supports only 1 or 2 CPUs\n",
 721                                     __func__);
 722                                 error = EINVAL;
 723                                 goto parse_out;
 724                         }
 725                         break;
 726                 case IWM_UCODE_TLV_SEC_RT:
 727                         if ((error = iwm_firmware_store_section(sc,
 728                             IWM_UCODE_REGULAR, tlv_data, tlv_len)) != 0) {
 729                                 device_printf(sc->sc_dev,
 730                                     "%s: IWM_UCODE_REGULAR: iwm_firmware_store_section() failed; %d\n",
 731                                     __func__,
 732                                     error);
 733                                 goto parse_out;
 734                         }
 735                         break;
 736                 case IWM_UCODE_TLV_SEC_INIT:
 737                         if ((error = iwm_firmware_store_section(sc,
 738                             IWM_UCODE_INIT, tlv_data, tlv_len)) != 0) {
 739                                 device_printf(sc->sc_dev,
 740                                     "%s: IWM_UCODE_INIT: iwm_firmware_store_section() failed; %d\n",
 741                                     __func__,
 742                                     error);
 743                                 goto parse_out;
 744                         }
 745                         break;
 746                 case IWM_UCODE_TLV_SEC_WOWLAN:
 747                         if ((error = iwm_firmware_store_section(sc,
 748                             IWM_UCODE_WOWLAN, tlv_data, tlv_len)) != 0) {
 749                                 device_printf(sc->sc_dev,
 750                                     "%s: IWM_UCODE_WOWLAN: iwm_firmware_store_section() failed; %d\n",
 751                                     __func__,
 752                                     error);
 753                                 goto parse_out;
 754                         }
 755                         break;
 756                 case IWM_UCODE_TLV_DEF_CALIB:
 757                         if (tlv_len != sizeof(struct iwm_tlv_calib_data)) {
 758                                 device_printf(sc->sc_dev,
 759                                     "%s: IWM_UCODE_TLV_DEV_CALIB: tlv_len (%d) < sizeof(iwm_tlv_calib_data) (%d)\n",
 760                                     __func__,
 761                                     (int) tlv_len,
 762                                     (int) sizeof(struct iwm_tlv_calib_data));
 763                                 error = EINVAL;
 764                                 goto parse_out;
 765                         }
 766                         if ((error = iwm_set_default_calib(sc, tlv_data)) != 0) {
 767                                 device_printf(sc->sc_dev,
 768                                     "%s: iwm_set_default_calib() failed: %d\n",
 769                                     __func__,
 770                                     error);
 771                                 goto parse_out;
 772                         }
 773                         break;
 774                 case IWM_UCODE_TLV_PHY_SKU:
 775                         if (tlv_len != sizeof(uint32_t)) {
 776                                 error = EINVAL;
 777                                 device_printf(sc->sc_dev,
 778                                     "%s: IWM_UCODE_TLV_PHY_SKU: tlv_len (%d) < sizeof(uint32_t)\n",
 779                                     __func__,
 780                                     (int) tlv_len);
 781                                 goto parse_out;
 782                         }
 783                         sc->sc_fw.phy_config =
 784                             le32toh(*(const uint32_t *)tlv_data);
 785                         sc->sc_fw.valid_tx_ant = (sc->sc_fw.phy_config &
 786                                                   IWM_FW_PHY_CFG_TX_CHAIN) >>
 787                                                   IWM_FW_PHY_CFG_TX_CHAIN_POS;
 788                         sc->sc_fw.valid_rx_ant = (sc->sc_fw.phy_config &
 789                                                   IWM_FW_PHY_CFG_RX_CHAIN) >>
 790                                                   IWM_FW_PHY_CFG_RX_CHAIN_POS;
 791                         break;
 792
 793                 case IWM_UCODE_TLV_API_CHANGES_SET: {
 794                         if (tlv_len != sizeof(struct iwm_ucode_api)) {
 795                                 error = EINVAL;
 796                                 goto parse_out;
 797                         }
 798                         if (iwm_set_ucode_api_flags(sc, tlv_data, capa)) {
 799                                 error = EINVAL;
 800                                 goto parse_out;
 801                         }
 802                         break;
 803                 }
 804
 805                 case IWM_UCODE_TLV_ENABLED_CAPABILITIES: {
 806                         if (tlv_len != sizeof(struct iwm_ucode_capa)) {
 807                                 error = EINVAL;
 808                                 goto parse_out;
 809                         }
 810                         if (iwm_set_ucode_capabilities(sc, tlv_data, capa)) {
 811                                 error = EINVAL;
 812                                 goto parse_out;
 813                         }
 814                         break;
 815                 }
 816
 817                 case 48: /* undocumented TLV */
 818                 case IWM_UCODE_TLV_SDIO_ADMA_ADDR:
 819                 case IWM_UCODE_TLV_FW_GSCAN_CAPA:
 820                         /* ignore, not used by current driver */
 821                         break;
 822
 823                 case IWM_UCODE_TLV_SEC_RT_USNIFFER:
 824                         if ((error = iwm_firmware_store_section(sc,
 825                             IWM_UCODE_REGULAR_USNIFFER, tlv_data,
 826                             tlv_len)) != 0)
 827                                 goto parse_out;
 828                         break;
 829
 830                 case IWM_UCODE_TLV_PAGING:
 831                         if (tlv_len != sizeof(uint32_t)) {
 832                                 error = EINVAL;
 833                                 goto parse_out;
 834                         }
 835                         paging_mem_size = le32toh(*(const uint32_t *)tlv_data);
 836
 837                         IWM_DPRINTF(sc, IWM_DEBUG_FIRMWARE_TLV,
 838                             "%s: Paging: paging enabled (size = %u bytes)\n",
 839                             __func__, paging_mem_size);
 840                         if (paging_mem_size > IWM_MAX_PAGING_IMAGE_SIZE) {
 841                                 device_printf(sc->sc_dev,
 842                                         "%s: Paging: driver supports up to %u bytes for paging image\n",
 843                                         __func__, IWM_MAX_PAGING_IMAGE_SIZE);
 844                                 error = EINVAL;
 845                                 goto out;
 846                         }
 847                         if (paging_mem_size & (IWM_FW_PAGING_SIZE - 1)) {
 848                                 device_printf(sc->sc_dev,
 849                                     "%s: Paging: image isn't multiple %u\n",
 850                                     __func__, IWM_FW_PAGING_SIZE);
 851                                 error = EINVAL;
 852                                 goto out;
 853                         }
 854
 855                         sc->sc_fw.fw_sects[IWM_UCODE_REGULAR].paging_mem_size =
 856                             paging_mem_size;
 857                         usniffer_img = IWM_UCODE_REGULAR_USNIFFER;
 858                         sc->sc_fw.fw_sects[usniffer_img].paging_mem_size =
 859                             paging_mem_size;
 860                         break;
 861
 862                 case IWM_UCODE_TLV_N_SCAN_CHANNELS:
 863                         if (tlv_len != sizeof(uint32_t)) {
 864                                 error = EINVAL;
 865                                 goto parse_out;
 866                         }
 867                         capa->n_scan_channels =
 868                             le32toh(*(const uint32_t *)tlv_data);
 869                         break;
 870
 871                 case IWM_UCODE_TLV_FW_VERSION:
 872                         if (tlv_len != sizeof(uint32_t) * 3) {
 873                                 error = EINVAL;
 874                                 goto parse_out;
 875                         }
 876                         snprintf(sc->sc_fwver, sizeof(sc->sc_fwver),
 877                             "%d.%d.%d",
 878                             le32toh(((const uint32_t *)tlv_data)[0]),
 879                             le32toh(((const uint32_t *)tlv_data)[1]),
 880                             le32toh(((const uint32_t *)tlv_data)[2]));
 881                         break;
 882
 883                 case IWM_UCODE_TLV_FW_MEM_SEG:
 884                         break;
 885
 886                 default:
 887                         device_printf(sc->sc_dev,
 888                             "%s: unknown firmware section %d, abort\n",
 889                             __func__, tlv_type);
 890                         error = EINVAL;
 891                         goto parse_out;
 892                 }
 893
 894                 len -= roundup(tlv_len, 4);
 895                 data += roundup(tlv_len, 4);
 896         }
 897
 898         KASSERT(error == 0, ("unhandled error"));
 899
 900  parse_out:
 901         if (error) {
 902                 device_printf(sc->sc_dev, "firmware parse error %d, "
 903                     "section type %d\n", error, tlv_type);
 904         }
 905
 906  out:
 907         if (error) {
 908                 fw->fw_status = IWM_FW_STATUS_NONE;
 909                 if (fw->fw_fp != NULL)
 910                         iwm_fw_info_free(fw);
 911         } else
 912                 fw->fw_status = IWM_FW_STATUS_DONE;
 913         wakeup(&sc->sc_fw);
 914
 915         return error;
 916 }
 917
 918 /*
 919  * DMA resource routines
 920  */
 921
 922 /* fwmem is used to load firmware onto the card */
 923 static int
 924 iwm_alloc_fwmem(struct iwm_softc *sc)
 925 {
 926         /* Must be aligned on a 16-byte boundary. */
 927         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->fw_dma,
 928             IWM_FH_MEM_TB_MAX_LENGTH, 16);
 929 }
 930
 931 /* tx scheduler rings.  not used? */
 932 static int
 933 iwm_alloc_sched(struct iwm_softc *sc)
 934 {
 935         /* TX scheduler rings must be aligned on a 1KB boundary. */
 936         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->sched_dma,
 937             nitems(sc->txq) * sizeof(struct iwm_agn_scd_bc_tbl), 1024);
 938 }
 939
 940 /* keep-warm page is used internally by the card.  see iwl-fh.h for more info */
 941 static int
 942 iwm_alloc_kw(struct iwm_softc *sc)
 943 {
 944         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->kw_dma, 4096, 4096);
 945 }
 946
 947 /* interrupt cause table */
 948 static int
 949 iwm_alloc_ict(struct iwm_softc *sc)
 950 {
 951         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->ict_dma,
 952             IWM_ICT_SIZE, 1<<IWM_ICT_PADDR_SHIFT);
 953 }
 954
 955 static int
 956 iwm_alloc_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring)
 957 {
 958         bus_size_t size;
 959         int i, error;
 960
 961         ring->cur = 0;
 962
 963         /* Allocate RX descriptors (256-byte aligned). */
 964         size = IWM_RX_RING_COUNT * sizeof(uint32_t);
 965         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->desc_dma, size, 256);
 966         if (error != 0) {
 967                 device_printf(sc->sc_dev,
 968                     "could not allocate RX ring DMA memory\n");
 969                 goto fail;
 970         }
 971         ring->desc = ring->desc_dma.vaddr;
 972
 973         /* Allocate RX status area (16-byte aligned). */
 974         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->stat_dma,
 975             sizeof(*ring->stat), 16);
 976         if (error != 0) {
 977                 device_printf(sc->sc_dev,
 978                     "could not allocate RX status DMA memory\n");
 979                 goto fail;
 980         }
 981         ring->stat = ring->stat_dma.vaddr;
 982
 983         /* Create RX buffer DMA tag. */
 984         error = bus_dma_tag_create(sc->sc_dmat, 1, 0,
 985             BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
 986             IWM_RBUF_SIZE, 1, IWM_RBUF_SIZE, 0, NULL, NULL, &ring->data_dmat);
 987         if (error != 0) {
 988                 device_printf(sc->sc_dev,
 989                     "%s: could not create RX buf DMA tag, error %d\n",
 990                     __func__, error);
 991                 goto fail;
 992         }
 993
 994         /* Allocate spare bus_dmamap_t for iwm_rx_addbuf() */
 995         error = bus_dmamap_create(ring->data_dmat, 0, &ring->spare_map);
 996         if (error != 0) {
 997                 device_printf(sc->sc_dev,
 998                     "%s: could not create RX buf DMA map, error %d\n",
 999                     __func__, error);
1000                 goto fail;
1001         }
1002         /*
1003          * Allocate and map RX buffers.
1004          */
1005         for (i = 0; i < IWM_RX_RING_COUNT; i++) {
1006                 struct iwm_rx_data *data = &ring->data[i];
1007                 error = bus_dmamap_create(ring->data_dmat, 0, &data->map);
1008                 if (error != 0) {
1009                         device_printf(sc->sc_dev,
1010                             "%s: could not create RX buf DMA map, error %d\n",
1011                             __func__, error);
1012                         goto fail;
1013                 }
1014                 data->m = NULL;
1015
1016                 if ((error = iwm_rx_addbuf(sc, IWM_RBUF_SIZE, i)) != 0) {
1017                         goto fail;
1018                 }
1019         }
1020         return 0;
1021
1022 fail:   iwm_free_rx_ring(sc, ring);
1023         return error;
1024 }
1025
1026 static void
1027 iwm_reset_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring)
1028 {
1029         /* Reset the ring state */
1030         ring->cur = 0;
1031
1032         /*
1033          * The hw rx ring index in shared memory must also be cleared,
1034          * otherwise the discrepancy can cause reprocessing chaos.
1035          */
1036         memset(sc->rxq.stat, 0, sizeof(*sc->rxq.stat));
1037 }
1038
1039 static void
1040 iwm_free_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring)
1041 {
1042         int i;
1043
1044         iwm_dma_contig_free(&ring->desc_dma);
1045         iwm_dma_contig_free(&ring->stat_dma);
1046
1047         for (i = 0; i < IWM_RX_RING_COUNT; i++) {
1048                 struct iwm_rx_data *data = &ring->data[i];
1049
1050                 if (data->m != NULL) {
1051                         bus_dmamap_sync(ring->data_dmat, data->map,
1052                             BUS_DMASYNC_POSTREAD);
1053                         bus_dmamap_unload(ring->data_dmat, data->map);
1054                         m_freem(data->m);
1055                         data->m = NULL;
1056                 }
1057                 if (data->map != NULL) {
1058                         bus_dmamap_destroy(ring->data_dmat, data->map);
1059                         data->map = NULL;
1060                 }
1061         }
1062         if (ring->spare_map != NULL) {
1063                 bus_dmamap_destroy(ring->data_dmat, ring->spare_map);
1064                 ring->spare_map = NULL;
1065         }
1066         if (ring->data_dmat != NULL) {
1067                 bus_dma_tag_destroy(ring->data_dmat);
1068                 ring->data_dmat = NULL;
1069         }
1070 }
1071
1072 static int
1073 iwm_alloc_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring, int qid)
1074 {
1075         bus_addr_t paddr;
1076         bus_size_t size;
1077         size_t maxsize;
1078         int nsegments;
1079         int i, error;
1080
1081         ring->qid = qid;
1082         ring->queued = 0;
1083         ring->cur = 0;
1084
1085         /* Allocate TX descriptors (256-byte aligned). */
1086         size = IWM_TX_RING_COUNT * sizeof (struct iwm_tfd);
1087         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->desc_dma, size, 256);
1088         if (error != 0) {
1089                 device_printf(sc->sc_dev,
1090                     "could not allocate TX ring DMA memory\n");
1091                 goto fail;
1092         }
1093         ring->desc = ring->desc_dma.vaddr;
1094
1095         /*
1096          * We only use rings 0 through 9 (4 EDCA + cmd) so there is no need
1097          * to allocate commands space for other rings.
1098          */
1099         if (qid > IWM_MVM_CMD_QUEUE)
1100                 return 0;
1101
1102         size = IWM_TX_RING_COUNT * sizeof(struct iwm_device_cmd);
1103         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->cmd_dma, size, 4);
1104         if (error != 0) {
1105                 device_printf(sc->sc_dev,
1106                     "could not allocate TX cmd DMA memory\n");
1107                 goto fail;
1108         }
1109         ring->cmd = ring->cmd_dma.vaddr;
1110
1111         /* FW commands may require more mapped space than packets. */
1112         if (qid == IWM_MVM_CMD_QUEUE) {
1113                 maxsize = IWM_RBUF_SIZE;
1114                 nsegments = 1;
1115         } else {
1116                 maxsize = MCLBYTES;
1117                 nsegments = IWM_MAX_SCATTER - 2;
1118         }
1119
1120         error = bus_dma_tag_create(sc->sc_dmat, 1, 0,
1121             BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, maxsize,
1122             nsegments, maxsize, 0, NULL, NULL, &ring->data_dmat);
1123         if (error != 0) {
1124                 device_printf(sc->sc_dev, "could not create TX buf DMA tag\n");
1125                 goto fail;
1126         }
1127
1128         paddr = ring->cmd_dma.paddr;
1129         for (i = 0; i < IWM_TX_RING_COUNT; i++) {
1130                 struct iwm_tx_data *data = &ring->data[i];
1131
1132                 data->cmd_paddr = paddr;
1133                 data->scratch_paddr = paddr + sizeof(struct iwm_cmd_header)
1134                     + offsetof(struct iwm_tx_cmd, scratch);
1135                 paddr += sizeof(struct iwm_device_cmd);
1136
1137                 error = bus_dmamap_create(ring->data_dmat, 0, &data->map);
1138                 if (error != 0) {
1139                         device_printf(sc->sc_dev,
1140                             "could not create TX buf DMA map\n");
1141                         goto fail;
1142                 }
1143         }
1144         KASSERT(paddr == ring->cmd_dma.paddr + size,
1145             ("invalid physical address"));
1146         return 0;
1147
1148 fail:   iwm_free_tx_ring(sc, ring);
1149         return error;
1150 }
1151
1152 static void
1153 iwm_reset_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring)
1154 {
1155         int i;
1156
1157         for (i = 0; i < IWM_TX_RING_COUNT; i++) {
1158                 struct iwm_tx_data *data = &ring->data[i];
1159
1160                 if (data->m != NULL) {
1161                         bus_dmamap_sync(ring->data_dmat, data->map,
1162                             BUS_DMASYNC_POSTWRITE);
1163                         bus_dmamap_unload(ring->data_dmat, data->map);
1164                         m_freem(data->m);
1165                         data->m = NULL;
1166                 }
1167         }
1168         /* Clear TX descriptors. */
1169         memset(ring->desc, 0, ring->desc_dma.size);
1170         bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map,
1171             BUS_DMASYNC_PREWRITE);
1172         sc->qfullmsk &= ~(1 << ring->qid);
1173         ring->queued = 0;
1174         ring->cur = 0;
1175
1176         if (ring->qid == IWM_MVM_CMD_QUEUE && sc->cmd_hold_nic_awake)
1177                 iwm_pcie_clear_cmd_in_flight(sc);
1178 }
1179
1180 static void
1181 iwm_free_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring)
1182 {
1183         int i;
1184
1185         iwm_dma_contig_free(&ring->desc_dma);
1186         iwm_dma_contig_free(&ring->cmd_dma);
1187
1188         for (i = 0; i < IWM_TX_RING_COUNT; i++) {
1189                 struct iwm_tx_data *data = &ring->data[i];
1190
1191                 if (data->m != NULL) {
1192                         bus_dmamap_sync(ring->data_dmat, data->map,
1193                             BUS_DMASYNC_POSTWRITE);
1194                         bus_dmamap_unload(ring->data_dmat, data->map);
1195                         m_freem(data->m);
1196                         data->m = NULL;
1197                 }
1198                 if (data->map != NULL) {
1199                         bus_dmamap_destroy(ring->data_dmat, data->map);
1200                         data->map = NULL;
1201                 }
1202         }
1203         if (ring->data_dmat != NULL) {
1204                 bus_dma_tag_destroy(ring->data_dmat);
1205                 ring->data_dmat = NULL;
1206         }
1207 }
1208
1209 /*
1210  * High-level hardware frobbing routines
1211  */
1212
1213 static void
1214 iwm_enable_interrupts(struct iwm_softc *sc)
1215 {
1216         sc->sc_intmask = IWM_CSR_INI_SET_MASK;
1217         IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask);
1218 }
1219
1220 static void
1221 iwm_restore_interrupts(struct iwm_softc *sc)
1222 {
1223         IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask);
1224 }
1225
1226 static void
1227 iwm_disable_interrupts(struct iwm_softc *sc)
1228 {
1229         /* disable interrupts */
1230         IWM_WRITE(sc, IWM_CSR_INT_MASK, 0);
1231
1232         /* acknowledge all interrupts */
1233         IWM_WRITE(sc, IWM_CSR_INT, ~0);
1234         IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, ~0);
1235 }
1236
1237 static void
1238 iwm_ict_reset(struct iwm_softc *sc)
1239 {
1240         iwm_disable_interrupts(sc);
1241
1242         /* Reset ICT table. */
1243         memset(sc->ict_dma.vaddr, 0, IWM_ICT_SIZE);
1244         sc->ict_cur = 0;
1245
1246         /* Set physical address of ICT table (4KB aligned). */
1247         IWM_WRITE(sc, IWM_CSR_DRAM_INT_TBL_REG,
1248             IWM_CSR_DRAM_INT_TBL_ENABLE
1249             | IWM_CSR_DRAM_INIT_TBL_WRITE_POINTER
1250             | IWM_CSR_DRAM_INIT_TBL_WRAP_CHECK
1251             | sc->ict_dma.paddr >> IWM_ICT_PADDR_SHIFT);
1252
1253         /* Switch to ICT interrupt mode in driver. */
1254         sc->sc_flags |= IWM_FLAG_USE_ICT;
1255
1256         /* Re-enable interrupts. */
1257         IWM_WRITE(sc, IWM_CSR_INT, ~0);
1258         iwm_enable_interrupts(sc);
1259 }
1260
1261 /* iwlwifi pcie/trans.c */
1262
1263 /*
1264  * Since this .. hard-resets things, it's time to actually
1265  * mark the first vap (if any) as having no mac context.
1266  * It's annoying, but since the driver is potentially being
1267  * stop/start'ed whilst active (thanks openbsd port!) we
1268  * have to correctly track this.
1269  */
1270 static void
1271 iwm_stop_device(struct iwm_softc *sc)
1272 {
1273         struct ieee80211com *ic = &sc->sc_ic;
1274         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
1275         int chnl, qid;
1276         uint32_t mask = 0;
1277
1278         /* tell the device to stop sending interrupts */
1279         iwm_disable_interrupts(sc);
1280
1281         /*
1282          * FreeBSD-local: mark the first vap as not-uploaded,
1283          * so the next transition through auth/assoc
1284          * will correctly populate the MAC context.
1285          */
1286         if (vap) {
1287                 struct iwm_vap *iv = IWM_VAP(vap);
1288                 iv->is_uploaded = 0;
1289         }
1290
1291         /* device going down, Stop using ICT table */
1292         sc->sc_flags &= ~IWM_FLAG_USE_ICT;
1293
1294         /* stop tx and rx.  tx and rx bits, as usual, are from if_iwn */
1295
1296         iwm_write_prph(sc, IWM_SCD_TXFACT, 0);
1297
1298         if (iwm_nic_lock(sc)) {
1299                 /* Stop each Tx DMA channel */
1300                 for (chnl = 0; chnl < IWM_FH_TCSR_CHNL_NUM; chnl++) {
1301                         IWM_WRITE(sc,
1302                             IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl), 0);
1303                         mask |= IWM_FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(chnl);
1304                 }
1305
1306                 /* Wait for DMA channels to be idle */
1307                 if (!iwm_poll_bit(sc, IWM_FH_TSSR_TX_STATUS_REG, mask, mask,
1308                     5000)) {
1309                         device_printf(sc->sc_dev,
1310                             "Failing on timeout while stopping DMA channel: [0x%08x]\n",
1311                             IWM_READ(sc, IWM_FH_TSSR_TX_STATUS_REG));
1312                 }
1313                 iwm_nic_unlock(sc);
1314         }
1315         iwm_pcie_rx_stop(sc);
1316
1317         /* Stop RX ring. */
1318         iwm_reset_rx_ring(sc, &sc->rxq);
1319
1320         /* Reset all TX rings. */
1321         for (qid = 0; qid < nitems(sc->txq); qid++)
1322                 iwm_reset_tx_ring(sc, &sc->txq[qid]);
1323
1324         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
1325                 /* Power-down device's busmaster DMA clocks */
1326                 iwm_write_prph(sc, IWM_APMG_CLK_DIS_REG,
1327                     IWM_APMG_CLK_VAL_DMA_CLK_RQT);
1328                 DELAY(5);
1329         }
1330
1331         /* Make sure (redundant) we've released our request to stay awake */
1332         IWM_CLRBITS(sc, IWM_CSR_GP_CNTRL,
1333             IWM_CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1334
1335         /* Stop the device, and put it in low power state */
1336         iwm_apm_stop(sc);
1337
1338         /* Upon stop, the APM issues an interrupt if HW RF kill is set.
1339          * Clean again the interrupt here
1340          */
1341         iwm_disable_interrupts(sc);
1342         /* stop and reset the on-board processor */
1343         IWM_WRITE(sc, IWM_CSR_RESET, IWM_CSR_RESET_REG_FLAG_SW_RESET);
1344
1345         /*
1346          * Even if we stop the HW, we still want the RF kill
1347          * interrupt
1348          */
1349         iwm_enable_rfkill_int(sc);
1350         iwm_check_rfkill(sc);
1351 }
1352
1353 /* iwlwifi: mvm/ops.c */
1354 static void
1355 iwm_mvm_nic_config(struct iwm_softc *sc)
1356 {
1357         uint8_t radio_cfg_type, radio_cfg_step, radio_cfg_dash;
1358         uint32_t reg_val = 0;
1359         uint32_t phy_config = iwm_mvm_get_phy_config(sc);
1360
1361         radio_cfg_type = (phy_config & IWM_FW_PHY_CFG_RADIO_TYPE) >>
1362             IWM_FW_PHY_CFG_RADIO_TYPE_POS;
1363         radio_cfg_step = (phy_config & IWM_FW_PHY_CFG_RADIO_STEP) >>
1364             IWM_FW_PHY_CFG_RADIO_STEP_POS;
1365         radio_cfg_dash = (phy_config & IWM_FW_PHY_CFG_RADIO_DASH) >>
1366             IWM_FW_PHY_CFG_RADIO_DASH_POS;
1367
1368         /* SKU control */
1369         reg_val |= IWM_CSR_HW_REV_STEP(sc->sc_hw_rev) <<
1370             IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_STEP;
1371         reg_val |= IWM_CSR_HW_REV_DASH(sc->sc_hw_rev) <<
1372             IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_DASH;
1373
1374         /* radio configuration */
1375         reg_val |= radio_cfg_type << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_TYPE;
1376         reg_val |= radio_cfg_step << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_STEP;
1377         reg_val |= radio_cfg_dash << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_DASH;
1378
1379         IWM_WRITE(sc, IWM_CSR_HW_IF_CONFIG_REG, reg_val);
1380
1381         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
1382             "Radio type=0x%x-0x%x-0x%x\n", radio_cfg_type,
1383             radio_cfg_step, radio_cfg_dash);
1384
1385         /*
1386          * W/A : NIC is stuck in a reset state after Early PCIe power off
1387          * (PCIe power is lost before PERST# is asserted), causing ME FW
1388          * to lose ownership and not being able to obtain it back.
1389          */
1390         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
1391                 iwm_set_bits_mask_prph(sc, IWM_APMG_PS_CTRL_REG,
1392                     IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS,
1393                     ~IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS);
1394         }
1395 }
1396
1397 static int
1398 iwm_nic_rx_init(struct iwm_softc *sc)
1399 {
1400         /*
1401          * Initialize RX ring.  This is from the iwn driver.
1402          */
1403         memset(sc->rxq.stat, 0, sizeof(*sc->rxq.stat));
1404
1405         /* Stop Rx DMA */
1406         iwm_pcie_rx_stop(sc);
1407
1408         if (!iwm_nic_lock(sc))
1409                 return EBUSY;
1410
1411         /* reset and flush pointers */
1412         IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_RBDCB_WPTR, 0);
1413         IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_FLUSH_RB_REQ, 0);
1414         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_RDPTR, 0);
1415         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_RBDCB_WPTR_REG, 0);
1416
1417         /* Set physical address of RX ring (256-byte aligned). */
1418         IWM_WRITE(sc,
1419             IWM_FH_RSCSR_CHNL0_RBDCB_BASE_REG, sc->rxq.desc_dma.paddr >> 8);
1420
1421         /* Set physical address of RX status (16-byte aligned). */
1422         IWM_WRITE(sc,
1423             IWM_FH_RSCSR_CHNL0_STTS_WPTR_REG, sc->rxq.stat_dma.paddr >> 4);
1424
1425         /* Enable RX. */
1426         IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_CONFIG_REG,
1427             IWM_FH_RCSR_RX_CONFIG_CHNL_EN_ENABLE_VAL            |
1428             IWM_FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY               |  /* HW bug */
1429             IWM_FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_INT_HOST_VAL   |
1430             IWM_FH_RCSR_CHNL0_RX_CONFIG_SINGLE_FRAME_MSK        |
1431             (IWM_RX_RB_TIMEOUT << IWM_FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS) |
1432             IWM_FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K            |
1433             IWM_RX_QUEUE_SIZE_LOG << IWM_FH_RCSR_RX_CONFIG_RBDCB_SIZE_POS);
1434
1435         IWM_WRITE_1(sc, IWM_CSR_INT_COALESCING, IWM_HOST_INT_TIMEOUT_DEF);
1436
1437         /* W/A for interrupt coalescing bug in 7260 and 3160 */
1438         if (sc->cfg->host_interrupt_operation_mode)
1439                 IWM_SETBITS(sc, IWM_CSR_INT_COALESCING, IWM_HOST_INT_OPER_MODE);
1440
1441         /*
1442          * Thus sayeth el jefe (iwlwifi) via a comment:
1443          *
1444          * This value should initially be 0 (before preparing any
1445          * RBs), should be 8 after preparing the first 8 RBs (for example)
1446          */
1447         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_WPTR, 8);
1448
1449         iwm_nic_unlock(sc);
1450
1451         return 0;
1452 }
1453
1454 static int
1455 iwm_nic_tx_init(struct iwm_softc *sc)
1456 {
1457         int qid;
1458
1459         if (!iwm_nic_lock(sc))
1460                 return EBUSY;
1461
1462         /* Deactivate TX scheduler. */
1463         iwm_write_prph(sc, IWM_SCD_TXFACT, 0);
1464
1465         /* Set physical address of "keep warm" page (16-byte aligned). */
1466         IWM_WRITE(sc, IWM_FH_KW_MEM_ADDR_REG, sc->kw_dma.paddr >> 4);
1467
1468         /* Initialize TX rings. */
1469         for (qid = 0; qid < nitems(sc->txq); qid++) {
1470                 struct iwm_tx_ring *txq = &sc->txq[qid];
1471
1472                 /* Set physical address of TX ring (256-byte aligned). */
1473                 IWM_WRITE(sc, IWM_FH_MEM_CBBC_QUEUE(qid),
1474                     txq->desc_dma.paddr >> 8);
1475                 IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
1476                     "%s: loading ring %d descriptors (%p) at %lx\n",
1477                     __func__,
1478                     qid, txq->desc,
1479                     (unsigned long) (txq->desc_dma.paddr >> 8));
1480         }
1481
1482         iwm_write_prph(sc, IWM_SCD_GP_CTRL, IWM_SCD_GP_CTRL_AUTO_ACTIVE_MODE);
1483
1484         iwm_nic_unlock(sc);
1485
1486         return 0;
1487 }
1488
1489 static int
1490 iwm_nic_init(struct iwm_softc *sc)
1491 {
1492         int error;
1493
1494         iwm_apm_init(sc);
1495         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
1496                 iwm_set_pwr(sc);
1497
1498         iwm_mvm_nic_config(sc);
1499
1500         if ((error = iwm_nic_rx_init(sc)) != 0)
1501                 return error;
1502
1503         /*
1504          * Ditto for TX, from iwn
1505          */
1506         if ((error = iwm_nic_tx_init(sc)) != 0)
1507                 return error;
1508
1509         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
1510             "%s: shadow registers enabled\n", __func__);
1511         IWM_SETBITS(sc, IWM_CSR_MAC_SHADOW_REG_CTRL, 0x800fffff);
1512
1513         return 0;
1514 }
1515
1516 const uint8_t iwm_mvm_ac_to_tx_fifo[] = {
1517         IWM_MVM_TX_FIFO_VO,
1518         IWM_MVM_TX_FIFO_VI,
1519         IWM_MVM_TX_FIFO_BE,
1520         IWM_MVM_TX_FIFO_BK,
1521 };
1522
1523 static int
1524 iwm_enable_txq(struct iwm_softc *sc, int sta_id, int qid, int fifo)
1525 {
1526         if (!iwm_nic_lock(sc)) {
1527                 device_printf(sc->sc_dev,
1528                     "%s: cannot enable txq %d\n",
1529                     __func__,
1530                     qid);
1531                 return EBUSY;
1532         }
1533
1534         IWM_WRITE(sc, IWM_HBUS_TARG_WRPTR, qid << 8 | 0);
1535
1536         if (qid == IWM_MVM_CMD_QUEUE) {
1537                 /* unactivate before configuration */
1538                 iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid),
1539                     (0 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE)
1540                     | (1 << IWM_SCD_QUEUE_STTS_REG_POS_SCD_ACT_EN));
1541
1542                 iwm_nic_unlock(sc);
1543
1544                 iwm_clear_bits_prph(sc, IWM_SCD_AGGR_SEL, (1 << qid));
1545
1546                 if (!iwm_nic_lock(sc)) {
1547                         device_printf(sc->sc_dev,
1548                             "%s: cannot enable txq %d\n", __func__, qid);
1549                         return EBUSY;
1550                 }
1551                 iwm_write_prph(sc, IWM_SCD_QUEUE_RDPTR(qid), 0);
1552                 iwm_nic_unlock(sc);
1553
1554                 iwm_write_mem32(sc, sc->scd_base_addr + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid), 0);
1555                 /* Set scheduler window size and frame limit. */
1556                 iwm_write_mem32(sc,
1557                     sc->scd_base_addr + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid) +
1558                     sizeof(uint32_t),
1559                     ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_POS) &
1560                     IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_MSK) |
1561                     ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_POS) &
1562                     IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_MSK));
1563
1564                 if (!iwm_nic_lock(sc)) {
1565                         device_printf(sc->sc_dev,
1566                             "%s: cannot enable txq %d\n", __func__, qid);
1567                         return EBUSY;
1568                 }
1569                 iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid),
1570                     (1 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE) |
1571                     (fifo << IWM_SCD_QUEUE_STTS_REG_POS_TXF) |
1572                     (1 << IWM_SCD_QUEUE_STTS_REG_POS_WSL) |
1573                     IWM_SCD_QUEUE_STTS_REG_MSK);
1574         } else {
1575                 struct iwm_scd_txq_cfg_cmd cmd;
1576                 int error;
1577
1578                 iwm_nic_unlock(sc);
1579
1580                 memset(&cmd, 0, sizeof(cmd));
1581                 cmd.scd_queue = qid;
1582                 cmd.enable = 1;
1583                 cmd.sta_id = sta_id;
1584                 cmd.tx_fifo = fifo;
1585                 cmd.aggregate = 0;
1586                 cmd.window = IWM_FRAME_LIMIT;
1587
1588                 error = iwm_mvm_send_cmd_pdu(sc, IWM_SCD_QUEUE_CFG, IWM_CMD_SYNC,
1589                     sizeof(cmd), &cmd);
1590                 if (error) {
1591                         device_printf(sc->sc_dev,
1592                             "cannot enable txq %d\n", qid);
1593                         return error;
1594                 }
1595
1596                 if (!iwm_nic_lock(sc))
1597                         return EBUSY;
1598         }
1599
1600         iwm_write_prph(sc, IWM_SCD_EN_CTRL,
1601             iwm_read_prph(sc, IWM_SCD_EN_CTRL) | qid);
1602
1603         iwm_nic_unlock(sc);
1604
1605         IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "%s: enabled txq %d FIFO %d\n",
1606             __func__, qid, fifo);
1607
1608         return 0;
1609 }
1610
1611 static int
1612 iwm_trans_pcie_fw_alive(struct iwm_softc *sc, uint32_t scd_base_addr)
1613 {
1614         int error, chnl;
1615
1616         int clear_dwords = (IWM_SCD_TRANS_TBL_MEM_UPPER_BOUND -
1617             IWM_SCD_CONTEXT_MEM_LOWER_BOUND) / sizeof(uint32_t);
1618
1619         if (!iwm_nic_lock(sc))
1620                 return EBUSY;
1621
1622         iwm_ict_reset(sc);
1623
1624         iwm_nic_unlock(sc);
1625
1626         sc->scd_base_addr = iwm_read_prph(sc, IWM_SCD_SRAM_BASE_ADDR);
1627         if (scd_base_addr != 0 &&
1628             scd_base_addr != sc->scd_base_addr) {
1629                 device_printf(sc->sc_dev,
1630                     "%s: sched addr mismatch: alive: 0x%x prph: 0x%x\n",
1631                     __func__, sc->scd_base_addr, scd_base_addr);
1632         }
1633
1634         /* reset context data, TX status and translation data */
1635         error = iwm_write_mem(sc,
1636             sc->scd_base_addr + IWM_SCD_CONTEXT_MEM_LOWER_BOUND,
1637             NULL, clear_dwords);
1638         if (error)
1639                 return EBUSY;
1640
1641         if (!iwm_nic_lock(sc))
1642                 return EBUSY;
1643
1644         /* Set physical address of TX scheduler rings (1KB aligned). */
1645         iwm_write_prph(sc, IWM_SCD_DRAM_BASE_ADDR, sc->sched_dma.paddr >> 10);
1646
1647         iwm_write_prph(sc, IWM_SCD_CHAINEXT_EN, 0);
1648
1649         iwm_nic_unlock(sc);
1650
1651         /* enable command channel */
1652         error = iwm_enable_txq(sc, 0 /* unused */, IWM_MVM_CMD_QUEUE, 7);
1653         if (error)
1654                 return error;
1655
1656         if (!iwm_nic_lock(sc))
1657                 return EBUSY;
1658
1659         iwm_write_prph(sc, IWM_SCD_TXFACT, 0xff);
1660
1661         /* Enable DMA channels. */
1662         for (chnl = 0; chnl < IWM_FH_TCSR_CHNL_NUM; chnl++) {
1663                 IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl),
1664                     IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE |
1665                     IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE);
1666         }
1667
1668         IWM_SETBITS(sc, IWM_FH_TX_CHICKEN_BITS_REG,
1669             IWM_FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN);
1670
1671         iwm_nic_unlock(sc);
1672
1673         /* Enable L1-Active */
1674         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
1675                 iwm_clear_bits_prph(sc, IWM_APMG_PCIDEV_STT_REG,
1676                     IWM_APMG_PCIDEV_STT_VAL_L1_ACT_DIS);
1677         }
1678
1679         return error;
1680 }
1681
1682 /*
1683  * NVM read access and content parsing.  We do not support
1684  * external NVM or writing NVM.
1685  * iwlwifi/mvm/nvm.c
1686  */
1687
1688 /* Default NVM size to read */
1689 #define IWM_NVM_DEFAULT_CHUNK_SIZE      (2*1024)
1690
1691 #define IWM_NVM_WRITE_OPCODE 1
1692 #define IWM_NVM_READ_OPCODE 0
1693
1694 /* load nvm chunk response */
1695 enum {
1696         IWM_READ_NVM_CHUNK_SUCCEED = 0,
1697         IWM_READ_NVM_CHUNK_NOT_VALID_ADDRESS = 1
1698 };
1699
1700 static int
1701 iwm_nvm_read_chunk(struct iwm_softc *sc, uint16_t section,
1702         uint16_t offset, uint16_t length, uint8_t *data, uint16_t *len)
1703 {
1704         struct iwm_nvm_access_cmd nvm_access_cmd = {
1705                 .offset = htole16(offset),
1706                 .length = htole16(length),
1707                 .type = htole16(section),
1708                 .op_code = IWM_NVM_READ_OPCODE,
1709         };
1710         struct iwm_nvm_access_resp *nvm_resp;
1711         struct iwm_rx_packet *pkt;
1712         struct iwm_host_cmd cmd = {
1713                 .id = IWM_NVM_ACCESS_CMD,
1714                 .flags = IWM_CMD_WANT_SKB | IWM_CMD_SEND_IN_RFKILL,
1715                 .data = { &nvm_access_cmd, },
1716         };
1717         int ret, bytes_read, offset_read;
1718         uint8_t *resp_data;
1719
1720         cmd.len[0] = sizeof(struct iwm_nvm_access_cmd);
1721
1722         ret = iwm_send_cmd(sc, &cmd);
1723         if (ret) {
1724                 device_printf(sc->sc_dev,
1725                     "Could not send NVM_ACCESS command (error=%d)\n", ret);
1726                 return ret;
1727         }
1728
1729         pkt = cmd.resp_pkt;
1730
1731         /* Extract NVM response */
1732         nvm_resp = (void *)pkt->data;
1733         ret = le16toh(nvm_resp->status);
1734         bytes_read = le16toh(nvm_resp->length);
1735         offset_read = le16toh(nvm_resp->offset);
1736         resp_data = nvm_resp->data;
1737         if (ret) {
1738                 if ((offset != 0) &&
1739                     (ret == IWM_READ_NVM_CHUNK_NOT_VALID_ADDRESS)) {
1740                         /*
1741                          * meaning of NOT_VALID_ADDRESS:
1742                          * driver try to read chunk from address that is
1743                          * multiple of 2K and got an error since addr is empty.
1744                          * meaning of (offset != 0): driver already
1745                          * read valid data from another chunk so this case
1746                          * is not an error.
1747                          */
1748                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1749                                     "NVM access command failed on offset 0x%x since that section size is multiple 2K\n",
1750                                     offset);
1751                         *len = 0;
1752                         ret = 0;
1753                 } else {
1754                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1755                                     "NVM access command failed with status %d\n", ret);
1756                         ret = EIO;
1757                 }
1758                 goto exit;
1759         }
1760
1761         if (offset_read != offset) {
1762                 device_printf(sc->sc_dev,
1763                     "NVM ACCESS response with invalid offset %d\n",
1764                     offset_read);
1765                 ret = EINVAL;
1766                 goto exit;
1767         }
1768
1769         if (bytes_read > length) {
1770                 device_printf(sc->sc_dev,
1771                     "NVM ACCESS response with too much data "
1772                     "(%d bytes requested, %d bytes received)\n",
1773                     length, bytes_read);
1774                 ret = EINVAL;
1775                 goto exit;
1776         }
1777
1778         /* Write data to NVM */
1779         memcpy(data + offset, resp_data, bytes_read);
1780         *len = bytes_read;
1781
1782  exit:
1783         iwm_free_resp(sc, &cmd);
1784         return ret;
1785 }
1786
1787 /*
1788  * Reads an NVM section completely.
1789  * NICs prior to 7000 family don't have a real NVM, but just read
1790  * section 0 which is the EEPROM. Because the EEPROM reading is unlimited
1791  * by uCode, we need to manually check in this case that we don't
1792  * overflow and try to read more than the EEPROM size.
1793  * For 7000 family NICs, we supply the maximal size we can read, and
1794  * the uCode fills the response with as much data as we can,
1795  * without overflowing, so no check is needed.
1796  */
1797 static int
1798 iwm_nvm_read_section(struct iwm_softc *sc,
1799         uint16_t section, uint8_t *data, uint16_t *len, uint32_t size_read)
1800 {
1801         uint16_t seglen, length, offset = 0;
1802         int ret;
1803
1804         /* Set nvm section read length */
1805         length = IWM_NVM_DEFAULT_CHUNK_SIZE;
1806
1807         seglen = length;
1808
1809         /* Read the NVM until exhausted (reading less than requested) */
1810         while (seglen == length) {
1811                 /* Check no memory assumptions fail and cause an overflow */
1812                 if ((size_read + offset + length) >
1813                     sc->cfg->eeprom_size) {
1814                         device_printf(sc->sc_dev,
1815                             "EEPROM size is too small for NVM\n");
1816                         return ENOBUFS;
1817                 }
1818
1819                 ret = iwm_nvm_read_chunk(sc, section, offset, length, data, &seglen);
1820                 if (ret) {
1821                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1822                                     "Cannot read NVM from section %d offset %d, length %d\n",
1823                                     section, offset, length);
1824                         return ret;
1825                 }
1826                 offset += seglen;
1827         }
1828
1829         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1830                     "NVM section %d read completed\n", section);
1831         *len = offset;
1832         return 0;
1833 }
1834
1835 /*
1836  * BEGIN IWM_NVM_PARSE
1837  */
1838
1839 /* iwlwifi/iwl-nvm-parse.c */
1840
1841 /* NVM offsets (in words) definitions */
1842 enum iwm_nvm_offsets {
1843         /* NVM HW-Section offset (in words) definitions */
1844         IWM_HW_ADDR = 0x15,
1845
1846 /* NVM SW-Section offset (in words) definitions */
1847         IWM_NVM_SW_SECTION = 0x1C0,
1848         IWM_NVM_VERSION = 0,
1849         IWM_RADIO_CFG = 1,
1850         IWM_SKU = 2,
1851         IWM_N_HW_ADDRS = 3,
1852         IWM_NVM_CHANNELS = 0x1E0 - IWM_NVM_SW_SECTION,
1853
1854 /* NVM calibration section offset (in words) definitions */
1855         IWM_NVM_CALIB_SECTION = 0x2B8,
1856         IWM_XTAL_CALIB = 0x316 - IWM_NVM_CALIB_SECTION
1857 };
1858
1859 enum iwm_8000_nvm_offsets {
1860         /* NVM HW-Section offset (in words) definitions */
1861         IWM_HW_ADDR0_WFPM_8000 = 0x12,
1862         IWM_HW_ADDR1_WFPM_8000 = 0x16,
1863         IWM_HW_ADDR0_PCIE_8000 = 0x8A,
1864         IWM_HW_ADDR1_PCIE_8000 = 0x8E,
1865         IWM_MAC_ADDRESS_OVERRIDE_8000 = 1,
1866
1867         /* NVM SW-Section offset (in words) definitions */
1868         IWM_NVM_SW_SECTION_8000 = 0x1C0,
1869         IWM_NVM_VERSION_8000 = 0,
1870         IWM_RADIO_CFG_8000 = 0,
1871         IWM_SKU_8000 = 2,
1872         IWM_N_HW_ADDRS_8000 = 3,
1873
1874         /* NVM REGULATORY -Section offset (in words) definitions */
1875         IWM_NVM_CHANNELS_8000 = 0,
1876         IWM_NVM_LAR_OFFSET_8000_OLD = 0x4C7,
1877         IWM_NVM_LAR_OFFSET_8000 = 0x507,
1878         IWM_NVM_LAR_ENABLED_8000 = 0x7,
1879
1880         /* NVM calibration section offset (in words) definitions */
1881         IWM_NVM_CALIB_SECTION_8000 = 0x2B8,
1882         IWM_XTAL_CALIB_8000 = 0x316 - IWM_NVM_CALIB_SECTION_8000
1883 };
1884
1885 /* SKU Capabilities (actual values from NVM definition) */
1886 enum nvm_sku_bits {
1887         IWM_NVM_SKU_CAP_BAND_24GHZ      = (1 << 0),
1888         IWM_NVM_SKU_CAP_BAND_52GHZ      = (1 << 1),
1889         IWM_NVM_SKU_CAP_11N_ENABLE      = (1 << 2),
1890         IWM_NVM_SKU_CAP_11AC_ENABLE     = (1 << 3),
1891 };
1892
1893 /* radio config bits (actual values from NVM definition) */
1894 #define IWM_NVM_RF_CFG_DASH_MSK(x)   (x & 0x3)         /* bits 0-1   */
1895 #define IWM_NVM_RF_CFG_STEP_MSK(x)   ((x >> 2)  & 0x3) /* bits 2-3   */
1896 #define IWM_NVM_RF_CFG_TYPE_MSK(x)   ((x >> 4)  & 0x3) /* bits 4-5   */
1897 #define IWM_NVM_RF_CFG_PNUM_MSK(x)   ((x >> 6)  & 0x3) /* bits 6-7   */
1898 #define IWM_NVM_RF_CFG_TX_ANT_MSK(x) ((x >> 8)  & 0xF) /* bits 8-11  */
1899 #define IWM_NVM_RF_CFG_RX_ANT_MSK(x) ((x >> 12) & 0xF) /* bits 12-15 */
1900
1901 #define IWM_NVM_RF_CFG_FLAVOR_MSK_8000(x)       (x & 0xF)
1902 #define IWM_NVM_RF_CFG_DASH_MSK_8000(x)         ((x >> 4) & 0xF)
1903 #define IWM_NVM_RF_CFG_STEP_MSK_8000(x)         ((x >> 8) & 0xF)
1904 #define IWM_NVM_RF_CFG_TYPE_MSK_8000(x)         ((x >> 12) & 0xFFF)
1905 #define IWM_NVM_RF_CFG_TX_ANT_MSK_8000(x)       ((x >> 24) & 0xF)
1906 #define IWM_NVM_RF_CFG_RX_ANT_MSK_8000(x)       ((x >> 28) & 0xF)
1907
1908 #define DEFAULT_MAX_TX_POWER 16
1909
1910 /**
1911  * enum iwm_nvm_channel_flags - channel flags in NVM
1912  * @IWM_NVM_CHANNEL_VALID: channel is usable for this SKU/geo
1913  * @IWM_NVM_CHANNEL_IBSS: usable as an IBSS channel
1914  * @IWM_NVM_CHANNEL_ACTIVE: active scanning allowed
1915  * @IWM_NVM_CHANNEL_RADAR: radar detection required
1916  * XXX cannot find this (DFS) flag in iwm-nvm-parse.c
1917  * @IWM_NVM_CHANNEL_DFS: dynamic freq selection candidate
1918  * @IWM_NVM_CHANNEL_WIDE: 20 MHz channel okay (?)
1919  * @IWM_NVM_CHANNEL_40MHZ: 40 MHz channel okay (?)
1920  * @IWM_NVM_CHANNEL_80MHZ: 80 MHz channel okay (?)
1921  * @IWM_NVM_CHANNEL_160MHZ: 160 MHz channel okay (?)
1922  */
1923 enum iwm_nvm_channel_flags {
1924         IWM_NVM_CHANNEL_VALID = (1 << 0),
1925         IWM_NVM_CHANNEL_IBSS = (1 << 1),
1926         IWM_NVM_CHANNEL_ACTIVE = (1 << 3),
1927         IWM_NVM_CHANNEL_RADAR = (1 << 4),
1928         IWM_NVM_CHANNEL_DFS = (1 << 7),
1929         IWM_NVM_CHANNEL_WIDE = (1 << 8),
1930         IWM_NVM_CHANNEL_40MHZ = (1 << 9),
1931         IWM_NVM_CHANNEL_80MHZ = (1 << 10),
1932         IWM_NVM_CHANNEL_160MHZ = (1 << 11),
1933 };
1934
1935 /*
1936  * Translate EEPROM flags to net80211.
1937  */
1938 static uint32_t
1939 iwm_eeprom_channel_flags(uint16_t ch_flags)
1940 {
1941         uint32_t nflags;
1942
1943         nflags = 0;
1944         if ((ch_flags & IWM_NVM_CHANNEL_ACTIVE) == 0)
1945                 nflags |= IEEE80211_CHAN_PASSIVE;
1946         if ((ch_flags & IWM_NVM_CHANNEL_IBSS) == 0)
1947                 nflags |= IEEE80211_CHAN_NOADHOC;
1948         if (ch_flags & IWM_NVM_CHANNEL_RADAR) {
1949                 nflags |= IEEE80211_CHAN_DFS;
1950                 /* Just in case. */
1951                 nflags |= IEEE80211_CHAN_NOADHOC;
1952         }
1953
1954         return (nflags);
1955 }
1956
1957 static void
1958 iwm_add_channel_band(struct iwm_softc *sc, struct ieee80211_channel chans[],
1959     int maxchans, int *nchans, int ch_idx, size_t ch_num,
1960     const uint8_t bands[])
1961 {
1962         const uint16_t * const nvm_ch_flags = sc->nvm_data->nvm_ch_flags;
1963         uint32_t nflags;
1964         uint16_t ch_flags;
1965         uint8_t ieee;
1966         int error;
1967
1968         for (; ch_idx < ch_num; ch_idx++) {
1969                 ch_flags = le16_to_cpup(nvm_ch_flags + ch_idx);
1970                 if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
1971                         ieee = iwm_nvm_channels[ch_idx];
1972                 else
1973                         ieee = iwm_nvm_channels_8000[ch_idx];
1974
1975                 if (!(ch_flags & IWM_NVM_CHANNEL_VALID)) {
1976                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM,
1977                             "Ch. %d Flags %x [%sGHz] - No traffic\n",
1978                             ieee, ch_flags,
1979                             (ch_idx >= IWM_NUM_2GHZ_CHANNELS) ?
1980                             "5.2" : "2.4");
1981                         continue;
1982                 }
1983
1984                 nflags = iwm_eeprom_channel_flags(ch_flags);
1985                 error = ieee80211_add_channel(chans, maxchans, nchans,
1986                     ieee, 0, 0, nflags, bands);
1987                 if (error != 0)
1988                         break;
1989
1990                 IWM_DPRINTF(sc, IWM_DEBUG_EEPROM,
1991                     "Ch. %d Flags %x [%sGHz] - Added\n",
1992                     ieee, ch_flags,
1993                     (ch_idx >= IWM_NUM_2GHZ_CHANNELS) ?
1994                     "5.2" : "2.4");
1995         }
1996 }
1997
1998 static void
1999 iwm_init_channel_map(struct ieee80211com *ic, int maxchans, int *nchans,
2000     struct ieee80211_channel chans[])
2001 {
2002         struct iwm_softc *sc = ic->ic_softc;
2003         struct iwm_nvm_data *data = sc->nvm_data;
2004         uint8_t bands[IEEE80211_MODE_BYTES];
2005         size_t ch_num;
2006
2007         memset(bands, 0, sizeof(bands));
2008         /* 1-13: 11b/g channels. */
2009         setbit(bands, IEEE80211_MODE_11B);
2010         setbit(bands, IEEE80211_MODE_11G);
2011         iwm_add_channel_band(sc, chans, maxchans, nchans, 0,
2012             IWM_NUM_2GHZ_CHANNELS - 1, bands);
2013
2014         /* 14: 11b channel only. */
2015         clrbit(bands, IEEE80211_MODE_11G);
2016         iwm_add_channel_band(sc, chans, maxchans, nchans,
2017             IWM_NUM_2GHZ_CHANNELS - 1, IWM_NUM_2GHZ_CHANNELS, bands);
2018
2019         if (data->sku_cap_band_52GHz_enable) {
2020                 if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
2021                         ch_num = nitems(iwm_nvm_channels);
2022                 else
2023                         ch_num = nitems(iwm_nvm_channels_8000);
2024                 memset(bands, 0, sizeof(bands));
2025                 setbit(bands, IEEE80211_MODE_11A);
2026                 iwm_add_channel_band(sc, chans, maxchans, nchans,
2027                     IWM_NUM_2GHZ_CHANNELS, ch_num, bands);
2028         }
2029 }
2030
2031 static void
2032 iwm_set_hw_address_family_8000(struct iwm_softc *sc, struct iwm_nvm_data *data,
2033         const uint16_t *mac_override, const uint16_t *nvm_hw)
2034 {
2035         const uint8_t *hw_addr;
2036
2037         if (mac_override) {
2038                 static const uint8_t reserved_mac[] = {
2039                         0x02, 0xcc, 0xaa, 0xff, 0xee, 0x00
2040                 };
2041
2042                 hw_addr = (const uint8_t *)(mac_override +
2043                                  IWM_MAC_ADDRESS_OVERRIDE_8000);
2044
2045                 /*
2046                  * Store the MAC address from MAO section.
2047                  * No byte swapping is required in MAO section
2048                  */
2049                 IEEE80211_ADDR_COPY(data->hw_addr, hw_addr);
2050
2051                 /*
2052                  * Force the use of the OTP MAC address in case of reserved MAC
2053                  * address in the NVM, or if address is given but invalid.
2054                  */
2055                 if (!IEEE80211_ADDR_EQ(reserved_mac, hw_addr) &&
2056                     !IEEE80211_ADDR_EQ(ieee80211broadcastaddr, data->hw_addr) &&
2057                     iwm_is_valid_ether_addr(data->hw_addr) &&
2058                     !IEEE80211_IS_MULTICAST(data->hw_addr))
2059                         return;
2060
2061                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2062                     "%s: mac address from nvm override section invalid\n",
2063                     __func__);
2064         }
2065
2066         if (nvm_hw) {
2067                 /* read the mac address from WFMP registers */
2068                 uint32_t mac_addr0 =
2069                     htole32(iwm_read_prph(sc, IWM_WFMP_MAC_ADDR_0));
2070                 uint32_t mac_addr1 =
2071                     htole32(iwm_read_prph(sc, IWM_WFMP_MAC_ADDR_1));
2072
2073                 hw_addr = (const uint8_t *)&mac_addr0;
2074                 data->hw_addr[0] = hw_addr[3];
2075                 data->hw_addr[1] = hw_addr[2];
2076                 data->hw_addr[2] = hw_addr[1];
2077                 data->hw_addr[3] = hw_addr[0];
2078
2079                 hw_addr = (const uint8_t *)&mac_addr1;
2080                 data->hw_addr[4] = hw_addr[1];
2081                 data->hw_addr[5] = hw_addr[0];
2082
2083                 return;
2084         }
2085
2086         device_printf(sc->sc_dev, "%s: mac address not found\n", __func__);
2087         memset(data->hw_addr, 0, sizeof(data->hw_addr));
2088 }
2089
2090 static int
2091 iwm_get_sku(const struct iwm_softc *sc, const uint16_t *nvm_sw,
2092             const uint16_t *phy_sku)
2093 {
2094         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2095                 return le16_to_cpup(nvm_sw + IWM_SKU);
2096
2097         return le32_to_cpup((const uint32_t *)(phy_sku + IWM_SKU_8000));
2098 }
2099
2100 static int
2101 iwm_get_nvm_version(const struct iwm_softc *sc, const uint16_t *nvm_sw)
2102 {
2103         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2104                 return le16_to_cpup(nvm_sw + IWM_NVM_VERSION);
2105         else
2106                 return le32_to_cpup((const uint32_t *)(nvm_sw +
2107                                                 IWM_NVM_VERSION_8000));
2108 }
2109
2110 static int
2111 iwm_get_radio_cfg(const struct iwm_softc *sc, const uint16_t *nvm_sw,
2112                   const uint16_t *phy_sku)
2113 {
2114         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2115                 return le16_to_cpup(nvm_sw + IWM_RADIO_CFG);
2116
2117         return le32_to_cpup((const uint32_t *)(phy_sku + IWM_RADIO_CFG_8000));
2118 }
2119
2120 static int
2121 iwm_get_n_hw_addrs(const struct iwm_softc *sc, const uint16_t *nvm_sw)
2122 {
2123         int n_hw_addr;
2124
2125         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2126                 return le16_to_cpup(nvm_sw + IWM_N_HW_ADDRS);
2127
2128         n_hw_addr = le32_to_cpup((const uint32_t *)(nvm_sw + IWM_N_HW_ADDRS_8000));
2129
2130         return n_hw_addr & IWM_N_HW_ADDR_MASK;
2131 }
2132
2133 static void
2134 iwm_set_radio_cfg(const struct iwm_softc *sc, struct iwm_nvm_data *data,
2135                   uint32_t radio_cfg)
2136 {
2137         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
2138                 data->radio_cfg_type = IWM_NVM_RF_CFG_TYPE_MSK(radio_cfg);
2139                 data->radio_cfg_step = IWM_NVM_RF_CFG_STEP_MSK(radio_cfg);
2140                 data->radio_cfg_dash = IWM_NVM_RF_CFG_DASH_MSK(radio_cfg);
2141                 data->radio_cfg_pnum = IWM_NVM_RF_CFG_PNUM_MSK(radio_cfg);
2142                 return;
2143         }
2144
2145         /* set the radio configuration for family 8000 */
2146         data->radio_cfg_type = IWM_NVM_RF_CFG_TYPE_MSK_8000(radio_cfg);
2147         data->radio_cfg_step = IWM_NVM_RF_CFG_STEP_MSK_8000(radio_cfg);
2148         data->radio_cfg_dash = IWM_NVM_RF_CFG_DASH_MSK_8000(radio_cfg);
2149         data->radio_cfg_pnum = IWM_NVM_RF_CFG_FLAVOR_MSK_8000(radio_cfg);
2150         data->valid_tx_ant = IWM_NVM_RF_CFG_TX_ANT_MSK_8000(radio_cfg);
2151         data->valid_rx_ant = IWM_NVM_RF_CFG_RX_ANT_MSK_8000(radio_cfg);
2152 }
2153
2154 static int
2155 iwm_set_hw_address(struct iwm_softc *sc, struct iwm_nvm_data *data,
2156                    const uint16_t *nvm_hw, const uint16_t *mac_override)
2157 {
2158 #ifdef notyet /* for FAMILY 9000 */
2159         if (cfg->mac_addr_from_csr) {
2160                 iwm_set_hw_address_from_csr(sc, data);
2161         } else
2162 #endif
2163         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
2164                 const uint8_t *hw_addr = (const uint8_t *)(nvm_hw + IWM_HW_ADDR);
2165
2166                 /* The byte order is little endian 16 bit, meaning 214365 */
2167                 data->hw_addr[0] = hw_addr[1];
2168                 data->hw_addr[1] = hw_addr[0];
2169                 data->hw_addr[2] = hw_addr[3];
2170                 data->hw_addr[3] = hw_addr[2];
2171                 data->hw_addr[4] = hw_addr[5];
2172                 data->hw_addr[5] = hw_addr[4];
2173         } else {
2174                 iwm_set_hw_address_family_8000(sc, data, mac_override, nvm_hw);
2175         }
2176
2177         if (!iwm_is_valid_ether_addr(data->hw_addr)) {
2178                 device_printf(sc->sc_dev, "no valid mac address was found\n");
2179                 return EINVAL;
2180         }
2181
2182         return 0;
2183 }
2184
2185 static struct iwm_nvm_data *
2186 iwm_parse_nvm_data(struct iwm_softc *sc,
2187                    const uint16_t *nvm_hw, const uint16_t *nvm_sw,
2188                    const uint16_t *nvm_calib, const uint16_t *mac_override,
2189                    const uint16_t *phy_sku, const uint16_t *regulatory)
2190 {
2191         struct iwm_nvm_data *data;
2192         uint32_t sku, radio_cfg;
2193
2194         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
2195                 data = malloc(sizeof(*data) +
2196                     IWM_NUM_CHANNELS * sizeof(uint16_t),
2197                     M_DEVBUF, M_NOWAIT | M_ZERO);
2198         } else {
2199                 data = malloc(sizeof(*data) +
2200                     IWM_NUM_CHANNELS_8000 * sizeof(uint16_t),
2201                     M_DEVBUF, M_NOWAIT | M_ZERO);
2202         }
2203         if (!data)
2204                 return NULL;
2205
2206         data->nvm_version = iwm_get_nvm_version(sc, nvm_sw);
2207
2208         radio_cfg = iwm_get_radio_cfg(sc, nvm_sw, phy_sku);
2209         iwm_set_radio_cfg(sc, data, radio_cfg);
2210
2211         sku = iwm_get_sku(sc, nvm_sw, phy_sku);
2212         data->sku_cap_band_24GHz_enable = sku & IWM_NVM_SKU_CAP_BAND_24GHZ;
2213         data->sku_cap_band_52GHz_enable = sku & IWM_NVM_SKU_CAP_BAND_52GHZ;
2214         data->sku_cap_11n_enable = 0;
2215
2216         data->n_hw_addrs = iwm_get_n_hw_addrs(sc, nvm_sw);
2217
2218         /* If no valid mac address was found - bail out */
2219         if (iwm_set_hw_address(sc, data, nvm_hw, mac_override)) {
2220                 free(data, M_DEVBUF);
2221                 return NULL;
2222         }
2223
2224         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
2225                 memcpy(data->nvm_ch_flags, &nvm_sw[IWM_NVM_CHANNELS],
2226                     IWM_NUM_CHANNELS * sizeof(uint16_t));
2227         } else {
2228                 memcpy(data->nvm_ch_flags, &regulatory[IWM_NVM_CHANNELS_8000],
2229                     IWM_NUM_CHANNELS_8000 * sizeof(uint16_t));
2230         }
2231
2232         return data;
2233 }
2234
2235 static void
2236 iwm_free_nvm_data(struct iwm_nvm_data *data)
2237 {
2238         if (data != NULL)
2239                 free(data, M_DEVBUF);
2240 }
2241
2242 static struct iwm_nvm_data *
2243 iwm_parse_nvm_sections(struct iwm_softc *sc, struct iwm_nvm_section *sections)
2244 {
2245         const uint16_t *hw, *sw, *calib, *regulatory, *mac_override, *phy_sku;
2246
2247         /* Checking for required sections */
2248         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
2249                 if (!sections[IWM_NVM_SECTION_TYPE_SW].data ||
2250                     !sections[sc->cfg->nvm_hw_section_num].data) {
2251                         device_printf(sc->sc_dev,
2252                             "Can't parse empty OTP/NVM sections\n");
2253                         return NULL;
2254                 }
2255         } else if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) {
2256                 /* SW and REGULATORY sections are mandatory */
2257                 if (!sections[IWM_NVM_SECTION_TYPE_SW].data ||
2258                     !sections[IWM_NVM_SECTION_TYPE_REGULATORY].data) {
2259                         device_printf(sc->sc_dev,
2260                             "Can't parse empty OTP/NVM sections\n");
2261                         return NULL;
2262                 }
2263                 /* MAC_OVERRIDE or at least HW section must exist */
2264                 if (!sections[sc->cfg->nvm_hw_section_num].data &&
2265                     !sections[IWM_NVM_SECTION_TYPE_MAC_OVERRIDE].data) {
2266                         device_printf(sc->sc_dev,
2267                             "Can't parse mac_address, empty sections\n");
2268                         return NULL;
2269                 }
2270
2271                 /* PHY_SKU section is mandatory in B0 */
2272                 if (!sections[IWM_NVM_SECTION_TYPE_PHY_SKU].data) {
2273                         device_printf(sc->sc_dev,
2274                             "Can't parse phy_sku in B0, empty sections\n");
2275                         return NULL;
2276                 }
2277         } else {
2278                 panic("unknown device family %d\n", sc->cfg->device_family);
2279         }
2280
2281         hw = (const uint16_t *) sections[sc->cfg->nvm_hw_section_num].data;
2282         sw = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_SW].data;
2283         calib = (const uint16_t *)
2284             sections[IWM_NVM_SECTION_TYPE_CALIBRATION].data;
2285         regulatory = (const uint16_t *)
2286             sections[IWM_NVM_SECTION_TYPE_REGULATORY].data;
2287         mac_override = (const uint16_t *)
2288             sections[IWM_NVM_SECTION_TYPE_MAC_OVERRIDE].data;
2289         phy_sku = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_PHY_SKU].data;
2290
2291         return iwm_parse_nvm_data(sc, hw, sw, calib, mac_override,
2292             phy_sku, regulatory);
2293 }
2294
2295 static int
2296 iwm_nvm_init(struct iwm_softc *sc)
2297 {
2298         struct iwm_nvm_section nvm_sections[IWM_NVM_MAX_NUM_SECTIONS];
2299         int i, ret, section;
2300         uint32_t size_read = 0;
2301         uint8_t *nvm_buffer, *temp;
2302         uint16_t len;
2303
2304         memset(nvm_sections, 0, sizeof(nvm_sections));
2305
2306         if (sc->cfg->nvm_hw_section_num >= IWM_NVM_MAX_NUM_SECTIONS)
2307                 return EINVAL;
2308
2309         /* load NVM values from nic */
2310         /* Read From FW NVM */
2311         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM, "Read from NVM\n");
2312
2313         nvm_buffer = malloc(sc->cfg->eeprom_size, M_DEVBUF, M_NOWAIT | M_ZERO);
2314         if (!nvm_buffer)
2315                 return ENOMEM;
2316         for (section = 0; section < IWM_NVM_MAX_NUM_SECTIONS; section++) {
2317                 /* we override the constness for initial read */
2318                 ret = iwm_nvm_read_section(sc, section, nvm_buffer,
2319                                            &len, size_read);
2320                 if (ret)
2321                         continue;
2322                 size_read += len;
2323                 temp = malloc(len, M_DEVBUF, M_NOWAIT);
2324                 if (!temp) {
2325                         ret = ENOMEM;
2326                         break;
2327                 }
2328                 memcpy(temp, nvm_buffer, len);
2329
2330                 nvm_sections[section].data = temp;
2331                 nvm_sections[section].length = len;
2332         }
2333         if (!size_read)
2334                 device_printf(sc->sc_dev, "OTP is blank\n");
2335         free(nvm_buffer, M_DEVBUF);
2336
2337         sc->nvm_data = iwm_parse_nvm_sections(sc, nvm_sections);
2338         if (!sc->nvm_data)
2339                 return EINVAL;
2340         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
2341                     "nvm version = %x\n", sc->nvm_data->nvm_version);
2342
2343         for (i = 0; i < IWM_NVM_MAX_NUM_SECTIONS; i++) {
2344                 if (nvm_sections[i].data != NULL)
2345                         free(nvm_sections[i].data, M_DEVBUF);
2346         }
2347
2348         return 0;
2349 }
2350
2351 static int
2352 iwm_pcie_load_section(struct iwm_softc *sc, uint8_t section_num,
2353         const struct iwm_fw_desc *section)
2354 {
2355         struct iwm_dma_info *dma = &sc->fw_dma;
2356         uint8_t *v_addr;
2357         bus_addr_t p_addr;
2358         uint32_t offset, chunk_sz = MIN(IWM_FH_MEM_TB_MAX_LENGTH, section->len);
2359         int ret = 0;
2360
2361         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2362                     "%s: [%d] uCode section being loaded...\n",
2363                     __func__, section_num);
2364
2365         v_addr = dma->vaddr;
2366         p_addr = dma->paddr;
2367
2368         for (offset = 0; offset < section->len; offset += chunk_sz) {
2369                 uint32_t copy_size, dst_addr;
2370                 int extended_addr = FALSE;
2371
2372                 copy_size = MIN(chunk_sz, section->len - offset);
2373                 dst_addr = section->offset + offset;
2374
2375                 if (dst_addr >= IWM_FW_MEM_EXTENDED_START &&
2376                     dst_addr <= IWM_FW_MEM_EXTENDED_END)
2377                         extended_addr = TRUE;
2378
2379                 if (extended_addr)
2380                         iwm_set_bits_prph(sc, IWM_LMPM_CHICK,
2381                                           IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE);
2382
2383                 memcpy(v_addr, (const uint8_t *)section->data + offset,
2384                     copy_size);
2385                 bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREWRITE);
2386                 ret = iwm_pcie_load_firmware_chunk(sc, dst_addr, p_addr,
2387                                                    copy_size);
2388
2389                 if (extended_addr)
2390                         iwm_clear_bits_prph(sc, IWM_LMPM_CHICK,
2391                                             IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE);
2392
2393                 if (ret) {
2394                         device_printf(sc->sc_dev,
2395                             "%s: Could not load the [%d] uCode section\n",
2396                             __func__, section_num);
2397                         break;
2398                 }
2399         }
2400
2401         return ret;
2402 }
2403
2404 /*
2405  * ucode
2406  */
2407 static int
2408 iwm_pcie_load_firmware_chunk(struct iwm_softc *sc, uint32_t dst_addr,
2409                              bus_addr_t phy_addr, uint32_t byte_cnt)
2410 {
2411         int ret;
2412
2413         sc->sc_fw_chunk_done = 0;
2414
2415         if (!iwm_nic_lock(sc))
2416                 return EBUSY;
2417
2418         IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(IWM_FH_SRVC_CHNL),
2419             IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE);
2420
2421         IWM_WRITE(sc, IWM_FH_SRVC_CHNL_SRAM_ADDR_REG(IWM_FH_SRVC_CHNL),
2422             dst_addr);
2423
2424         IWM_WRITE(sc, IWM_FH_TFDIB_CTRL0_REG(IWM_FH_SRVC_CHNL),
2425             phy_addr & IWM_FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK);
2426
2427         IWM_WRITE(sc, IWM_FH_TFDIB_CTRL1_REG(IWM_FH_SRVC_CHNL),
2428             (iwm_get_dma_hi_addr(phy_addr)
2429              << IWM_FH_MEM_TFDIB_REG1_ADDR_BITSHIFT) | byte_cnt);
2430
2431         IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_BUF_STS_REG(IWM_FH_SRVC_CHNL),
2432             1 << IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM |
2433             1 << IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX |
2434             IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID);
2435
2436         IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(IWM_FH_SRVC_CHNL),
2437             IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE    |
2438             IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE |
2439             IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD);
2440
2441         iwm_nic_unlock(sc);
2442
2443         /* wait up to 5s for this segment to load */
2444         ret = 0;
2445         while (!sc->sc_fw_chunk_done) {
2446                 ret = msleep(&sc->sc_fw, &sc->sc_mtx, 0, "iwmfw", hz);
2447                 if (ret)
2448                         break;
2449         }
2450
2451         if (ret != 0) {
2452                 device_printf(sc->sc_dev,
2453                     "fw chunk addr 0x%x len %d failed to load\n",
2454                     dst_addr, byte_cnt);
2455                 return ETIMEDOUT;
2456         }
2457
2458         return 0;
2459 }
2460
2461 static int
2462 iwm_pcie_load_cpu_sections_8000(struct iwm_softc *sc,
2463         const struct iwm_fw_sects *image, int cpu, int *first_ucode_section)
2464 {
2465         int shift_param;
2466         int i, ret = 0, sec_num = 0x1;
2467         uint32_t val, last_read_idx = 0;
2468
2469         if (cpu == 1) {
2470                 shift_param = 0;
2471                 *first_ucode_section = 0;
2472         } else {
2473                 shift_param = 16;
2474                 (*first_ucode_section)++;
2475         }
2476
2477         for (i = *first_ucode_section; i < IWM_UCODE_SECTION_MAX; i++) {
2478                 last_read_idx = i;
2479
2480                 /*
2481                  * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
2482                  * CPU1 to CPU2.
2483                  * PAGING_SEPARATOR_SECTION delimiter - separate between
2484                  * CPU2 non paged to CPU2 paging sec.
2485                  */
2486                 if (!image->fw_sect[i].data ||
2487                     image->fw_sect[i].offset == IWM_CPU1_CPU2_SEPARATOR_SECTION ||
2488                     image->fw_sect[i].offset == IWM_PAGING_SEPARATOR_SECTION) {
2489                         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2490                                     "Break since Data not valid or Empty section, sec = %d\n",
2491                                     i);
2492                         break;
2493                 }
2494                 ret = iwm_pcie_load_section(sc, i, &image->fw_sect[i]);
2495                 if (ret)
2496                         return ret;
2497
2498                 /* Notify the ucode of the loaded section number and status */
2499                 if (iwm_nic_lock(sc)) {
2500                         val = IWM_READ(sc, IWM_FH_UCODE_LOAD_STATUS);
2501                         val = val | (sec_num << shift_param);
2502                         IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, val);
2503                         sec_num = (sec_num << 1) | 0x1;
2504                         iwm_nic_unlock(sc);
2505                 }
2506         }
2507
2508         *first_ucode_section = last_read_idx;
2509
2510         iwm_enable_interrupts(sc);
2511
2512         if (iwm_nic_lock(sc)) {
2513                 if (cpu == 1)
2514                         IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, 0xFFFF);
2515                 else
2516                         IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, 0xFFFFFFFF);
2517                 iwm_nic_unlock(sc);
2518         }
2519
2520         return 0;
2521 }
2522
2523 static int
2524 iwm_pcie_load_cpu_sections(struct iwm_softc *sc,
2525         const struct iwm_fw_sects *image, int cpu, int *first_ucode_section)
2526 {
2527         int shift_param;
2528         int i, ret = 0;
2529         uint32_t last_read_idx = 0;
2530
2531         if (cpu == 1) {
2532                 shift_param = 0;
2533                 *first_ucode_section = 0;
2534         } else {
2535                 shift_param = 16;
2536                 (*first_ucode_section)++;
2537         }
2538
2539         for (i = *first_ucode_section; i < IWM_UCODE_SECTION_MAX; i++) {
2540                 last_read_idx = i;
2541
2542                 /*
2543                  * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
2544                  * CPU1 to CPU2.
2545                  * PAGING_SEPARATOR_SECTION delimiter - separate between
2546                  * CPU2 non paged to CPU2 paging sec.
2547                  */
2548                 if (!image->fw_sect[i].data ||
2549                     image->fw_sect[i].offset == IWM_CPU1_CPU2_SEPARATOR_SECTION ||
2550                     image->fw_sect[i].offset == IWM_PAGING_SEPARATOR_SECTION) {
2551                         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2552                                     "Break since Data not valid or Empty section, sec = %d\n",
2553                                      i);
2554                         break;
2555                 }
2556
2557                 ret = iwm_pcie_load_section(sc, i, &image->fw_sect[i]);
2558                 if (ret)
2559                         return ret;
2560         }
2561
2562         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000)
2563                 iwm_set_bits_prph(sc,
2564                                   IWM_CSR_UCODE_LOAD_STATUS_ADDR,
2565                                   (IWM_LMPM_CPU_UCODE_LOADING_COMPLETED |
2566                                    IWM_LMPM_CPU_HDRS_LOADING_COMPLETED |
2567                                    IWM_LMPM_CPU_UCODE_LOADING_STARTED) <<
2568                                         shift_param);
2569
2570         *first_ucode_section = last_read_idx;
2571
2572         return 0;
2573
2574 }
2575
2576 static int
2577 iwm_pcie_load_given_ucode(struct iwm_softc *sc,
2578         const struct iwm_fw_sects *image)
2579 {
2580         int ret = 0;
2581         int first_ucode_section;
2582
2583         IWM_DPRINTF(sc, IWM_DEBUG_RESET, "working with %s CPU\n",
2584                      image->is_dual_cpus ? "Dual" : "Single");
2585
2586         /* load to FW the binary non secured sections of CPU1 */
2587         ret = iwm_pcie_load_cpu_sections(sc, image, 1, &first_ucode_section);
2588         if (ret)
2589                 return ret;
2590
2591         if (image->is_dual_cpus) {
2592                 /* set CPU2 header address */
2593                 iwm_write_prph(sc,
2594                                IWM_LMPM_SECURE_UCODE_LOAD_CPU2_HDR_ADDR,
2595                                IWM_LMPM_SECURE_CPU2_HDR_MEM_SPACE);
2596
2597                 /* load to FW the binary sections of CPU2 */
2598                 ret = iwm_pcie_load_cpu_sections(sc, image, 2,
2599                                                  &first_ucode_section);
2600                 if (ret)
2601                         return ret;
2602         }
2603
2604         iwm_enable_interrupts(sc);
2605
2606         /* release CPU reset */
2607         IWM_WRITE(sc, IWM_CSR_RESET, 0);
2608
2609         return 0;
2610 }
2611
2612 int
2613 iwm_pcie_load_given_ucode_8000(struct iwm_softc *sc,
2614         const struct iwm_fw_sects *image)
2615 {
2616         int ret = 0;
2617         int first_ucode_section;
2618
2619         IWM_DPRINTF(sc, IWM_DEBUG_RESET, "working with %s CPU\n",
2620                     image->is_dual_cpus ? "Dual" : "Single");
2621
2622         /* configure the ucode to be ready to get the secured image */
2623         /* release CPU reset */
2624         iwm_write_prph(sc, IWM_RELEASE_CPU_RESET, IWM_RELEASE_CPU_RESET_BIT);
2625
2626         /* load to FW the binary Secured sections of CPU1 */
2627         ret = iwm_pcie_load_cpu_sections_8000(sc, image, 1,
2628             &first_ucode_section);
2629         if (ret)
2630                 return ret;
2631
2632         /* load to FW the binary sections of CPU2 */
2633         return iwm_pcie_load_cpu_sections_8000(sc, image, 2,
2634             &first_ucode_section);
2635 }
2636
2637 /* XXX Get rid of this definition */
2638 static inline void
2639 iwm_enable_fw_load_int(struct iwm_softc *sc)
2640 {
2641         IWM_DPRINTF(sc, IWM_DEBUG_INTR, "Enabling FW load interrupt\n");
2642         sc->sc_intmask = IWM_CSR_INT_BIT_FH_TX;
2643         IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask);
2644 }
2645
2646 /* XXX Add proper rfkill support code */
2647 static int
2648 iwm_start_fw(struct iwm_softc *sc,
2649         const struct iwm_fw_sects *fw)
2650 {
2651         int ret;
2652
2653         /* This may fail if AMT took ownership of the device */
2654         if (iwm_prepare_card_hw(sc)) {
2655                 device_printf(sc->sc_dev,
2656                     "%s: Exit HW not ready\n", __func__);
2657                 ret = EIO;
2658                 goto out;
2659         }
2660
2661         IWM_WRITE(sc, IWM_CSR_INT, 0xFFFFFFFF);
2662
2663         iwm_disable_interrupts(sc);
2664
2665         /* make sure rfkill handshake bits are cleared */
2666         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL);
2667         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR,
2668             IWM_CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED);
2669
2670         /* clear (again), then enable host interrupts */
2671         IWM_WRITE(sc, IWM_CSR_INT, 0xFFFFFFFF);
2672
2673         ret = iwm_nic_init(sc);
2674         if (ret) {
2675                 device_printf(sc->sc_dev, "%s: Unable to init nic\n", __func__);
2676                 goto out;
2677         }
2678
2679         /*
2680          * Now, we load the firmware and don't want to be interrupted, even
2681          * by the RF-Kill interrupt (hence mask all the interrupt besides the
2682          * FH_TX interrupt which is needed to load the firmware). If the
2683          * RF-Kill switch is toggled, we will find out after having loaded
2684          * the firmware and return the proper value to the caller.
2685          */
2686         iwm_enable_fw_load_int(sc);
2687
2688         /* really make sure rfkill handshake bits are cleared */
2689         /* maybe we should write a few times more?  just to make sure */
2690         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL);
2691         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL);
2692
2693         /* Load the given image to the HW */
2694         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000)
2695                 ret = iwm_pcie_load_given_ucode_8000(sc, fw);
2696         else
2697                 ret = iwm_pcie_load_given_ucode(sc, fw);
2698
2699         /* XXX re-check RF-Kill state */
2700
2701 out:
2702         return ret;
2703 }
2704
2705 static int
2706 iwm_send_tx_ant_cfg(struct iwm_softc *sc, uint8_t valid_tx_ant)
2707 {
2708         struct iwm_tx_ant_cfg_cmd tx_ant_cmd = {
2709                 .valid = htole32(valid_tx_ant),
2710         };
2711
2712         return iwm_mvm_send_cmd_pdu(sc, IWM_TX_ANT_CONFIGURATION_CMD,
2713             IWM_CMD_SYNC, sizeof(tx_ant_cmd), &tx_ant_cmd);
2714 }
2715
2716 /* iwlwifi: mvm/fw.c */
2717 static int
2718 iwm_send_phy_cfg_cmd(struct iwm_softc *sc)
2719 {
2720         struct iwm_phy_cfg_cmd phy_cfg_cmd;
2721         enum iwm_ucode_type ucode_type = sc->cur_ucode;
2722
2723         /* Set parameters */
2724         phy_cfg_cmd.phy_cfg = htole32(iwm_mvm_get_phy_config(sc));
2725         phy_cfg_cmd.calib_control.event_trigger =
2726             sc->sc_default_calib[ucode_type].event_trigger;
2727         phy_cfg_cmd.calib_control.flow_trigger =
2728             sc->sc_default_calib[ucode_type].flow_trigger;
2729
2730         IWM_DPRINTF(sc, IWM_DEBUG_CMD | IWM_DEBUG_RESET,
2731             "Sending Phy CFG command: 0x%x\n", phy_cfg_cmd.phy_cfg);
2732         return iwm_mvm_send_cmd_pdu(sc, IWM_PHY_CONFIGURATION_CMD, IWM_CMD_SYNC,
2733             sizeof(phy_cfg_cmd), &phy_cfg_cmd);
2734 }
2735
2736 static int
2737 iwm_alive_fn(struct iwm_softc *sc, struct iwm_rx_packet *pkt, void *data)
2738 {
2739         struct iwm_mvm_alive_data *alive_data = data;
2740         struct iwm_mvm_alive_resp_ver1 *palive1;
2741         struct iwm_mvm_alive_resp_ver2 *palive2;
2742         struct iwm_mvm_alive_resp *palive;
2743
2744         if (iwm_rx_packet_payload_len(pkt) == sizeof(*palive1)) {
2745                 palive1 = (void *)pkt->data;
2746
2747                 sc->support_umac_log = FALSE;
2748                 sc->error_event_table =
2749                         le32toh(palive1->error_event_table_ptr);
2750                 sc->log_event_table =
2751                         le32toh(palive1->log_event_table_ptr);
2752                 alive_data->scd_base_addr = le32toh(palive1->scd_base_ptr);
2753
2754                 alive_data->valid = le16toh(palive1->status) ==
2755                                     IWM_ALIVE_STATUS_OK;
2756                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2757                             "Alive VER1 ucode status 0x%04x revision 0x%01X 0x%01X flags 0x%01X\n",
2758                              le16toh(palive1->status), palive1->ver_type,
2759                              palive1->ver_subtype, palive1->flags);
2760         } else if (iwm_rx_packet_payload_len(pkt) == sizeof(*palive2)) {
2761                 palive2 = (void *)pkt->data;
2762                 sc->error_event_table =
2763                         le32toh(palive2->error_event_table_ptr);
2764                 sc->log_event_table =
2765                         le32toh(palive2->log_event_table_ptr);
2766                 alive_data->scd_base_addr = le32toh(palive2->scd_base_ptr);
2767                 sc->umac_error_event_table =
2768                         le32toh(palive2->error_info_addr);
2769
2770                 alive_data->valid = le16toh(palive2->status) ==
2771                                     IWM_ALIVE_STATUS_OK;
2772                 if (sc->umac_error_event_table)
2773                         sc->support_umac_log = TRUE;
2774
2775                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2776                             "Alive VER2 ucode status 0x%04x revision 0x%01X 0x%01X flags 0x%01X\n",
2777                             le16toh(palive2->status), palive2->ver_type,
2778                             palive2->ver_subtype, palive2->flags);
2779
2780                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2781                             "UMAC version: Major - 0x%x, Minor - 0x%x\n",
2782                             palive2->umac_major, palive2->umac_minor);
2783         } else if (iwm_rx_packet_payload_len(pkt) == sizeof(*palive)) {
2784                 palive = (void *)pkt->data;
2785
2786                 sc->error_event_table =
2787                         le32toh(palive->error_event_table_ptr);
2788                 sc->log_event_table =
2789                         le32toh(palive->log_event_table_ptr);
2790                 alive_data->scd_base_addr = le32toh(palive->scd_base_ptr);
2791                 sc->umac_error_event_table =
2792                         le32toh(palive->error_info_addr);
2793
2794                 alive_data->valid = le16toh(palive->status) ==
2795                                     IWM_ALIVE_STATUS_OK;
2796                 if (sc->umac_error_event_table)
2797                         sc->support_umac_log = TRUE;
2798
2799                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2800                             "Alive VER3 ucode status 0x%04x revision 0x%01X 0x%01X flags 0x%01X\n",
2801                             le16toh(palive->status), palive->ver_type,
2802                             palive->ver_subtype, palive->flags);
2803
2804                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2805                             "UMAC version: Major - 0x%x, Minor - 0x%x\n",
2806                             le32toh(palive->umac_major),
2807                             le32toh(palive->umac_minor));
2808         }
2809
2810         return TRUE;
2811 }
2812
2813 static int
2814 iwm_wait_phy_db_entry(struct iwm_softc *sc,
2815         struct iwm_rx_packet *pkt, void *data)
2816 {
2817         struct iwm_phy_db *phy_db = data;
2818
2819         if (pkt->hdr.code != IWM_CALIB_RES_NOTIF_PHY_DB) {
2820                 if(pkt->hdr.code != IWM_INIT_COMPLETE_NOTIF) {
2821                         device_printf(sc->sc_dev, "%s: Unexpected cmd: %d\n",
2822                             __func__, pkt->hdr.code);
2823                 }
2824                 return TRUE;
2825         }
2826
2827         if (iwm_phy_db_set_section(phy_db, pkt)) {
2828                 device_printf(sc->sc_dev,
2829                     "%s: iwm_phy_db_set_section failed\n", __func__);
2830         }
2831
2832         return FALSE;
2833 }
2834
2835 static int
2836 iwm_mvm_load_ucode_wait_alive(struct iwm_softc *sc,
2837         enum iwm_ucode_type ucode_type)
2838 {
2839         struct iwm_notification_wait alive_wait;
2840         struct iwm_mvm_alive_data alive_data;
2841         const struct iwm_fw_sects *fw;
2842         enum iwm_ucode_type old_type = sc->cur_ucode;
2843         int error;
2844         static const uint16_t alive_cmd[] = { IWM_MVM_ALIVE };
2845
2846         if ((error = iwm_read_firmware(sc, ucode_type)) != 0) {
2847                 device_printf(sc->sc_dev, "iwm_read_firmware: failed %d\n",
2848                         error);
2849                 return error;
2850         }
2851         fw = &sc->sc_fw.fw_sects[ucode_type];
2852         sc->cur_ucode = ucode_type;
2853         sc->ucode_loaded = FALSE;
2854
2855         memset(&alive_data, 0, sizeof(alive_data));
2856         iwm_init_notification_wait(sc->sc_notif_wait, &alive_wait,
2857                                    alive_cmd, nitems(alive_cmd),
2858                                    iwm_alive_fn, &alive_data);
2859
2860         error = iwm_start_fw(sc, fw);
2861         if (error) {
2862                 device_printf(sc->sc_dev, "iwm_start_fw: failed %d\n", error);
2863                 sc->cur_ucode = old_type;
2864                 iwm_remove_notification(sc->sc_notif_wait, &alive_wait);
2865                 return error;
2866         }
2867
2868         /*
2869          * Some things may run in the background now, but we
2870          * just wait for the ALIVE notification here.
2871          */
2872         IWM_UNLOCK(sc);
2873         error = iwm_wait_notification(sc->sc_notif_wait, &alive_wait,
2874                                       IWM_MVM_UCODE_ALIVE_TIMEOUT);
2875         IWM_LOCK(sc);
2876         if (error) {
2877                 if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) {
2878                         device_printf(sc->sc_dev,
2879                             "SecBoot CPU1 Status: 0x%x, CPU2 Status: 0x%x\n",
2880                             iwm_read_prph(sc, IWM_SB_CPU_1_STATUS),
2881                             iwm_read_prph(sc, IWM_SB_CPU_2_STATUS));
2882                 }
2883                 sc->cur_ucode = old_type;
2884                 return error;
2885         }
2886
2887         if (!alive_data.valid) {
2888                 device_printf(sc->sc_dev, "%s: Loaded ucode is not valid\n",
2889                     __func__);
2890                 sc->cur_ucode = old_type;
2891                 return EIO;
2892         }
2893
2894         iwm_trans_pcie_fw_alive(sc, alive_data.scd_base_addr);
2895
2896         /*
2897          * configure and operate fw paging mechanism.
2898          * driver configures the paging flow only once, CPU2 paging image
2899          * included in the IWM_UCODE_INIT image.
2900          */
2901         if (fw->paging_mem_size) {
2902                 error = iwm_save_fw_paging(sc, fw);
2903                 if (error) {
2904                         device_printf(sc->sc_dev,
2905                             "%s: failed to save the FW paging image\n",
2906                             __func__);
2907                         return error;
2908                 }
2909
2910                 error = iwm_send_paging_cmd(sc, fw);
2911                 if (error) {
2912                         device_printf(sc->sc_dev,
2913                             "%s: failed to send the paging cmd\n", __func__);
2914                         iwm_free_fw_paging(sc);
2915                         return error;
2916                 }
2917         }
2918
2919         if (!error)
2920                 sc->ucode_loaded = TRUE;
2921         return error;
2922 }
2923
2924 /*
2925  * mvm misc bits
2926  */
2927
2928 /*
2929  * follows iwlwifi/fw.c
2930  */
2931 static int
2932 iwm_run_init_mvm_ucode(struct iwm_softc *sc, int justnvm)
2933 {
2934         struct iwm_notification_wait calib_wait;
2935         static const uint16_t init_complete[] = {
2936                 IWM_INIT_COMPLETE_NOTIF,
2937                 IWM_CALIB_RES_NOTIF_PHY_DB
2938         };
2939         int ret;
2940
2941         /* do not operate with rfkill switch turned on */
2942         if ((sc->sc_flags & IWM_FLAG_RFKILL) && !justnvm) {
2943                 device_printf(sc->sc_dev,
2944                     "radio is disabled by hardware switch\n");
2945                 return EPERM;
2946         }
2947
2948         iwm_init_notification_wait(sc->sc_notif_wait,
2949                                    &calib_wait,
2950                                    init_complete,
2951                                    nitems(init_complete),
2952                                    iwm_wait_phy_db_entry,
2953                                    sc->sc_phy_db);
2954
2955         /* Will also start the device */
2956         ret = iwm_mvm_load_ucode_wait_alive(sc, IWM_UCODE_INIT);
2957         if (ret) {
2958                 device_printf(sc->sc_dev, "Failed to start INIT ucode: %d\n",
2959                     ret);
2960                 goto error;
2961         }
2962
2963         if (justnvm) {
2964                 /* Read nvm */
2965                 ret = iwm_nvm_init(sc);
2966                 if (ret) {
2967                         device_printf(sc->sc_dev, "failed to read nvm\n");
2968                         goto error;
2969                 }
2970                 IEEE80211_ADDR_COPY(sc->sc_ic.ic_macaddr, sc->nvm_data->hw_addr);
2971                 goto error;
2972         }
2973
2974         ret = iwm_send_bt_init_conf(sc);
2975         if (ret) {
2976                 device_printf(sc->sc_dev,
2977                     "failed to send bt coex configuration: %d\n", ret);
2978                 goto error;
2979         }
2980
2981         /* Init Smart FIFO. */
2982         ret = iwm_mvm_sf_config(sc, IWM_SF_INIT_OFF);
2983         if (ret)
2984                 goto error;
2985
2986         /* Send TX valid antennas before triggering calibrations */
2987         ret = iwm_send_tx_ant_cfg(sc, iwm_mvm_get_valid_tx_ant(sc));
2988         if (ret) {
2989                 device_printf(sc->sc_dev,
2990                     "failed to send antennas before calibration: %d\n", ret);
2991                 goto error;
2992         }
2993
2994         /*
2995          * Send phy configurations command to init uCode
2996          * to start the 16.0 uCode init image internal calibrations.
2997          */
2998         ret = iwm_send_phy_cfg_cmd(sc);
2999         if (ret) {
3000                 device_printf(sc->sc_dev,
3001                     "%s: Failed to run INIT calibrations: %d\n",
3002                     __func__, ret);
3003                 goto error;
3004         }
3005
3006         /*
3007          * Nothing to do but wait for the init complete notification
3008          * from the firmware.
3009          */
3010         IWM_UNLOCK(sc);
3011         ret = iwm_wait_notification(sc->sc_notif_wait, &calib_wait,
3012             IWM_MVM_UCODE_CALIB_TIMEOUT);
3013         IWM_LOCK(sc);
3014
3015
3016         goto out;
3017
3018 error:
3019         iwm_remove_notification(sc->sc_notif_wait, &calib_wait);
3020 out:
3021         return ret;
3022 }
3023
3024 /*
3025  * receive side
3026  */
3027
3028 /* (re)stock rx ring, called at init-time and at runtime */
3029 static int
3030 iwm_rx_addbuf(struct iwm_softc *sc, int size, int idx)
3031 {
3032         struct iwm_rx_ring *ring = &sc->rxq;
3033         struct iwm_rx_data *data = &ring->data[idx];
3034         struct mbuf *m;
3035         bus_dmamap_t dmamap = NULL;
3036         bus_dma_segment_t seg;
3037         int nsegs, error;
3038
3039         m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, IWM_RBUF_SIZE);
3040         if (m == NULL)
3041                 return ENOBUFS;
3042
3043         m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
3044         error = bus_dmamap_load_mbuf_sg(ring->data_dmat, ring->spare_map, m,
3045             &seg, &nsegs, BUS_DMA_NOWAIT);
3046         if (error != 0) {
3047                 device_printf(sc->sc_dev,
3048                     "%s: can't map mbuf, error %d\n", __func__, error);
3049                 goto fail;
3050         }
3051
3052         if (data->m != NULL)
3053                 bus_dmamap_unload(ring->data_dmat, data->map);
3054
3055         /* Swap ring->spare_map with data->map */
3056         dmamap = data->map;
3057         data->map = ring->spare_map;
3058         ring->spare_map = dmamap;
3059
3060         bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_PREREAD);
3061         data->m = m;
3062
3063         /* Update RX descriptor. */
3064         KASSERT((seg.ds_addr & 255) == 0, ("seg.ds_addr not aligned"));
3065         ring->desc[idx] = htole32(seg.ds_addr >> 8);
3066         bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map,
3067             BUS_DMASYNC_PREWRITE);
3068
3069         return 0;
3070 fail:
3071         m_freem(m);
3072         return error;
3073 }
3074
3075 /* iwlwifi: mvm/rx.c */
3076 /*
3077  * iwm_mvm_get_signal_strength - use new rx PHY INFO API
3078  * values are reported by the fw as positive values - need to negate
3079  * to obtain their dBM.  Account for missing antennas by replacing 0
3080  * values by -256dBm: practically 0 power and a non-feasible 8 bit value.
3081  */
3082 static int
3083 iwm_mvm_get_signal_strength(struct iwm_softc *sc, struct iwm_rx_phy_info *phy_info)
3084 {
3085         int energy_a, energy_b, energy_c, max_energy;
3086         uint32_t val;
3087
3088         val = le32toh(phy_info->non_cfg_phy[IWM_RX_INFO_ENERGY_ANT_ABC_IDX]);
3089         energy_a = (val & IWM_RX_INFO_ENERGY_ANT_A_MSK) >>
3090             IWM_RX_INFO_ENERGY_ANT_A_POS;
3091         energy_a = energy_a ? -energy_a : -256;
3092         energy_b = (val & IWM_RX_INFO_ENERGY_ANT_B_MSK) >>
3093             IWM_RX_INFO_ENERGY_ANT_B_POS;
3094         energy_b = energy_b ? -energy_b : -256;
3095         energy_c = (val & IWM_RX_INFO_ENERGY_ANT_C_MSK) >>
3096             IWM_RX_INFO_ENERGY_ANT_C_POS;
3097         energy_c = energy_c ? -energy_c : -256;
3098         max_energy = MAX(energy_a, energy_b);
3099         max_energy = MAX(max_energy, energy_c);
3100
3101         IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3102             "energy In A %d B %d C %d , and max %d\n",
3103             energy_a, energy_b, energy_c, max_energy);
3104
3105         return max_energy;
3106 }
3107
3108 static void
3109 iwm_mvm_rx_rx_phy_cmd(struct iwm_softc *sc,
3110         struct iwm_rx_packet *pkt, struct iwm_rx_data *data)
3111 {
3112         struct iwm_rx_phy_info *phy_info = (void *)pkt->data;
3113
3114         IWM_DPRINTF(sc, IWM_DEBUG_RECV, "received PHY stats\n");
3115         bus_dmamap_sync(sc->rxq.data_dmat, data->map, BUS_DMASYNC_POSTREAD);
3116
3117         memcpy(&sc->sc_last_phy_info, phy_info, sizeof(sc->sc_last_phy_info));
3118 }
3119
3120 /*
3121  * Retrieve the average noise (in dBm) among receivers.
3122  */
3123 static int
3124 iwm_get_noise(struct iwm_softc *sc,
3125     const struct iwm_mvm_statistics_rx_non_phy *stats)
3126 {
3127         int i, total, nbant, noise;
3128
3129         total = nbant = noise = 0;
3130         for (i = 0; i < 3; i++) {
3131                 noise = le32toh(stats->beacon_silence_rssi[i]) & 0xff;
3132                 IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: i=%d, noise=%d\n",
3133                     __func__,
3134                     i,
3135                     noise);
3136
3137                 if (noise) {
3138                         total += noise;
3139                         nbant++;
3140                 }
3141         }
3142
3143         IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: nbant=%d, total=%d\n",
3144             __func__, nbant, total);
3145 #if 0
3146         /* There should be at least one antenna but check anyway. */
3147         return (nbant == 0) ? -127 : (total / nbant) - 107;
3148 #else
3149         /* For now, just hard-code it to -96 to be safe */
3150         return (-96);
3151 #endif
3152 }
3153
3154 /*
3155  * iwm_mvm_rx_rx_mpdu - IWM_REPLY_RX_MPDU_CMD handler
3156  *
3157  * Handles the actual data of the Rx packet from the fw
3158  */
3159 static void
3160 iwm_mvm_rx_rx_mpdu(struct iwm_softc *sc, struct mbuf *m)
3161 {
3162         struct ieee80211com *ic = &sc->sc_ic;
3163         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
3164         struct ieee80211_frame *wh;
3165         struct ieee80211_node *ni;
3166         struct ieee80211_rx_stats rxs;
3167         struct iwm_rx_phy_info *phy_info;
3168         struct iwm_rx_mpdu_res_start *rx_res;
3169         struct iwm_rx_packet *pkt = mtod(m, struct iwm_rx_packet *);
3170         uint32_t len;
3171         uint32_t rx_pkt_status;
3172         int rssi;
3173
3174         phy_info = &sc->sc_last_phy_info;
3175         rx_res = (struct iwm_rx_mpdu_res_start *)pkt->data;
3176         wh = (struct ieee80211_frame *)(pkt->data + sizeof(*rx_res));
3177         len = le16toh(rx_res->byte_count);
3178         rx_pkt_status = le32toh(*(uint32_t *)(pkt->data + sizeof(*rx_res) + len));
3179
3180         if (__predict_false(phy_info->cfg_phy_cnt > 20)) {
3181                 device_printf(sc->sc_dev,
3182                     "dsp size out of range [0,20]: %d\n",
3183                     phy_info->cfg_phy_cnt);
3184                 goto fail;
3185         }
3186
3187         if (!(rx_pkt_status & IWM_RX_MPDU_RES_STATUS_CRC_OK) ||
3188             !(rx_pkt_status & IWM_RX_MPDU_RES_STATUS_OVERRUN_OK)) {
3189                 IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3190                     "Bad CRC or FIFO: 0x%08X.\n", rx_pkt_status);
3191                 goto fail;
3192         }
3193
3194         rssi = iwm_mvm_get_signal_strength(sc, phy_info);
3195
3196         /* Map it to relative value */
3197         rssi = rssi - sc->sc_noise;
3198
3199         /* replenish ring for the buffer we're going to feed to the sharks */
3200         if (iwm_rx_addbuf(sc, IWM_RBUF_SIZE, sc->rxq.cur) != 0) {
3201                 device_printf(sc->sc_dev, "%s: unable to add more buffers\n",
3202                     __func__);
3203                 goto fail;
3204         }
3205
3206         m->m_data = pkt->data + sizeof(*rx_res);
3207         m->m_pkthdr.len = m->m_len = len;
3208
3209         IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3210             "%s: rssi=%d, noise=%d\n", __func__, rssi, sc->sc_noise);
3211
3212         ni = ieee80211_find_rxnode(ic, (struct ieee80211_frame_min *)wh);
3213
3214         IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3215             "%s: phy_info: channel=%d, flags=0x%08x\n",
3216             __func__,
3217             le16toh(phy_info->channel),
3218             le16toh(phy_info->phy_flags));
3219
3220         /*
3221          * Populate an RX state struct with the provided information.
3222          */
3223         bzero(&rxs, sizeof(rxs));
3224         rxs.r_flags |= IEEE80211_R_IEEE | IEEE80211_R_FREQ;
3225         rxs.r_flags |= IEEE80211_R_NF | IEEE80211_R_RSSI;
3226         rxs.c_ieee = le16toh(phy_info->channel);
3227         if (le16toh(phy_info->phy_flags & IWM_RX_RES_PHY_FLAGS_BAND_24)) {
3228                 rxs.c_freq = ieee80211_ieee2mhz(rxs.c_ieee, IEEE80211_CHAN_2GHZ);
3229         } else {
3230                 rxs.c_freq = ieee80211_ieee2mhz(rxs.c_ieee, IEEE80211_CHAN_5GHZ);
3231         }
3232
3233         /* rssi is in 1/2db units */
3234         rxs.c_rssi = rssi * 2;
3235         rxs.c_nf = sc->sc_noise;
3236         if (ieee80211_add_rx_params(m, &rxs) == 0) {
3237                 if (ni)
3238                         ieee80211_free_node(ni);
3239                 goto fail;
3240         }
3241
3242         if (ieee80211_radiotap_active_vap(vap)) {
3243                 struct iwm_rx_radiotap_header *tap = &sc->sc_rxtap;
3244
3245                 tap->wr_flags = 0;
3246                 if (phy_info->phy_flags & htole16(IWM_PHY_INFO_FLAG_SHPREAMBLE))
3247                         tap->wr_flags |= IEEE80211_RADIOTAP_F_SHORTPRE;
3248                 tap->wr_chan_freq = htole16(rxs.c_freq);
3249                 /* XXX only if ic->ic_curchan->ic_ieee == rxs.c_ieee */
3250                 tap->wr_chan_flags = htole16(ic->ic_curchan->ic_flags);
3251                 tap->wr_dbm_antsignal = (int8_t)rssi;
3252                 tap->wr_dbm_antnoise = (int8_t)sc->sc_noise;
3253                 tap->wr_tsft = phy_info->system_timestamp;
3254                 switch (phy_info->rate) {
3255                 /* CCK rates. */
3256                 case  10: tap->wr_rate =   2; break;
3257                 case  20: tap->wr_rate =   4; break;
3258                 case  55: tap->wr_rate =  11; break;
3259                 case 110: tap->wr_rate =  22; break;
3260                 /* OFDM rates. */
3261                 case 0xd: tap->wr_rate =  12; break;
3262                 case 0xf: tap->wr_rate =  18; break;
3263                 case 0x5: tap->wr_rate =  24; break;
3264                 case 0x7: tap->wr_rate =  36; break;
3265                 case 0x9: tap->wr_rate =  48; break;
3266                 case 0xb: tap->wr_rate =  72; break;
3267                 case 0x1: tap->wr_rate =  96; break;
3268                 case 0x3: tap->wr_rate = 108; break;
3269                 /* Unknown rate: should not happen. */
3270                 default:  tap->wr_rate =   0;
3271                 }
3272         }
3273
3274         IWM_UNLOCK(sc);
3275         if (ni != NULL) {
3276                 IWM_DPRINTF(sc, IWM_DEBUG_RECV, "input m %p\n", m);
3277                 ieee80211_input_mimo(ni, m);
3278                 ieee80211_free_node(ni);
3279         } else {
3280                 IWM_DPRINTF(sc, IWM_DEBUG_RECV, "inputall m %p\n", m);
3281                 ieee80211_input_mimo_all(ic, m);
3282         }
3283         IWM_LOCK(sc);
3284
3285         return;
3286
3287 fail:
3288         counter_u64_add(ic->ic_ierrors, 1);
3289 }
3290
3291 static int
3292 iwm_mvm_rx_tx_cmd_single(struct iwm_softc *sc, struct iwm_rx_packet *pkt,
3293         struct iwm_node *in)
3294 {
3295         struct iwm_mvm_tx_resp *tx_resp = (void *)pkt->data;
3296         struct ieee80211_ratectl_tx_status *txs = &sc->sc_txs;
3297         struct ieee80211_node *ni = &in->in_ni;
3298         int status = le16toh(tx_resp->status.status) & IWM_TX_STATUS_MSK;
3299
3300         KASSERT(tx_resp->frame_count == 1, ("too many frames"));
3301
3302         /* Update rate control statistics. */
3303         IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "%s: status=0x%04x, seq=%d, fc=%d, btc=%d, frts=%d, ff=%d, irate=%08x, wmt=%d\n",
3304             __func__,
3305             (int) le16toh(tx_resp->status.status),
3306             (int) le16toh(tx_resp->status.sequence),
3307             tx_resp->frame_count,
3308             tx_resp->bt_kill_count,
3309             tx_resp->failure_rts,
3310             tx_resp->failure_frame,
3311             le32toh(tx_resp->initial_rate),
3312             (int) le16toh(tx_resp->wireless_media_time));
3313
3314         txs->flags = IEEE80211_RATECTL_STATUS_SHORT_RETRY |
3315                      IEEE80211_RATECTL_STATUS_LONG_RETRY;
3316         txs->short_retries = tx_resp->failure_rts;
3317         txs->long_retries = tx_resp->failure_frame;
3318         if (status != IWM_TX_STATUS_SUCCESS &&
3319             status != IWM_TX_STATUS_DIRECT_DONE) {
3320                 switch (status) {
3321                 case IWM_TX_STATUS_FAIL_SHORT_LIMIT:
3322                         txs->status = IEEE80211_RATECTL_TX_FAIL_SHORT;
3323                         break;
3324                 case IWM_TX_STATUS_FAIL_LONG_LIMIT:
3325                         txs->status = IEEE80211_RATECTL_TX_FAIL_LONG;
3326                         break;
3327                 case IWM_TX_STATUS_FAIL_LIFE_EXPIRE:
3328                         txs->status = IEEE80211_RATECTL_TX_FAIL_EXPIRED;
3329                         break;
3330                 default:
3331                         txs->status = IEEE80211_RATECTL_TX_FAIL_UNSPECIFIED;
3332                         break;
3333                 }
3334         } else {
3335                 txs->status = IEEE80211_RATECTL_TX_SUCCESS;
3336         }
3337         ieee80211_ratectl_tx_complete(ni, txs);
3338
3339         return (txs->status != IEEE80211_RATECTL_TX_SUCCESS);
3340 }
3341
3342 static void
3343 iwm_mvm_rx_tx_cmd(struct iwm_softc *sc,
3344         struct iwm_rx_packet *pkt, struct iwm_rx_data *data)
3345 {
3346         struct iwm_cmd_header *cmd_hdr = &pkt->hdr;
3347         int idx = cmd_hdr->idx;
3348         int qid = cmd_hdr->qid;
3349         struct iwm_tx_ring *ring = &sc->txq[qid];
3350         struct iwm_tx_data *txd = &ring->data[idx];
3351         struct iwm_node *in = txd->in;
3352         struct mbuf *m = txd->m;
3353         int status;
3354
3355         KASSERT(txd->done == 0, ("txd not done"));
3356         KASSERT(txd->in != NULL, ("txd without node"));
3357         KASSERT(txd->m != NULL, ("txd without mbuf"));
3358
3359         bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTREAD);
3360
3361         sc->sc_tx_timer = 0;
3362
3363         status = iwm_mvm_rx_tx_cmd_single(sc, pkt, in);
3364
3365         /* Unmap and free mbuf. */
3366         bus_dmamap_sync(ring->data_dmat, txd->map, BUS_DMASYNC_POSTWRITE);
3367         bus_dmamap_unload(ring->data_dmat, txd->map);
3368
3369         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3370             "free txd %p, in %p\n", txd, txd->in);
3371         txd->done = 1;
3372         txd->m = NULL;
3373         txd->in = NULL;
3374
3375         ieee80211_tx_complete(&in->in_ni, m, status);
3376
3377         if (--ring->queued < IWM_TX_RING_LOMARK) {
3378                 sc->qfullmsk &= ~(1 << ring->qid);
3379                 if (sc->qfullmsk == 0) {
3380                         iwm_start(sc);
3381                 }
3382         }
3383 }
3384
3385 /*
3386  * transmit side
3387  */
3388
3389 /*
3390  * Process a "command done" firmware notification.  This is where we wakeup
3391  * processes waiting for a synchronous command completion.
3392  * from if_iwn
3393  */
3394 static void
3395 iwm_cmd_done(struct iwm_softc *sc, struct iwm_rx_packet *pkt)
3396 {
3397         struct iwm_tx_ring *ring = &sc->txq[IWM_MVM_CMD_QUEUE];
3398         struct iwm_tx_data *data;
3399
3400         if (pkt->hdr.qid != IWM_MVM_CMD_QUEUE) {
3401                 return; /* Not a command ack. */
3402         }
3403
3404         /* XXX wide commands? */
3405         IWM_DPRINTF(sc, IWM_DEBUG_CMD,
3406             "cmd notification type 0x%x qid %d idx %d\n",
3407             pkt->hdr.code, pkt->hdr.qid, pkt->hdr.idx);
3408
3409         data = &ring->data[pkt->hdr.idx];
3410
3411         /* If the command was mapped in an mbuf, free it. */
3412         if (data->m != NULL) {
3413                 bus_dmamap_sync(ring->data_dmat, data->map,
3414                     BUS_DMASYNC_POSTWRITE);
3415                 bus_dmamap_unload(ring->data_dmat, data->map);
3416                 m_freem(data->m);
3417                 data->m = NULL;
3418         }
3419         wakeup(&ring->desc[pkt->hdr.idx]);
3420
3421         if (((pkt->hdr.idx + ring->queued) % IWM_TX_RING_COUNT) != ring->cur) {
3422                 device_printf(sc->sc_dev,
3423                     "%s: Some HCMDs skipped?: idx=%d queued=%d cur=%d\n",
3424                     __func__, pkt->hdr.idx, ring->queued, ring->cur);
3425                 /* XXX call iwm_force_nmi() */
3426         }
3427
3428         KASSERT(ring->queued > 0, ("ring->queued is empty?"));
3429         ring->queued--;
3430         if (ring->queued == 0)
3431                 iwm_pcie_clear_cmd_in_flight(sc);
3432 }
3433
3434 #if 0
3435 /*
3436  * necessary only for block ack mode
3437  */
3438 void
3439 iwm_update_sched(struct iwm_softc *sc, int qid, int idx, uint8_t sta_id,
3440         uint16_t len)
3441 {
3442         struct iwm_agn_scd_bc_tbl *scd_bc_tbl;
3443         uint16_t w_val;
3444
3445         scd_bc_tbl = sc->sched_dma.vaddr;
3446
3447         len += 8; /* magic numbers came naturally from paris */
3448         if (sc->sc_capaflags & IWM_UCODE_TLV_FLAGS_DW_BC_TABLE)
3449                 len = roundup(len, 4) / 4;
3450
3451         w_val = htole16(sta_id << 12 | len);
3452
3453         /* Update TX scheduler. */
3454         scd_bc_tbl[qid].tfd_offset[idx] = w_val;
3455         bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map,
3456             BUS_DMASYNC_PREWRITE);
3457
3458         /* I really wonder what this is ?!? */
3459         if (idx < IWM_TFD_QUEUE_SIZE_BC_DUP) {
3460                 scd_bc_tbl[qid].tfd_offset[IWM_TFD_QUEUE_SIZE_MAX + idx] = w_val;
3461                 bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map,
3462                     BUS_DMASYNC_PREWRITE);
3463         }
3464 }
3465 #endif
3466
3467 /*
3468  * Take an 802.11 (non-n) rate, find the relevant rate
3469  * table entry.  return the index into in_ridx[].
3470  *
3471  * The caller then uses that index back into in_ridx
3472  * to figure out the rate index programmed /into/
3473  * the firmware for this given node.
3474  */
3475 static int
3476 iwm_tx_rateidx_lookup(struct iwm_softc *sc, struct iwm_node *in,
3477     uint8_t rate)
3478 {
3479         int i;
3480         uint8_t r;
3481
3482         for (i = 0; i < nitems(in->in_ridx); i++) {
3483                 r = iwm_rates[in->in_ridx[i]].rate;
3484                 if (rate == r)
3485                         return (i);
3486         }
3487
3488         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE,
3489             "%s: couldn't find an entry for rate=%d\n",
3490             __func__,
3491             rate);
3492
3493         /* XXX Return the first */
3494         /* XXX TODO: have it return the /lowest/ */
3495         return (0);
3496 }
3497
3498 static int
3499 iwm_tx_rateidx_global_lookup(struct iwm_softc *sc, uint8_t rate)
3500 {
3501         int i;
3502
3503         for (i = 0; i < nitems(iwm_rates); i++) {
3504                 if (iwm_rates[i].rate == rate)
3505                         return (i);
3506         }
3507         /* XXX error? */
3508         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE,
3509             "%s: couldn't find an entry for rate=%d\n",
3510             __func__,
3511             rate);
3512         return (0);
3513 }
3514
3515 /*
3516  * Fill in the rate related information for a transmit command.
3517  */
3518 static const struct iwm_rate *
3519 iwm_tx_fill_cmd(struct iwm_softc *sc, struct iwm_node *in,
3520         struct mbuf *m, struct iwm_tx_cmd *tx)
3521 {
3522         struct ieee80211_node *ni = &in->in_ni;
3523         struct ieee80211_frame *wh;
3524         const struct ieee80211_txparam *tp = ni->ni_txparms;
3525         const struct iwm_rate *rinfo;
3526         int type;
3527         int ridx, rate_flags;
3528
3529         wh = mtod(m, struct ieee80211_frame *);
3530         type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK;
3531
3532         tx->rts_retry_limit = IWM_RTS_DFAULT_RETRY_LIMIT;
3533         tx->data_retry_limit = IWM_DEFAULT_TX_RETRY;
3534
3535         if (type == IEEE80211_FC0_TYPE_MGT ||
3536             type == IEEE80211_FC0_TYPE_CTL ||
3537             (m->m_flags & M_EAPOL) != 0) {
3538                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->mgmtrate);
3539                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3540                     "%s: MGT (%d)\n", __func__, tp->mgmtrate);
3541         } else if (IEEE80211_IS_MULTICAST(wh->i_addr1)) {
3542                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->mcastrate);
3543                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3544                     "%s: MCAST (%d)\n", __func__, tp->mcastrate);
3545         } else if (tp->ucastrate != IEEE80211_FIXED_RATE_NONE) {
3546                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->ucastrate);
3547                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3548                     "%s: FIXED_RATE (%d)\n", __func__, tp->ucastrate);
3549         } else {
3550                 int i;
3551
3552                 /* for data frames, use RS table */
3553                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: DATA\n", __func__);
3554                 /* XXX pass pktlen */
3555                 (void) ieee80211_ratectl_rate(ni, NULL, 0);
3556                 i = iwm_tx_rateidx_lookup(sc, in, ni->ni_txrate);
3557                 ridx = in->in_ridx[i];
3558
3559                 /* This is the index into the programmed table */
3560                 tx->initial_rate_index = i;
3561                 tx->tx_flags |= htole32(IWM_TX_CMD_FLG_STA_RATE);
3562
3563                 IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE,
3564                     "%s: start with i=%d, txrate %d\n",
3565                     __func__, i, iwm_rates[ridx].rate);
3566         }
3567
3568         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE,
3569             "%s: frame type=%d txrate %d\n",
3570                 __func__, type, iwm_rates[ridx].rate);
3571
3572         rinfo = &iwm_rates[ridx];
3573
3574         IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: ridx=%d; rate=%d, CCK=%d\n",
3575             __func__, ridx,
3576             rinfo->rate,
3577             !! (IWM_RIDX_IS_CCK(ridx))
3578             );
3579
3580         /* XXX TODO: hard-coded TX antenna? */
3581         rate_flags = 1 << IWM_RATE_MCS_ANT_POS;
3582         if (IWM_RIDX_IS_CCK(ridx))
3583                 rate_flags |= IWM_RATE_MCS_CCK_MSK;
3584         tx->rate_n_flags = htole32(rate_flags | rinfo->plcp);
3585
3586         return rinfo;
3587 }
3588
3589 #define TB0_SIZE 16
3590 static int
3591 iwm_tx(struct iwm_softc *sc, struct mbuf *m, struct ieee80211_node *ni, int ac)
3592 {
3593         struct ieee80211com *ic = &sc->sc_ic;
3594         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
3595         struct iwm_node *in = IWM_NODE(ni);
3596         struct iwm_tx_ring *ring;
3597         struct iwm_tx_data *data;
3598         struct iwm_tfd *desc;
3599         struct iwm_device_cmd *cmd;
3600         struct iwm_tx_cmd *tx;
3601         struct ieee80211_frame *wh;
3602         struct ieee80211_key *k = NULL;
3603         struct mbuf *m1;
3604         const struct iwm_rate *rinfo;
3605         uint32_t flags;
3606         u_int hdrlen;
3607         bus_dma_segment_t *seg, segs[IWM_MAX_SCATTER];
3608         int nsegs;
3609         uint8_t tid, type;
3610         int i, totlen, error, pad;
3611
3612         wh = mtod(m, struct ieee80211_frame *);
3613         hdrlen = ieee80211_anyhdrsize(wh);
3614         type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK;
3615         tid = 0;
3616         ring = &sc->txq[ac];
3617         desc = &ring->desc[ring->cur];
3618         memset(desc, 0, sizeof(*desc));
3619         data = &ring->data[ring->cur];
3620
3621         /* Fill out iwm_tx_cmd to send to the firmware */
3622         cmd = &ring->cmd[ring->cur];
3623         cmd->hdr.code = IWM_TX_CMD;
3624         cmd->hdr.flags = 0;
3625         cmd->hdr.qid = ring->qid;
3626         cmd->hdr.idx = ring->cur;
3627
3628         tx = (void *)cmd->data;
3629         memset(tx, 0, sizeof(*tx));
3630
3631         rinfo = iwm_tx_fill_cmd(sc, in, m, tx);
3632
3633         /* Encrypt the frame if need be. */
3634         if (wh->i_fc[1] & IEEE80211_FC1_PROTECTED) {
3635                 /* Retrieve key for TX && do software encryption. */
3636                 k = ieee80211_crypto_encap(ni, m);
3637                 if (k == NULL) {
3638                         m_freem(m);
3639                         return (ENOBUFS);
3640                 }
3641                 /* 802.11 header may have moved. */
3642                 wh = mtod(m, struct ieee80211_frame *);
3643         }
3644
3645         if (ieee80211_radiotap_active_vap(vap)) {
3646                 struct iwm_tx_radiotap_header *tap = &sc->sc_txtap;
3647
3648                 tap->wt_flags = 0;
3649                 tap->wt_chan_freq = htole16(ni->ni_chan->ic_freq);
3650                 tap->wt_chan_flags = htole16(ni->ni_chan->ic_flags);
3651                 tap->wt_rate = rinfo->rate;
3652                 if (k != NULL)
3653                         tap->wt_flags |= IEEE80211_RADIOTAP_F_WEP;
3654                 ieee80211_radiotap_tx(vap, m);
3655         }
3656
3657
3658         totlen = m->m_pkthdr.len;
3659
3660         flags = 0;
3661         if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) {
3662                 flags |= IWM_TX_CMD_FLG_ACK;
3663         }
3664
3665         if (type == IEEE80211_FC0_TYPE_DATA
3666             && (totlen + IEEE80211_CRC_LEN > vap->iv_rtsthreshold)
3667             && !IEEE80211_IS_MULTICAST(wh->i_addr1)) {
3668                 flags |= IWM_TX_CMD_FLG_PROT_REQUIRE;
3669         }
3670
3671         if (IEEE80211_IS_MULTICAST(wh->i_addr1) ||
3672             type != IEEE80211_FC0_TYPE_DATA)
3673                 tx->sta_id = sc->sc_aux_sta.sta_id;
3674         else
3675                 tx->sta_id = IWM_STATION_ID;
3676
3677         if (type == IEEE80211_FC0_TYPE_MGT) {
3678                 uint8_t subtype = wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK;
3679
3680                 if (subtype == IEEE80211_FC0_SUBTYPE_ASSOC_REQ ||
3681                     subtype == IEEE80211_FC0_SUBTYPE_REASSOC_REQ) {
3682                         tx->pm_frame_timeout = htole16(IWM_PM_FRAME_ASSOC);
3683                 } else if (subtype == IEEE80211_FC0_SUBTYPE_ACTION) {
3684                         tx->pm_frame_timeout = htole16(IWM_PM_FRAME_NONE);
3685                 } else {
3686                         tx->pm_frame_timeout = htole16(IWM_PM_FRAME_MGMT);
3687                 }
3688         } else {
3689                 tx->pm_frame_timeout = htole16(IWM_PM_FRAME_NONE);
3690         }
3691
3692         if (hdrlen & 3) {
3693                 /* First segment length must be a multiple of 4. */
3694                 flags |= IWM_TX_CMD_FLG_MH_PAD;
3695                 pad = 4 - (hdrlen & 3);
3696         } else
3697                 pad = 0;
3698
3699         tx->driver_txop = 0;
3700         tx->next_frame_len = 0;
3701
3702         tx->len = htole16(totlen);
3703         tx->tid_tspec = tid;
3704         tx->life_time = htole32(IWM_TX_CMD_LIFE_TIME_INFINITE);
3705
3706         /* Set physical address of "scratch area". */
3707         tx->dram_lsb_ptr = htole32(data->scratch_paddr);
3708         tx->dram_msb_ptr = iwm_get_dma_hi_addr(data->scratch_paddr);
3709
3710         /* Copy 802.11 header in TX command. */
3711         memcpy(((uint8_t *)tx) + sizeof(*tx), wh, hdrlen);
3712
3713         flags |= IWM_TX_CMD_FLG_BT_DIS | IWM_TX_CMD_FLG_SEQ_CTL;
3714
3715         tx->sec_ctl = 0;
3716         tx->tx_flags |= htole32(flags);
3717
3718         /* Trim 802.11 header. */
3719         m_adj(m, hdrlen);
3720         error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m,
3721             segs, &nsegs, BUS_DMA_NOWAIT);
3722         if (error != 0) {
3723                 if (error != EFBIG) {
3724                         device_printf(sc->sc_dev, "can't map mbuf (error %d)\n",
3725                             error);
3726                         m_freem(m);
3727                         return error;
3728                 }
3729                 /* Too many DMA segments, linearize mbuf. */
3730                 m1 = m_collapse(m, M_NOWAIT, IWM_MAX_SCATTER - 2);
3731                 if (m1 == NULL) {
3732                         device_printf(sc->sc_dev,
3733                             "%s: could not defrag mbuf\n", __func__);
3734                         m_freem(m);
3735                         return (ENOBUFS);
3736                 }
3737                 m = m1;
3738
3739                 error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m,
3740                     segs, &nsegs, BUS_DMA_NOWAIT);
3741                 if (error != 0) {
3742                         device_printf(sc->sc_dev, "can't map mbuf (error %d)\n",
3743                             error);
3744                         m_freem(m);
3745                         return error;
3746                 }
3747         }
3748         data->m = m;
3749         data->in = in;
3750         data->done = 0;
3751
3752         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3753             "sending txd %p, in %p\n", data, data->in);
3754         KASSERT(data->in != NULL, ("node is NULL"));
3755
3756         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3757             "sending data: qid=%d idx=%d len=%d nsegs=%d txflags=0x%08x rate_n_flags=0x%08x rateidx=%u\n",
3758             ring->qid, ring->cur, totlen, nsegs,
3759             le32toh(tx->tx_flags),
3760             le32toh(tx->rate_n_flags),
3761             tx->initial_rate_index
3762             );
3763
3764         /* Fill TX descriptor. */
3765         desc->num_tbs = 2 + nsegs;
3766
3767         desc->tbs[0].lo = htole32(data->cmd_paddr);
3768         desc->tbs[0].hi_n_len = htole16(iwm_get_dma_hi_addr(data->cmd_paddr)) |
3769             (TB0_SIZE << 4);
3770         desc->tbs[1].lo = htole32(data->cmd_paddr + TB0_SIZE);
3771         desc->tbs[1].hi_n_len = htole16(iwm_get_dma_hi_addr(data->cmd_paddr)) |
3772             ((sizeof(struct iwm_cmd_header) + sizeof(*tx)
3773               + hdrlen + pad - TB0_SIZE) << 4);
3774
3775         /* Other DMA segments are for data payload. */
3776         for (i = 0; i < nsegs; i++) {
3777                 seg = &segs[i];
3778                 desc->tbs[i+2].lo = htole32(seg->ds_addr);
3779                 desc->tbs[i+2].hi_n_len = \
3780                     htole16(iwm_get_dma_hi_addr(seg->ds_addr))
3781                     | ((seg->ds_len) << 4);
3782         }
3783
3784         bus_dmamap_sync(ring->data_dmat, data->map,
3785             BUS_DMASYNC_PREWRITE);
3786         bus_dmamap_sync(ring->cmd_dma.tag, ring->cmd_dma.map,
3787             BUS_DMASYNC_PREWRITE);
3788         bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map,
3789             BUS_DMASYNC_PREWRITE);
3790
3791 #if 0
3792         iwm_update_sched(sc, ring->qid, ring->cur, tx->sta_id, le16toh(tx->len));
3793 #endif
3794
3795         /* Kick TX ring. */
3796         ring->cur = (ring->cur + 1) % IWM_TX_RING_COUNT;
3797         IWM_WRITE(sc, IWM_HBUS_TARG_WRPTR, ring->qid << 8 | ring->cur);
3798
3799         /* Mark TX ring as full if we reach a certain threshold. */
3800         if (++ring->queued > IWM_TX_RING_HIMARK) {
3801                 sc->qfullmsk |= 1 << ring->qid;
3802         }
3803
3804         return 0;
3805 }
3806
3807 static int
3808 iwm_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
3809     const struct ieee80211_bpf_params *params)
3810 {
3811         struct ieee80211com *ic = ni->ni_ic;
3812         struct iwm_softc *sc = ic->ic_softc;
3813         int error = 0;
3814
3815         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3816             "->%s begin\n", __func__);
3817
3818         if ((sc->sc_flags & IWM_FLAG_HW_INITED) == 0) {
3819                 m_freem(m);
3820                 IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3821                     "<-%s not RUNNING\n", __func__);
3822                 return (ENETDOWN);
3823         }
3824
3825         IWM_LOCK(sc);
3826         /* XXX fix this */
3827         if (params == NULL) {
3828                 error = iwm_tx(sc, m, ni, 0);
3829         } else {
3830                 error = iwm_tx(sc, m, ni, 0);
3831         }
3832         sc->sc_tx_timer = 5;
3833         IWM_UNLOCK(sc);
3834
3835         return (error);
3836 }
3837
3838 /*
3839  * mvm/tx.c
3840  */
3841
3842 /*
3843  * Note that there are transports that buffer frames before they reach
3844  * the firmware. This means that after flush_tx_path is called, the
3845  * queue might not be empty. The race-free way to handle this is to:
3846  * 1) set the station as draining
3847  * 2) flush the Tx path
3848  * 3) wait for the transport queues to be empty
3849  */
3850 int
3851 iwm_mvm_flush_tx_path(struct iwm_softc *sc, uint32_t tfd_msk, uint32_t flags)
3852 {
3853         int ret;
3854         struct iwm_tx_path_flush_cmd flush_cmd = {
3855                 .queues_ctl = htole32(tfd_msk),
3856                 .flush_ctl = htole16(IWM_DUMP_TX_FIFO_FLUSH),
3857         };
3858
3859         ret = iwm_mvm_send_cmd_pdu(sc, IWM_TXPATH_FLUSH, flags,
3860             sizeof(flush_cmd), &flush_cmd);
3861         if (ret)
3862                 device_printf(sc->sc_dev,
3863                     "Flushing tx queue failed: %d\n", ret);
3864         return ret;
3865 }
3866
3867 /*
3868  * BEGIN mvm/sta.c
3869  */
3870
3871 static int
3872 iwm_mvm_send_add_sta_cmd_status(struct iwm_softc *sc,
3873         struct iwm_mvm_add_sta_cmd *cmd, int *status)
3874 {
3875         return iwm_mvm_send_cmd_pdu_status(sc, IWM_ADD_STA, sizeof(*cmd),
3876             cmd, status);
3877 }
3878
3879 /* send station add/update command to firmware */
3880 static int
3881 iwm_mvm_sta_send_to_fw(struct iwm_softc *sc, struct iwm_node *in, int update)
3882 {
3883         struct iwm_mvm_add_sta_cmd add_sta_cmd;
3884         int ret;
3885         uint32_t status;
3886
3887         memset(&add_sta_cmd, 0, sizeof(add_sta_cmd));
3888
3889         add_sta_cmd.sta_id = IWM_STATION_ID;
3890         add_sta_cmd.mac_id_n_color
3891             = htole32(IWM_FW_CMD_ID_AND_COLOR(IWM_DEFAULT_MACID,
3892                 IWM_DEFAULT_COLOR));
3893         if (!update) {
3894                 int ac;
3895                 for (ac = 0; ac < WME_NUM_AC; ac++) {
3896                         add_sta_cmd.tfd_queue_msk |=
3897                             htole32(1 << iwm_mvm_ac_to_tx_fifo[ac]);
3898                 }
3899                 IEEE80211_ADDR_COPY(&add_sta_cmd.addr, in->in_ni.ni_bssid);
3900         }
3901         add_sta_cmd.add_modify = update ? 1 : 0;
3902         add_sta_cmd.station_flags_msk
3903             |= htole32(IWM_STA_FLG_FAT_EN_MSK | IWM_STA_FLG_MIMO_EN_MSK);
3904         add_sta_cmd.tid_disable_tx = htole16(0xffff);
3905         if (update)
3906                 add_sta_cmd.modify_mask |= (IWM_STA_MODIFY_TID_DISABLE_TX);
3907
3908         status = IWM_ADD_STA_SUCCESS;
3909         ret = iwm_mvm_send_add_sta_cmd_status(sc, &add_sta_cmd, &status);
3910         if (ret)
3911                 return ret;
3912
3913         switch (status) {
3914         case IWM_ADD_STA_SUCCESS:
3915                 break;
3916         default:
3917                 ret = EIO;
3918                 device_printf(sc->sc_dev, "IWM_ADD_STA failed\n");
3919                 break;
3920         }
3921
3922         return ret;
3923 }
3924
3925 static int
3926 iwm_mvm_add_sta(struct iwm_softc *sc, struct iwm_node *in)
3927 {
3928         return iwm_mvm_sta_send_to_fw(sc, in, 0);
3929 }
3930
3931 static int
3932 iwm_mvm_update_sta(struct iwm_softc *sc, struct iwm_node *in)
3933 {
3934         return iwm_mvm_sta_send_to_fw(sc, in, 1);
3935 }
3936
3937 static int
3938 iwm_mvm_add_int_sta_common(struct iwm_softc *sc, struct iwm_int_sta *sta,
3939         const uint8_t *addr, uint16_t mac_id, uint16_t color)
3940 {
3941         struct iwm_mvm_add_sta_cmd cmd;
3942         int ret;
3943         uint32_t status;
3944
3945         memset(&cmd, 0, sizeof(cmd));
3946         cmd.sta_id = sta->sta_id;
3947         cmd.mac_id_n_color = htole32(IWM_FW_CMD_ID_AND_COLOR(mac_id, color));
3948
3949         cmd.tfd_queue_msk = htole32(sta->tfd_queue_msk);
3950         cmd.tid_disable_tx = htole16(0xffff);
3951
3952         if (addr)
3953                 IEEE80211_ADDR_COPY(cmd.addr, addr);
3954
3955         ret = iwm_mvm_send_add_sta_cmd_status(sc, &cmd, &status);
3956         if (ret)
3957                 return ret;
3958
3959         switch (status) {
3960         case IWM_ADD_STA_SUCCESS:
3961                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
3962                     "%s: Internal station added.\n", __func__);
3963                 return 0;
3964         default:
3965                 device_printf(sc->sc_dev,
3966                     "%s: Add internal station failed, status=0x%x\n",
3967                     __func__, status);
3968                 ret = EIO;
3969                 break;
3970         }
3971         return ret;
3972 }
3973
3974 static int
3975 iwm_mvm_add_aux_sta(struct iwm_softc *sc)
3976 {
3977         int ret;
3978
3979         sc->sc_aux_sta.sta_id = IWM_AUX_STA_ID;
3980         sc->sc_aux_sta.tfd_queue_msk = (1 << IWM_MVM_AUX_QUEUE);
3981
3982         ret = iwm_enable_txq(sc, 0, IWM_MVM_AUX_QUEUE, IWM_MVM_TX_FIFO_MCAST);
3983         if (ret)
3984                 return ret;
3985
3986         ret = iwm_mvm_add_int_sta_common(sc,
3987             &sc->sc_aux_sta, NULL, IWM_MAC_INDEX_AUX, 0);
3988
3989         if (ret)
3990                 memset(&sc->sc_aux_sta, 0, sizeof(sc->sc_aux_sta));
3991         return ret;
3992 }
3993
3994 /*
3995  * END mvm/sta.c
3996  */
3997
3998 /*
3999  * BEGIN mvm/quota.c
4000  */
4001
4002 static int
4003 iwm_mvm_update_quotas(struct iwm_softc *sc, struct iwm_node *in)
4004 {
4005         struct iwm_time_quota_cmd cmd;
4006         int i, idx, ret, num_active_macs, quota, quota_rem;
4007         int colors[IWM_MAX_BINDINGS] = { -1, -1, -1, -1, };
4008         int n_ifs[IWM_MAX_BINDINGS] = {0, };
4009         uint16_t id;
4010
4011         memset(&cmd, 0, sizeof(cmd));
4012
4013         /* currently, PHY ID == binding ID */
4014         if (in) {
4015                 id = in->in_phyctxt->id;
4016                 KASSERT(id < IWM_MAX_BINDINGS, ("invalid id"));
4017                 colors[id] = in->in_phyctxt->color;
4018
4019                 if (1)
4020                         n_ifs[id] = 1;
4021         }
4022
4023         /*
4024          * The FW's scheduling session consists of
4025          * IWM_MVM_MAX_QUOTA fragments. Divide these fragments
4026          * equally between all the bindings that require quota
4027          */
4028         num_active_macs = 0;
4029         for (i = 0; i < IWM_MAX_BINDINGS; i++) {
4030                 cmd.quotas[i].id_and_color = htole32(IWM_FW_CTXT_INVALID);
4031                 num_active_macs += n_ifs[i];
4032         }
4033
4034         quota = 0;
4035         quota_rem = 0;
4036         if (num_active_macs) {
4037                 quota = IWM_MVM_MAX_QUOTA / num_active_macs;
4038                 quota_rem = IWM_MVM_MAX_QUOTA % num_active_macs;
4039         }
4040
4041         for (idx = 0, i = 0; i < IWM_MAX_BINDINGS; i++) {
4042                 if (colors[i] < 0)
4043                         continue;
4044
4045                 cmd.quotas[idx].id_and_color =
4046                         htole32(IWM_FW_CMD_ID_AND_COLOR(i, colors[i]));
4047
4048                 if (n_ifs[i] <= 0) {
4049                         cmd.quotas[idx].quota = htole32(0);
4050                         cmd.quotas[idx].max_duration = htole32(0);
4051                 } else {
4052                         cmd.quotas[idx].quota = htole32(quota * n_ifs[i]);
4053                         cmd.quotas[idx].max_duration = htole32(0);
4054                 }
4055                 idx++;
4056         }
4057
4058         /* Give the remainder of the session to the first binding */
4059         cmd.quotas[0].quota = htole32(le32toh(cmd.quotas[0].quota) + quota_rem);
4060
4061         ret = iwm_mvm_send_cmd_pdu(sc, IWM_TIME_QUOTA_CMD, IWM_CMD_SYNC,
4062             sizeof(cmd), &cmd);
4063         if (ret)
4064                 device_printf(sc->sc_dev,
4065                     "%s: Failed to send quota: %d\n", __func__, ret);
4066         return ret;
4067 }
4068
4069 /*
4070  * END mvm/quota.c
4071  */
4072
4073 /*
4074  * ieee80211 routines
4075  */
4076
4077 /*
4078  * Change to AUTH state in 80211 state machine.  Roughly matches what
4079  * Linux does in bss_info_changed().
4080  */
4081 static int
4082 iwm_auth(struct ieee80211vap *vap, struct iwm_softc *sc)
4083 {
4084         struct ieee80211_node *ni;
4085         struct iwm_node *in;
4086         struct iwm_vap *iv = IWM_VAP(vap);
4087         uint32_t duration;
4088         int error;
4089
4090         /*
4091          * XXX i have a feeling that the vap node is being
4092          * freed from underneath us. Grr.
4093          */
4094         ni = ieee80211_ref_node(vap->iv_bss);
4095         in = IWM_NODE(ni);
4096         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_STATE,
4097             "%s: called; vap=%p, bss ni=%p\n",
4098             __func__,
4099             vap,
4100             ni);
4101
4102         in->in_assoc = 0;
4103
4104         error = iwm_mvm_sf_config(sc, IWM_SF_FULL_ON);
4105         if (error != 0)
4106                 return error;
4107
4108         error = iwm_allow_mcast(vap, sc);
4109         if (error) {
4110                 device_printf(sc->sc_dev,
4111                     "%s: failed to set multicast\n", __func__);
4112                 goto out;
4113         }
4114
4115         /*
4116          * This is where it deviates from what Linux does.
4117          *
4118          * Linux iwlwifi doesn't reset the nic each time, nor does it
4119          * call ctxt_add() here.  Instead, it adds it during vap creation,
4120          * and always does a mac_ctx_changed().
4121          *
4122          * The openbsd port doesn't attempt to do that - it reset things
4123          * at odd states and does the add here.
4124          *
4125          * So, until the state handling is fixed (ie, we never reset
4126          * the NIC except for a firmware failure, which should drag
4127          * the NIC back to IDLE, re-setup and re-add all the mac/phy
4128          * contexts that are required), let's do a dirty hack here.
4129          */
4130         if (iv->is_uploaded) {
4131                 if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) {
4132                         device_printf(sc->sc_dev,
4133                             "%s: failed to update MAC\n", __func__);
4134                         goto out;
4135                 }
4136                 if ((error = iwm_mvm_phy_ctxt_changed(sc, &sc->sc_phyctxt[0],
4137                     in->in_ni.ni_chan, 1, 1)) != 0) {
4138                         device_printf(sc->sc_dev,
4139                             "%s: failed update phy ctxt\n", __func__);
4140                         goto out;
4141                 }
4142                 in->in_phyctxt = &sc->sc_phyctxt[0];
4143
4144                 if ((error = iwm_mvm_binding_update(sc, in)) != 0) {
4145                         device_printf(sc->sc_dev,
4146                             "%s: binding update cmd\n", __func__);
4147                         goto out;
4148                 }
4149                 if ((error = iwm_mvm_update_sta(sc, in)) != 0) {
4150                         device_printf(sc->sc_dev,
4151                             "%s: failed to update sta\n", __func__);
4152                         goto out;
4153                 }
4154         } else {
4155                 if ((error = iwm_mvm_mac_ctxt_add(sc, vap)) != 0) {
4156                         device_printf(sc->sc_dev,
4157                             "%s: failed to add MAC\n", __func__);
4158                         goto out;
4159                 }
4160                 if ((error = iwm_mvm_phy_ctxt_changed(sc, &sc->sc_phyctxt[0],
4161                     in->in_ni.ni_chan, 1, 1)) != 0) {
4162                         device_printf(sc->sc_dev,
4163                             "%s: failed add phy ctxt!\n", __func__);
4164                         error = ETIMEDOUT;
4165                         goto out;
4166                 }
4167                 in->in_phyctxt = &sc->sc_phyctxt[0];
4168
4169                 if ((error = iwm_mvm_binding_add_vif(sc, in)) != 0) {
4170                         device_printf(sc->sc_dev,
4171                             "%s: binding add cmd\n", __func__);
4172                         goto out;
4173                 }
4174                 if ((error = iwm_mvm_add_sta(sc, in)) != 0) {
4175                         device_printf(sc->sc_dev,
4176                             "%s: failed to add sta\n", __func__);
4177                         goto out;
4178                 }
4179         }
4180
4181         /*
4182          * Prevent the FW from wandering off channel during association
4183          * by "protecting" the session with a time event.
4184          */
4185         /* XXX duration is in units of TU, not MS */
4186         duration = IWM_MVM_TE_SESSION_PROTECTION_MAX_TIME_MS;
4187         iwm_mvm_protect_session(sc, in, duration, 500 /* XXX magic number */);
4188         DELAY(100);
4189
4190         error = 0;
4191 out:
4192         ieee80211_free_node(ni);
4193         return (error);
4194 }
4195
4196 static int
4197 iwm_assoc(struct ieee80211vap *vap, struct iwm_softc *sc)
4198 {
4199         struct iwm_node *in = IWM_NODE(vap->iv_bss);
4200         int error;
4201
4202         if ((error = iwm_mvm_update_sta(sc, in)) != 0) {
4203                 device_printf(sc->sc_dev,
4204                     "%s: failed to update STA\n", __func__);
4205                 return error;
4206         }
4207
4208         in->in_assoc = 1;
4209         if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) {
4210                 device_printf(sc->sc_dev,
4211                     "%s: failed to update MAC\n", __func__);
4212                 return error;
4213         }
4214
4215         return 0;
4216 }
4217
4218 static int
4219 iwm_release(struct iwm_softc *sc, struct iwm_node *in)
4220 {
4221         uint32_t tfd_msk;
4222
4223         /*
4224          * Ok, so *technically* the proper set of calls for going
4225          * from RUN back to SCAN is:
4226          *
4227          * iwm_mvm_power_mac_disable(sc, in);
4228          * iwm_mvm_mac_ctxt_changed(sc, in);
4229          * iwm_mvm_rm_sta(sc, in);
4230          * iwm_mvm_update_quotas(sc, NULL);
4231          * iwm_mvm_mac_ctxt_changed(sc, in);
4232          * iwm_mvm_binding_remove_vif(sc, in);
4233          * iwm_mvm_mac_ctxt_remove(sc, in);
4234          *
4235          * However, that freezes the device not matter which permutations
4236          * and modifications are attempted.  Obviously, this driver is missing
4237          * something since it works in the Linux driver, but figuring out what
4238          * is missing is a little more complicated.  Now, since we're going
4239          * back to nothing anyway, we'll just do a complete device reset.
4240          * Up your's, device!
4241          */
4242         /*
4243          * Just using 0xf for the queues mask is fine as long as we only
4244          * get here from RUN state.
4245          */
4246         tfd_msk = 0xf;
4247         mbufq_drain(&sc->sc_snd);
4248         iwm_mvm_flush_tx_path(sc, tfd_msk, IWM_CMD_SYNC);
4249         /*
4250          * We seem to get away with just synchronously sending the
4251          * IWM_TXPATH_FLUSH command.
4252          */
4253 //      iwm_trans_wait_tx_queue_empty(sc, tfd_msk);
4254         iwm_stop_device(sc);
4255         iwm_init_hw(sc);
4256         if (in)
4257                 in->in_assoc = 0;
4258         return 0;
4259
4260 #if 0
4261         int error;
4262
4263         iwm_mvm_power_mac_disable(sc, in);
4264
4265         if ((error = iwm_mvm_mac_ctxt_changed(sc, in)) != 0) {
4266                 device_printf(sc->sc_dev, "mac ctxt change fail 1 %d\n", error);
4267                 return error;
4268         }
4269
4270         if ((error = iwm_mvm_rm_sta(sc, in)) != 0) {
4271                 device_printf(sc->sc_dev, "sta remove fail %d\n", error);
4272                 return error;
4273         }
4274         error = iwm_mvm_rm_sta(sc, in);
4275         in->in_assoc = 0;
4276         iwm_mvm_update_quotas(sc, NULL);
4277         if ((error = iwm_mvm_mac_ctxt_changed(sc, in)) != 0) {
4278                 device_printf(sc->sc_dev, "mac ctxt change fail 2 %d\n", error);
4279                 return error;
4280         }
4281         iwm_mvm_binding_remove_vif(sc, in);
4282
4283         iwm_mvm_mac_ctxt_remove(sc, in);
4284
4285         return error;
4286 #endif
4287 }
4288
4289 static struct ieee80211_node *
4290 iwm_node_alloc(struct ieee80211vap *vap, const uint8_t mac[IEEE80211_ADDR_LEN])
4291 {
4292         return malloc(sizeof (struct iwm_node), M_80211_NODE,
4293             M_NOWAIT | M_ZERO);
4294 }
4295
4296 static void
4297 iwm_setrates(struct iwm_softc *sc, struct iwm_node *in)
4298 {
4299         struct ieee80211_node *ni = &in->in_ni;
4300         struct iwm_lq_cmd *lq = &in->in_lq;
4301         int nrates = ni->ni_rates.rs_nrates;
4302         int i, ridx, tab = 0;
4303 //      int txant = 0;
4304
4305         if (nrates > nitems(lq->rs_table)) {
4306                 device_printf(sc->sc_dev,
4307                     "%s: node supports %d rates, driver handles "
4308                     "only %zu\n", __func__, nrates, nitems(lq->rs_table));
4309                 return;
4310         }
4311         if (nrates == 0) {
4312                 device_printf(sc->sc_dev,
4313                     "%s: node supports 0 rates, odd!\n", __func__);
4314                 return;
4315         }
4316
4317         /*
4318          * XXX .. and most of iwm_node is not initialised explicitly;
4319          * it's all just 0x0 passed to the firmware.
4320          */
4321
4322         /* first figure out which rates we should support */
4323         /* XXX TODO: this isn't 11n aware /at all/ */
4324         memset(&in->in_ridx, -1, sizeof(in->in_ridx));
4325         IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
4326             "%s: nrates=%d\n", __func__, nrates);
4327
4328         /*
4329          * Loop over nrates and populate in_ridx from the highest
4330          * rate to the lowest rate.  Remember, in_ridx[] has
4331          * IEEE80211_RATE_MAXSIZE entries!
4332          */
4333         for (i = 0; i < min(nrates, IEEE80211_RATE_MAXSIZE); i++) {
4334                 int rate = ni->ni_rates.rs_rates[(nrates - 1) - i] & IEEE80211_RATE_VAL;
4335
4336                 /* Map 802.11 rate to HW rate index. */
4337                 for (ridx = 0; ridx <= IWM_RIDX_MAX; ridx++)
4338                         if (iwm_rates[ridx].rate == rate)
4339                                 break;
4340                 if (ridx > IWM_RIDX_MAX) {
4341                         device_printf(sc->sc_dev,
4342                             "%s: WARNING: device rate for %d not found!\n",
4343                             __func__, rate);
4344                 } else {
4345                         IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
4346                             "%s: rate: i: %d, rate=%d, ridx=%d\n",
4347                             __func__,
4348                             i,
4349                             rate,
4350                             ridx);
4351                         in->in_ridx[i] = ridx;
4352                 }
4353         }
4354
4355         /* then construct a lq_cmd based on those */
4356         memset(lq, 0, sizeof(*lq));
4357         lq->sta_id = IWM_STATION_ID;
4358
4359         /* For HT, always enable RTS/CTS to avoid excessive retries. */
4360         if (ni->ni_flags & IEEE80211_NODE_HT)
4361                 lq->flags |= IWM_LQ_FLAG_USE_RTS_MSK;
4362
4363         /*
4364          * are these used? (we don't do SISO or MIMO)
4365          * need to set them to non-zero, though, or we get an error.
4366          */
4367         lq->single_stream_ant_msk = 1;
4368         lq->dual_stream_ant_msk = 1;
4369
4370         /*
4371          * Build the actual rate selection table.
4372          * The lowest bits are the rates.  Additionally,
4373          * CCK needs bit 9 to be set.  The rest of the bits
4374          * we add to the table select the tx antenna
4375          * Note that we add the rates in the highest rate first
4376          * (opposite of ni_rates).
4377          */
4378         /*
4379          * XXX TODO: this should be looping over the min of nrates
4380          * and LQ_MAX_RETRY_NUM.  Sigh.
4381          */
4382         for (i = 0; i < nrates; i++) {
4383                 int nextant;
4384
4385 #if 0
4386                 if (txant == 0)
4387                         txant = iwm_mvm_get_valid_tx_ant(sc);
4388                 nextant = 1<<(ffs(txant)-1);
4389                 txant &= ~nextant;
4390 #else
4391                 nextant = iwm_mvm_get_valid_tx_ant(sc);
4392 #endif
4393                 /*
4394                  * Map the rate id into a rate index into
4395                  * our hardware table containing the
4396                  * configuration to use for this rate.
4397                  */
4398                 ridx = in->in_ridx[i];
4399                 tab = iwm_rates[ridx].plcp;
4400                 tab |= nextant << IWM_RATE_MCS_ANT_POS;
4401                 if (IWM_RIDX_IS_CCK(ridx))
4402                         tab |= IWM_RATE_MCS_CCK_MSK;
4403                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
4404                     "station rate i=%d, rate=%d, hw=%x\n",
4405                     i, iwm_rates[ridx].rate, tab);
4406                 lq->rs_table[i] = htole32(tab);
4407         }
4408         /* then fill the rest with the lowest possible rate */
4409         for (i = nrates; i < nitems(lq->rs_table); i++) {
4410                 KASSERT(tab != 0, ("invalid tab"));
4411                 lq->rs_table[i] = htole32(tab);
4412         }
4413 }
4414
4415 static int
4416 iwm_media_change(struct ifnet *ifp)
4417 {
4418         struct ieee80211vap *vap = ifp->if_softc;
4419         struct ieee80211com *ic = vap->iv_ic;
4420         struct iwm_softc *sc = ic->ic_softc;
4421         int error;
4422
4423         error = ieee80211_media_change(ifp);
4424         if (error != ENETRESET)
4425                 return error;
4426
4427         IWM_LOCK(sc);
4428         if (ic->ic_nrunning > 0) {
4429                 iwm_stop(sc);
4430                 iwm_init(sc);
4431         }
4432         IWM_UNLOCK(sc);
4433         return error;
4434 }
4435
4436
4437 static int
4438 iwm_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
4439 {
4440         struct iwm_vap *ivp = IWM_VAP(vap);
4441         struct ieee80211com *ic = vap->iv_ic;
4442         struct iwm_softc *sc = ic->ic_softc;
4443         struct iwm_node *in;
4444         int error;
4445
4446         IWM_DPRINTF(sc, IWM_DEBUG_STATE,
4447             "switching state %s -> %s\n",
4448             ieee80211_state_name[vap->iv_state],
4449             ieee80211_state_name[nstate]);
4450         IEEE80211_UNLOCK(ic);
4451         IWM_LOCK(sc);
4452
4453         if (vap->iv_state == IEEE80211_S_SCAN && nstate != vap->iv_state)
4454                 iwm_led_blink_stop(sc);
4455
4456         /* disable beacon filtering if we're hopping out of RUN */
4457         if (vap->iv_state == IEEE80211_S_RUN && nstate != vap->iv_state) {
4458                 iwm_mvm_disable_beacon_filter(sc);
4459
4460                 if (((in = IWM_NODE(vap->iv_bss)) != NULL))
4461                         in->in_assoc = 0;
4462
4463                 if (nstate == IEEE80211_S_INIT) {
4464                         IWM_UNLOCK(sc);
4465                         IEEE80211_LOCK(ic);
4466                         error = ivp->iv_newstate(vap, nstate, arg);
4467                         IEEE80211_UNLOCK(ic);
4468                         IWM_LOCK(sc);
4469                         iwm_release(sc, NULL);
4470                         IWM_UNLOCK(sc);
4471                         IEEE80211_LOCK(ic);
4472                         return error;
4473                 }
4474
4475                 /*
4476                  * It's impossible to directly go RUN->SCAN. If we iwm_release()
4477                  * above then the card will be completely reinitialized,
4478                  * so the driver must do everything necessary to bring the card
4479                  * from INIT to SCAN.
4480                  *
4481                  * Additionally, upon receiving deauth frame from AP,
4482                  * OpenBSD 802.11 stack puts the driver in IEEE80211_S_AUTH
4483                  * state. This will also fail with this driver, so bring the FSM
4484                  * from IEEE80211_S_RUN to IEEE80211_S_SCAN in this case as well.
4485                  *
4486                  * XXX TODO: fix this for FreeBSD!
4487                  */
4488                 if (nstate == IEEE80211_S_SCAN ||
4489                     nstate == IEEE80211_S_AUTH ||
4490                     nstate == IEEE80211_S_ASSOC) {
4491                         IWM_DPRINTF(sc, IWM_DEBUG_STATE,
4492                             "Force transition to INIT; MGT=%d\n", arg);
4493                         IWM_UNLOCK(sc);
4494                         IEEE80211_LOCK(ic);
4495                         /* Always pass arg as -1 since we can't Tx right now. */
4496                         /*
4497                          * XXX arg is just ignored anyway when transitioning
4498                          *     to IEEE80211_S_INIT.
4499                          */
4500                         vap->iv_newstate(vap, IEEE80211_S_INIT, -1);
4501                         IWM_DPRINTF(sc, IWM_DEBUG_STATE,
4502                             "Going INIT->SCAN\n");
4503                         nstate = IEEE80211_S_SCAN;
4504                         IEEE80211_UNLOCK(ic);
4505                         IWM_LOCK(sc);
4506                 }
4507         }
4508
4509         switch (nstate) {
4510         case IEEE80211_S_INIT:
4511                 break;
4512
4513         case IEEE80211_S_AUTH:
4514                 if ((error = iwm_auth(vap, sc)) != 0) {
4515                         device_printf(sc->sc_dev,
4516                             "%s: could not move to auth state: %d\n",
4517                             __func__, error);
4518                         break;
4519                 }
4520                 break;
4521
4522         case IEEE80211_S_ASSOC:
4523                 if ((error = iwm_assoc(vap, sc)) != 0) {
4524                         device_printf(sc->sc_dev,
4525                             "%s: failed to associate: %d\n", __func__,
4526                             error);
4527                         break;
4528                 }
4529                 break;
4530
4531         case IEEE80211_S_RUN:
4532         {
4533                 struct iwm_host_cmd cmd = {
4534                         .id = IWM_LQ_CMD,
4535                         .len = { sizeof(in->in_lq), },
4536                         .flags = IWM_CMD_SYNC,
4537                 };
4538
4539                 /* Update the association state, now we have it all */
4540                 /* (eg associd comes in at this point */
4541                 error = iwm_assoc(vap, sc);
4542                 if (error != 0) {
4543                         device_printf(sc->sc_dev,
4544                             "%s: failed to update association state: %d\n",
4545                             __func__,
4546                             error);
4547                         break;
4548                 }
4549
4550                 in = IWM_NODE(vap->iv_bss);
4551                 iwm_mvm_power_mac_update_mode(sc, in);
4552                 iwm_mvm_enable_beacon_filter(sc, in);
4553                 iwm_mvm_update_quotas(sc, in);
4554                 iwm_setrates(sc, in);
4555
4556                 cmd.data[0] = &in->in_lq;
4557                 if ((error = iwm_send_cmd(sc, &cmd)) != 0) {
4558                         device_printf(sc->sc_dev,
4559                             "%s: IWM_LQ_CMD failed\n", __func__);
4560                 }
4561
4562                 iwm_mvm_led_enable(sc);
4563                 break;
4564         }
4565
4566         default:
4567                 break;
4568         }
4569         IWM_UNLOCK(sc);
4570         IEEE80211_LOCK(ic);
4571
4572         return (ivp->iv_newstate(vap, nstate, arg));
4573 }
4574
4575 void
4576 iwm_endscan_cb(void *arg, int pending)
4577 {
4578         struct iwm_softc *sc = arg;
4579         struct ieee80211com *ic = &sc->sc_ic;
4580
4581         IWM_DPRINTF(sc, IWM_DEBUG_SCAN | IWM_DEBUG_TRACE,
4582             "%s: scan ended\n",
4583             __func__);
4584
4585         ieee80211_scan_done(TAILQ_FIRST(&ic->ic_vaps));
4586 }
4587
4588 /*
4589  * Aging and idle timeouts for the different possible scenarios
4590  * in default configuration
4591  */
4592 static const uint32_t
4593 iwm_sf_full_timeout_def[IWM_SF_NUM_SCENARIO][IWM_SF_NUM_TIMEOUT_TYPES] = {
4594         {
4595                 htole32(IWM_SF_SINGLE_UNICAST_AGING_TIMER_DEF),
4596                 htole32(IWM_SF_SINGLE_UNICAST_IDLE_TIMER_DEF)
4597         },
4598         {
4599                 htole32(IWM_SF_AGG_UNICAST_AGING_TIMER_DEF),
4600                 htole32(IWM_SF_AGG_UNICAST_IDLE_TIMER_DEF)
4601         },
4602         {
4603                 htole32(IWM_SF_MCAST_AGING_TIMER_DEF),
4604                 htole32(IWM_SF_MCAST_IDLE_TIMER_DEF)
4605         },
4606         {
4607                 htole32(IWM_SF_BA_AGING_TIMER_DEF),
4608                 htole32(IWM_SF_BA_IDLE_TIMER_DEF)
4609         },
4610         {
4611                 htole32(IWM_SF_TX_RE_AGING_TIMER_DEF),
4612                 htole32(IWM_SF_TX_RE_IDLE_TIMER_DEF)
4613         },
4614 };
4615
4616 /*
4617  * Aging and idle timeouts for the different possible scenarios
4618  * in single BSS MAC configuration.
4619  */
4620 static const uint32_t
4621 iwm_sf_full_timeout[IWM_SF_NUM_SCENARIO][IWM_SF_NUM_TIMEOUT_TYPES] = {
4622         {
4623                 htole32(IWM_SF_SINGLE_UNICAST_AGING_TIMER),
4624                 htole32(IWM_SF_SINGLE_UNICAST_IDLE_TIMER)
4625         },
4626         {
4627                 htole32(IWM_SF_AGG_UNICAST_AGING_TIMER),
4628                 htole32(IWM_SF_AGG_UNICAST_IDLE_TIMER)
4629         },
4630         {
4631                 htole32(IWM_SF_MCAST_AGING_TIMER),
4632                 htole32(IWM_SF_MCAST_IDLE_TIMER)
4633         },
4634         {
4635                 htole32(IWM_SF_BA_AGING_TIMER),
4636                 htole32(IWM_SF_BA_IDLE_TIMER)
4637         },
4638         {
4639                 htole32(IWM_SF_TX_RE_AGING_TIMER),
4640                 htole32(IWM_SF_TX_RE_IDLE_TIMER)
4641         },
4642 };
4643
4644 static void
4645 iwm_mvm_fill_sf_command(struct iwm_softc *sc, struct iwm_sf_cfg_cmd *sf_cmd,
4646     struct ieee80211_node *ni)
4647 {
4648         int i, j, watermark;
4649
4650         sf_cmd->watermark[IWM_SF_LONG_DELAY_ON] = htole32(IWM_SF_W_MARK_SCAN);
4651
4652         /*
4653          * If we are in association flow - check antenna configuration
4654          * capabilities of the AP station, and choose the watermark accordingly.
4655          */
4656         if (ni) {
4657                 if (ni->ni_flags & IEEE80211_NODE_HT) {
4658 #ifdef notyet
4659                         if (ni->ni_rxmcs[2] != 0)
4660                                 watermark = IWM_SF_W_MARK_MIMO3;
4661                         else if (ni->ni_rxmcs[1] != 0)
4662                                 watermark = IWM_SF_W_MARK_MIMO2;
4663                         else
4664 #endif
4665                                 watermark = IWM_SF_W_MARK_SISO;
4666                 } else {
4667                         watermark = IWM_SF_W_MARK_LEGACY;
4668                 }
4669         /* default watermark value for unassociated mode. */
4670         } else {
4671                 watermark = IWM_SF_W_MARK_MIMO2;
4672         }
4673         sf_cmd->watermark[IWM_SF_FULL_ON] = htole32(watermark);
4674
4675         for (i = 0; i < IWM_SF_NUM_SCENARIO; i++) {
4676                 for (j = 0; j < IWM_SF_NUM_TIMEOUT_TYPES; j++) {
4677                         sf_cmd->long_delay_timeouts[i][j] =
4678                                         htole32(IWM_SF_LONG_DELAY_AGING_TIMER);
4679                 }
4680         }
4681
4682         if (ni) {
4683                 memcpy(sf_cmd->full_on_timeouts, iwm_sf_full_timeout,
4684                        sizeof(iwm_sf_full_timeout));
4685         } else {
4686                 memcpy(sf_cmd->full_on_timeouts, iwm_sf_full_timeout_def,
4687                        sizeof(iwm_sf_full_timeout_def));
4688         }
4689 }
4690
4691 static int
4692 iwm_mvm_sf_config(struct iwm_softc *sc, enum iwm_sf_state new_state)
4693 {
4694         struct ieee80211com *ic = &sc->sc_ic;
4695         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
4696         struct iwm_sf_cfg_cmd sf_cmd = {
4697                 .state = htole32(IWM_SF_FULL_ON),
4698         };
4699         int ret = 0;
4700
4701         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000)
4702                 sf_cmd.state |= htole32(IWM_SF_CFG_DUMMY_NOTIF_OFF);
4703
4704         switch (new_state) {
4705         case IWM_SF_UNINIT:
4706         case IWM_SF_INIT_OFF:
4707                 iwm_mvm_fill_sf_command(sc, &sf_cmd, NULL);
4708                 break;
4709         case IWM_SF_FULL_ON:
4710                 iwm_mvm_fill_sf_command(sc, &sf_cmd, vap->iv_bss);
4711                 break;
4712         default:
4713                 IWM_DPRINTF(sc, IWM_DEBUG_PWRSAVE,
4714                     "Invalid state: %d. not sending Smart Fifo cmd\n",
4715                           new_state);
4716                 return EINVAL;
4717         }
4718
4719         ret = iwm_mvm_send_cmd_pdu(sc, IWM_REPLY_SF_CFG_CMD, IWM_CMD_ASYNC,
4720                                    sizeof(sf_cmd), &sf_cmd);
4721         return ret;
4722 }
4723
4724 static int
4725 iwm_send_bt_init_conf(struct iwm_softc *sc)
4726 {
4727         struct iwm_bt_coex_cmd bt_cmd;
4728
4729         bt_cmd.mode = htole32(IWM_BT_COEX_WIFI);
4730         bt_cmd.enabled_modules = htole32(IWM_BT_COEX_HIGH_BAND_RET);
4731
4732         return iwm_mvm_send_cmd_pdu(sc, IWM_BT_CONFIG, 0, sizeof(bt_cmd),
4733             &bt_cmd);
4734 }
4735
4736 static int
4737 iwm_send_update_mcc_cmd(struct iwm_softc *sc, const char *alpha2)
4738 {
4739         struct iwm_mcc_update_cmd mcc_cmd;
4740         struct iwm_host_cmd hcmd = {
4741                 .id = IWM_MCC_UPDATE_CMD,
4742                 .flags = (IWM_CMD_SYNC | IWM_CMD_WANT_SKB),
4743                 .data = { &mcc_cmd },
4744         };
4745         int ret;
4746 #ifdef IWM_DEBUG
4747         struct iwm_rx_packet *pkt;
4748         struct iwm_mcc_update_resp_v1 *mcc_resp_v1 = NULL;
4749         struct iwm_mcc_update_resp *mcc_resp;
4750         int n_channels;
4751         uint16_t mcc;
4752 #endif
4753         int resp_v2 = fw_has_capa(&sc->ucode_capa,
4754             IWM_UCODE_TLV_CAPA_LAR_SUPPORT_V2);
4755
4756         memset(&mcc_cmd, 0, sizeof(mcc_cmd));
4757         mcc_cmd.mcc = htole16(alpha2[0] << 8 | alpha2[1]);
4758         if (fw_has_api(&sc->ucode_capa, IWM_UCODE_TLV_API_WIFI_MCC_UPDATE) ||
4759             fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_LAR_MULTI_MCC))
4760                 mcc_cmd.source_id = IWM_MCC_SOURCE_GET_CURRENT;
4761         else
4762                 mcc_cmd.source_id = IWM_MCC_SOURCE_OLD_FW;
4763
4764         if (resp_v2)
4765                 hcmd.len[0] = sizeof(struct iwm_mcc_update_cmd);
4766         else
4767                 hcmd.len[0] = sizeof(struct iwm_mcc_update_cmd_v1);
4768
4769         IWM_DPRINTF(sc, IWM_DEBUG_NODE,
4770             "send MCC update to FW with '%c%c' src = %d\n",
4771             alpha2[0], alpha2[1], mcc_cmd.source_id);
4772
4773         ret = iwm_send_cmd(sc, &hcmd);
4774         if (ret)
4775                 return ret;
4776
4777 #ifdef IWM_DEBUG
4778         pkt = hcmd.resp_pkt;
4779
4780         /* Extract MCC response */
4781         if (resp_v2) {
4782                 mcc_resp = (void *)pkt->data;
4783                 mcc = mcc_resp->mcc;
4784                 n_channels =  le32toh(mcc_resp->n_channels);
4785         } else {
4786                 mcc_resp_v1 = (void *)pkt->data;
4787                 mcc = mcc_resp_v1->mcc;
4788                 n_channels =  le32toh(mcc_resp_v1->n_channels);
4789         }
4790
4791         /* W/A for a FW/NVM issue - returns 0x00 for the world domain */
4792         if (mcc == 0)
4793                 mcc = 0x3030;  /* "00" - world */
4794
4795         IWM_DPRINTF(sc, IWM_DEBUG_NODE,
4796             "regulatory domain '%c%c' (%d channels available)\n",
4797             mcc >> 8, mcc & 0xff, n_channels);
4798 #endif
4799         iwm_free_resp(sc, &hcmd);
4800
4801         return 0;
4802 }
4803
4804 static void
4805 iwm_mvm_tt_tx_backoff(struct iwm_softc *sc, uint32_t backoff)
4806 {
4807         struct iwm_host_cmd cmd = {
4808                 .id = IWM_REPLY_THERMAL_MNG_BACKOFF,
4809                 .len = { sizeof(uint32_t), },
4810                 .data = { &backoff, },
4811         };
4812
4813         if (iwm_send_cmd(sc, &cmd) != 0) {
4814                 device_printf(sc->sc_dev,
4815                     "failed to change thermal tx backoff\n");
4816         }
4817 }
4818
4819 static int
4820 iwm_init_hw(struct iwm_softc *sc)
4821 {
4822         struct ieee80211com *ic = &sc->sc_ic;
4823         int error, i, ac;
4824
4825         if ((error = iwm_start_hw(sc)) != 0) {
4826                 printf("iwm_start_hw: failed %d\n", error);
4827                 return error;
4828         }
4829
4830         if ((error = iwm_run_init_mvm_ucode(sc, 0)) != 0) {
4831                 printf("iwm_run_init_mvm_ucode: failed %d\n", error);
4832                 return error;
4833         }
4834
4835         /*
4836          * should stop and start HW since that INIT
4837          * image just loaded
4838          */
4839         iwm_stop_device(sc);
4840         if ((error = iwm_start_hw(sc)) != 0) {
4841                 device_printf(sc->sc_dev, "could not initialize hardware\n");
4842                 return error;
4843         }
4844
4845         /* omstart, this time with the regular firmware */
4846         error = iwm_mvm_load_ucode_wait_alive(sc, IWM_UCODE_REGULAR);
4847         if (error) {
4848                 device_printf(sc->sc_dev, "could not load firmware\n");
4849                 goto error;
4850         }
4851
4852         if ((error = iwm_send_bt_init_conf(sc)) != 0) {
4853                 device_printf(sc->sc_dev, "bt init conf failed\n");
4854                 goto error;
4855         }
4856
4857         error = iwm_send_tx_ant_cfg(sc, iwm_mvm_get_valid_tx_ant(sc));
4858         if (error != 0) {
4859                 device_printf(sc->sc_dev, "antenna config failed\n");
4860                 goto error;
4861         }
4862
4863         /* Send phy db control command and then phy db calibration */
4864         if ((error = iwm_send_phy_db_data(sc->sc_phy_db)) != 0)
4865                 goto error;
4866
4867         if ((error = iwm_send_phy_cfg_cmd(sc)) != 0) {
4868                 device_printf(sc->sc_dev, "phy_cfg_cmd failed\n");
4869                 goto error;
4870         }
4871
4872         /* Add auxiliary station for scanning */
4873         if ((error = iwm_mvm_add_aux_sta(sc)) != 0) {
4874                 device_printf(sc->sc_dev, "add_aux_sta failed\n");
4875                 goto error;
4876         }
4877
4878         for (i = 0; i < IWM_NUM_PHY_CTX; i++) {
4879                 /*
4880                  * The channel used here isn't relevant as it's
4881                  * going to be overwritten in the other flows.
4882                  * For now use the first channel we have.
4883                  */
4884                 if ((error = iwm_mvm_phy_ctxt_add(sc,
4885                     &sc->sc_phyctxt[i], &ic->ic_channels[1], 1, 1)) != 0)
4886                         goto error;
4887         }
4888
4889         /* Initialize tx backoffs to the minimum. */
4890         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
4891                 iwm_mvm_tt_tx_backoff(sc, 0);
4892
4893         error = iwm_mvm_power_update_device(sc);
4894         if (error)
4895                 goto error;
4896
4897         if (fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_LAR_SUPPORT)) {
4898                 if ((error = iwm_send_update_mcc_cmd(sc, "ZZ")) != 0)
4899                         goto error;
4900         }
4901
4902         if (fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_UMAC_SCAN)) {
4903                 if ((error = iwm_mvm_config_umac_scan(sc)) != 0)
4904                         goto error;
4905         }
4906
4907         /* Enable Tx queues. */
4908         for (ac = 0; ac < WME_NUM_AC; ac++) {
4909                 error = iwm_enable_txq(sc, IWM_STATION_ID, ac,
4910                     iwm_mvm_ac_to_tx_fifo[ac]);
4911                 if (error)
4912                         goto error;
4913         }
4914
4915         if ((error = iwm_mvm_disable_beacon_filter(sc)) != 0) {
4916                 device_printf(sc->sc_dev, "failed to disable beacon filter\n");
4917                 goto error;
4918         }
4919
4920         return 0;
4921
4922  error:
4923         iwm_stop_device(sc);
4924         return error;
4925 }
4926
4927 /* Allow multicast from our BSSID. */
4928 static int
4929 iwm_allow_mcast(struct ieee80211vap *vap, struct iwm_softc *sc)
4930 {
4931         struct ieee80211_node *ni = vap->iv_bss;
4932         struct iwm_mcast_filter_cmd *cmd;
4933         size_t size;
4934         int error;
4935
4936         size = roundup(sizeof(*cmd), 4);
4937         cmd = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
4938         if (cmd == NULL)
4939                 return ENOMEM;
4940         cmd->filter_own = 1;
4941         cmd->port_id = 0;
4942         cmd->count = 0;
4943         cmd->pass_all = 1;
4944         IEEE80211_ADDR_COPY(cmd->bssid, ni->ni_bssid);
4945
4946         error = iwm_mvm_send_cmd_pdu(sc, IWM_MCAST_FILTER_CMD,
4947             IWM_CMD_SYNC, size, cmd);
4948         free(cmd, M_DEVBUF);
4949
4950         return (error);
4951 }
4952
4953 /*
4954  * ifnet interfaces
4955  */
4956
4957 static void
4958 iwm_init(struct iwm_softc *sc)
4959 {
4960         int error;
4961
4962         if (sc->sc_flags & IWM_FLAG_HW_INITED) {
4963                 return;
4964         }
4965         sc->sc_generation++;
4966         sc->sc_flags &= ~IWM_FLAG_STOPPED;
4967
4968         if ((error = iwm_init_hw(sc)) != 0) {
4969                 printf("iwm_init_hw failed %d\n", error);
4970                 iwm_stop(sc);
4971                 return;
4972         }
4973
4974         /*
4975          * Ok, firmware loaded and we are jogging
4976          */
4977         sc->sc_flags |= IWM_FLAG_HW_INITED;
4978         callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog, sc);
4979 }
4980
4981 static int
4982 iwm_transmit(struct ieee80211com *ic, struct mbuf *m)
4983 {
4984         struct iwm_softc *sc;
4985         int error;
4986
4987         sc = ic->ic_softc;
4988
4989         IWM_LOCK(sc);
4990         if ((sc->sc_flags & IWM_FLAG_HW_INITED) == 0) {
4991                 IWM_UNLOCK(sc);
4992                 return (ENXIO);
4993         }
4994         error = mbufq_enqueue(&sc->sc_snd, m);
4995         if (error) {
4996                 IWM_UNLOCK(sc);
4997                 return (error);
4998         }
4999         iwm_start(sc);
5000         IWM_UNLOCK(sc);
5001         return (0);
5002 }
5003
5004 /*
5005  * Dequeue packets from sendq and call send.
5006  */
5007 static void
5008 iwm_start(struct iwm_softc *sc)
5009 {
5010         struct ieee80211_node *ni;
5011         struct mbuf *m;
5012         int ac = 0;
5013
5014         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TRACE, "->%s\n", __func__);
5015         while (sc->qfullmsk == 0 &&
5016                 (m = mbufq_dequeue(&sc->sc_snd)) != NULL) {
5017                 ni = (struct ieee80211_node *)m->m_pkthdr.rcvif;
5018                 if (iwm_tx(sc, m, ni, ac) != 0) {
5019                         if_inc_counter(ni->ni_vap->iv_ifp,
5020                             IFCOUNTER_OERRORS, 1);
5021                         ieee80211_free_node(ni);
5022                         continue;
5023                 }
5024                 sc->sc_tx_timer = 15;
5025         }
5026         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TRACE, "<-%s\n", __func__);
5027 }
5028
5029 static void
5030 iwm_stop(struct iwm_softc *sc)
5031 {
5032
5033         sc->sc_flags &= ~IWM_FLAG_HW_INITED;
5034         sc->sc_flags |= IWM_FLAG_STOPPED;
5035         sc->sc_generation++;
5036         iwm_led_blink_stop(sc);
5037         sc->sc_tx_timer = 0;
5038         iwm_stop_device(sc);
5039         sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
5040 }
5041
5042 static void
5043 iwm_watchdog(void *arg)
5044 {
5045         struct iwm_softc *sc = arg;
5046         struct ieee80211com *ic = &sc->sc_ic;
5047
5048         if (sc->sc_tx_timer > 0) {
5049                 if (--sc->sc_tx_timer == 0) {
5050                         device_printf(sc->sc_dev, "device timeout\n");
5051 #ifdef IWM_DEBUG
5052                         iwm_nic_error(sc);
5053 #endif
5054                         ieee80211_restart_all(ic);
5055                         counter_u64_add(sc->sc_ic.ic_oerrors, 1);
5056                         return;
5057                 }
5058         }
5059         callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog, sc);
5060 }
5061
5062 static void
5063 iwm_parent(struct ieee80211com *ic)
5064 {
5065         struct iwm_softc *sc = ic->ic_softc;
5066         int startall = 0;
5067
5068         IWM_LOCK(sc);
5069         if (ic->ic_nrunning > 0) {
5070                 if (!(sc->sc_flags & IWM_FLAG_HW_INITED)) {
5071                         iwm_init(sc);
5072                         startall = 1;
5073                 }
5074         } else if (sc->sc_flags & IWM_FLAG_HW_INITED)
5075                 iwm_stop(sc);
5076         IWM_UNLOCK(sc);
5077         if (startall)
5078                 ieee80211_start_all(ic);
5079 }
5080
5081 /*
5082  * The interrupt side of things
5083  */
5084
5085 /*
5086  * error dumping routines are from iwlwifi/mvm/utils.c
5087  */
5088
5089 /*
5090  * Note: This structure is read from the device with IO accesses,
5091  * and the reading already does the endian conversion. As it is
5092  * read with uint32_t-sized accesses, any members with a different size
5093  * need to be ordered correctly though!
5094  */
5095 struct iwm_error_event_table {
5096         uint32_t valid;         /* (nonzero) valid, (0) log is empty */
5097         uint32_t error_id;              /* type of error */
5098         uint32_t trm_hw_status0;        /* TRM HW status */
5099         uint32_t trm_hw_status1;        /* TRM HW status */
5100         uint32_t blink2;                /* branch link */
5101         uint32_t ilink1;                /* interrupt link */
5102         uint32_t ilink2;                /* interrupt link */
5103         uint32_t data1;         /* error-specific data */
5104         uint32_t data2;         /* error-specific data */
5105         uint32_t data3;         /* error-specific data */
5106         uint32_t bcon_time;             /* beacon timer */
5107         uint32_t tsf_low;               /* network timestamp function timer */
5108         uint32_t tsf_hi;                /* network timestamp function timer */
5109         uint32_t gp1;           /* GP1 timer register */
5110         uint32_t gp2;           /* GP2 timer register */
5111         uint32_t fw_rev_type;   /* firmware revision type */
5112         uint32_t major;         /* uCode version major */
5113         uint32_t minor;         /* uCode version minor */
5114         uint32_t hw_ver;                /* HW Silicon version */
5115         uint32_t brd_ver;               /* HW board version */
5116         uint32_t log_pc;                /* log program counter */
5117         uint32_t frame_ptr;             /* frame pointer */
5118         uint32_t stack_ptr;             /* stack pointer */
5119         uint32_t hcmd;          /* last host command header */
5120         uint32_t isr0;          /* isr status register LMPM_NIC_ISR0:
5121                                  * rxtx_flag */
5122         uint32_t isr1;          /* isr status register LMPM_NIC_ISR1:
5123                                  * host_flag */
5124         uint32_t isr2;          /* isr status register LMPM_NIC_ISR2:
5125                                  * enc_flag */
5126         uint32_t isr3;          /* isr status register LMPM_NIC_ISR3:
5127                                  * time_flag */
5128         uint32_t isr4;          /* isr status register LMPM_NIC_ISR4:
5129                                  * wico interrupt */
5130         uint32_t last_cmd_id;   /* last HCMD id handled by the firmware */
5131         uint32_t wait_event;            /* wait event() caller address */
5132         uint32_t l2p_control;   /* L2pControlField */
5133         uint32_t l2p_duration;  /* L2pDurationField */
5134         uint32_t l2p_mhvalid;   /* L2pMhValidBits */
5135         uint32_t l2p_addr_match;        /* L2pAddrMatchStat */
5136         uint32_t lmpm_pmg_sel;  /* indicate which clocks are turned on
5137                                  * (LMPM_PMG_SEL) */
5138         uint32_t u_timestamp;   /* indicate when the date and time of the
5139                                  * compilation */
5140         uint32_t flow_handler;  /* FH read/write pointers, RX credit */
5141 } __packed /* LOG_ERROR_TABLE_API_S_VER_3 */;
5142
5143 /*
5144  * UMAC error struct - relevant starting from family 8000 chip.
5145  * Note: This structure is read from the device with IO accesses,
5146  * and the reading already does the endian conversion. As it is
5147  * read with u32-sized accesses, any members with a different size
5148  * need to be ordered correctly though!
5149  */
5150 struct iwm_umac_error_event_table {
5151         uint32_t valid;         /* (nonzero) valid, (0) log is empty */
5152         uint32_t error_id;      /* type of error */
5153         uint32_t blink1;        /* branch link */
5154         uint32_t blink2;        /* branch link */
5155         uint32_t ilink1;        /* interrupt link */
5156         uint32_t ilink2;        /* interrupt link */
5157         uint32_t data1;         /* error-specific data */
5158         uint32_t data2;         /* error-specific data */
5159         uint32_t data3;         /* error-specific data */
5160         uint32_t umac_major;
5161         uint32_t umac_minor;
5162         uint32_t frame_pointer; /* core register 27*/
5163         uint32_t stack_pointer; /* core register 28 */
5164         uint32_t cmd_header;    /* latest host cmd sent to UMAC */
5165         uint32_t nic_isr_pref;  /* ISR status register */
5166 } __packed;
5167
5168 #define ERROR_START_OFFSET  (1 * sizeof(uint32_t))
5169 #define ERROR_ELEM_SIZE     (7 * sizeof(uint32_t))
5170
5171 #ifdef IWM_DEBUG
5172 struct {
5173         const char *name;
5174         uint8_t num;
5175 } advanced_lookup[] = {
5176         { "NMI_INTERRUPT_WDG", 0x34 },
5177         { "SYSASSERT", 0x35 },
5178         { "UCODE_VERSION_MISMATCH", 0x37 },
5179         { "BAD_COMMAND", 0x38 },
5180         { "NMI_INTERRUPT_DATA_ACTION_PT", 0x3C },
5181         { "FATAL_ERROR", 0x3D },
5182         { "NMI_TRM_HW_ERR", 0x46 },
5183         { "NMI_INTERRUPT_TRM", 0x4C },
5184         { "NMI_INTERRUPT_BREAK_POINT", 0x54 },
5185         { "NMI_INTERRUPT_WDG_RXF_FULL", 0x5C },
5186         { "NMI_INTERRUPT_WDG_NO_RBD_RXF_FULL", 0x64 },
5187         { "NMI_INTERRUPT_HOST", 0x66 },
5188         { "NMI_INTERRUPT_ACTION_PT", 0x7C },
5189         { "NMI_INTERRUPT_UNKNOWN", 0x84 },
5190         { "NMI_INTERRUPT_INST_ACTION_PT", 0x86 },
5191         { "ADVANCED_SYSASSERT", 0 },
5192 };
5193
5194 static const char *
5195 iwm_desc_lookup(uint32_t num)
5196 {
5197         int i;
5198
5199         for (i = 0; i < nitems(advanced_lookup) - 1; i++)
5200                 if (advanced_lookup[i].num == num)
5201                         return advanced_lookup[i].name;
5202
5203         /* No entry matches 'num', so it is the last: ADVANCED_SYSASSERT */
5204         return advanced_lookup[i].name;
5205 }
5206
5207 static void
5208 iwm_nic_umac_error(struct iwm_softc *sc)
5209 {
5210         struct iwm_umac_error_event_table table;
5211         uint32_t base;
5212
5213         base = sc->umac_error_event_table;
5214
5215         if (base < 0x800000) {
5216                 device_printf(sc->sc_dev, "Invalid error log pointer 0x%08x\n",
5217                     base);
5218                 return;
5219         }
5220
5221         if (iwm_read_mem(sc, base, &table, sizeof(table)/sizeof(uint32_t))) {
5222                 device_printf(sc->sc_dev, "reading errlog failed\n");
5223                 return;
5224         }
5225
5226         if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) {
5227                 device_printf(sc->sc_dev, "Start UMAC Error Log Dump:\n");
5228                 device_printf(sc->sc_dev, "Status: 0x%x, count: %d\n",
5229                     sc->sc_flags, table.valid);
5230         }
5231
5232         device_printf(sc->sc_dev, "0x%08X | %s\n", table.error_id,
5233                 iwm_desc_lookup(table.error_id));
5234         device_printf(sc->sc_dev, "0x%08X | umac branchlink1\n", table.blink1);
5235         device_printf(sc->sc_dev, "0x%08X | umac branchlink2\n", table.blink2);
5236         device_printf(sc->sc_dev, "0x%08X | umac interruptlink1\n",
5237             table.ilink1);
5238         device_printf(sc->sc_dev, "0x%08X | umac interruptlink2\n",
5239             table.ilink2);
5240         device_printf(sc->sc_dev, "0x%08X | umac data1\n", table.data1);
5241         device_printf(sc->sc_dev, "0x%08X | umac data2\n", table.data2);
5242         device_printf(sc->sc_dev, "0x%08X | umac data3\n", table.data3);
5243         device_printf(sc->sc_dev, "0x%08X | umac major\n", table.umac_major);
5244         device_printf(sc->sc_dev, "0x%08X | umac minor\n", table.umac_minor);
5245         device_printf(sc->sc_dev, "0x%08X | frame pointer\n",
5246             table.frame_pointer);
5247         device_printf(sc->sc_dev, "0x%08X | stack pointer\n",
5248             table.stack_pointer);
5249         device_printf(sc->sc_dev, "0x%08X | last host cmd\n", table.cmd_header);
5250         device_printf(sc->sc_dev, "0x%08X | isr status reg\n",
5251             table.nic_isr_pref);
5252 }
5253
5254 /*
5255  * Support for dumping the error log seemed like a good idea ...
5256  * but it's mostly hex junk and the only sensible thing is the
5257  * hw/ucode revision (which we know anyway).  Since it's here,
5258  * I'll just leave it in, just in case e.g. the Intel guys want to
5259  * help us decipher some "ADVANCED_SYSASSERT" later.
5260  */
5261 static void
5262 iwm_nic_error(struct iwm_softc *sc)
5263 {
5264         struct iwm_error_event_table table;
5265         uint32_t base;
5266
5267         device_printf(sc->sc_dev, "dumping device error log\n");
5268         base = sc->error_event_table;
5269         if (base < 0x800000) {
5270                 device_printf(sc->sc_dev,
5271                     "Invalid error log pointer 0x%08x\n", base);
5272                 return;
5273         }
5274
5275         if (iwm_read_mem(sc, base, &table, sizeof(table)/sizeof(uint32_t))) {
5276                 device_printf(sc->sc_dev, "reading errlog failed\n");
5277                 return;
5278         }
5279
5280         if (!table.valid) {
5281                 device_printf(sc->sc_dev, "errlog not found, skipping\n");
5282                 return;
5283         }
5284
5285         if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) {
5286                 device_printf(sc->sc_dev, "Start Error Log Dump:\n");
5287                 device_printf(sc->sc_dev, "Status: 0x%x, count: %d\n",
5288                     sc->sc_flags, table.valid);
5289         }
5290
5291         device_printf(sc->sc_dev, "0x%08X | %-28s\n", table.error_id,
5292             iwm_desc_lookup(table.error_id));
5293         device_printf(sc->sc_dev, "%08X | trm_hw_status0\n",
5294             table.trm_hw_status0);
5295         device_printf(sc->sc_dev, "%08X | trm_hw_status1\n",
5296             table.trm_hw_status1);
5297         device_printf(sc->sc_dev, "%08X | branchlink2\n", table.blink2);
5298         device_printf(sc->sc_dev, "%08X | interruptlink1\n", table.ilink1);
5299         device_printf(sc->sc_dev, "%08X | interruptlink2\n", table.ilink2);
5300         device_printf(sc->sc_dev, "%08X | data1\n", table.data1);
5301         device_printf(sc->sc_dev, "%08X | data2\n", table.data2);
5302         device_printf(sc->sc_dev, "%08X | data3\n", table.data3);
5303         device_printf(sc->sc_dev, "%08X | beacon time\n", table.bcon_time);
5304         device_printf(sc->sc_dev, "%08X | tsf low\n", table.tsf_low);
5305         device_printf(sc->sc_dev, "%08X | tsf hi\n", table.tsf_hi);
5306         device_printf(sc->sc_dev, "%08X | time gp1\n", table.gp1);
5307         device_printf(sc->sc_dev, "%08X | time gp2\n", table.gp2);
5308         device_printf(sc->sc_dev, "%08X | uCode revision type\n",
5309             table.fw_rev_type);
5310         device_printf(sc->sc_dev, "%08X | uCode version major\n", table.major);
5311         device_printf(sc->sc_dev, "%08X | uCode version minor\n", table.minor);
5312         device_printf(sc->sc_dev, "%08X | hw version\n", table.hw_ver);
5313         device_printf(sc->sc_dev, "%08X | board version\n", table.brd_ver);
5314         device_printf(sc->sc_dev, "%08X | hcmd\n", table.hcmd);
5315         device_printf(sc->sc_dev, "%08X | isr0\n", table.isr0);
5316         device_printf(sc->sc_dev, "%08X | isr1\n", table.isr1);
5317         device_printf(sc->sc_dev, "%08X | isr2\n", table.isr2);
5318         device_printf(sc->sc_dev, "%08X | isr3\n", table.isr3);
5319         device_printf(sc->sc_dev, "%08X | isr4\n", table.isr4);
5320         device_printf(sc->sc_dev, "%08X | last cmd Id\n", table.last_cmd_id);
5321         device_printf(sc->sc_dev, "%08X | wait_event\n", table.wait_event);
5322         device_printf(sc->sc_dev, "%08X | l2p_control\n", table.l2p_control);
5323         device_printf(sc->sc_dev, "%08X | l2p_duration\n", table.l2p_duration);
5324         device_printf(sc->sc_dev, "%08X | l2p_mhvalid\n", table.l2p_mhvalid);
5325         device_printf(sc->sc_dev, "%08X | l2p_addr_match\n", table.l2p_addr_match);
5326         device_printf(sc->sc_dev, "%08X | lmpm_pmg_sel\n", table.lmpm_pmg_sel);
5327         device_printf(sc->sc_dev, "%08X | timestamp\n", table.u_timestamp);
5328         device_printf(sc->sc_dev, "%08X | flow_handler\n", table.flow_handler);
5329
5330         if (sc->umac_error_event_table)
5331                 iwm_nic_umac_error(sc);
5332 }
5333 #endif
5334
5335 #define ADVANCE_RXQ(sc) (sc->rxq.cur = (sc->rxq.cur + 1) % IWM_RX_RING_COUNT);
5336
5337 /*
5338  * Process an IWM_CSR_INT_BIT_FH_RX or IWM_CSR_INT_BIT_SW_RX interrupt.
5339  * Basic structure from if_iwn
5340  */
5341 static void
5342 iwm_notif_intr(struct iwm_softc *sc)
5343 {
5344         struct ieee80211com *ic = &sc->sc_ic;
5345         uint16_t hw;
5346
5347         bus_dmamap_sync(sc->rxq.stat_dma.tag, sc->rxq.stat_dma.map,
5348             BUS_DMASYNC_POSTREAD);
5349
5350         hw = le16toh(sc->rxq.stat->closed_rb_num) & 0xfff;
5351
5352         /*
5353          * Process responses
5354          */
5355         while (sc->rxq.cur != hw) {
5356                 struct iwm_rx_ring *ring = &sc->rxq;
5357                 struct iwm_rx_data *data = &ring->data[ring->cur];
5358                 struct iwm_rx_packet *pkt;
5359                 struct iwm_cmd_response *cresp;
5360                 int qid, idx, code;
5361
5362                 bus_dmamap_sync(ring->data_dmat, data->map,
5363                     BUS_DMASYNC_POSTREAD);
5364                 pkt = mtod(data->m, struct iwm_rx_packet *);
5365
5366                 qid = pkt->hdr.qid & ~0x80;
5367                 idx = pkt->hdr.idx;
5368
5369                 code = IWM_WIDE_ID(pkt->hdr.flags, pkt->hdr.code);
5370                 IWM_DPRINTF(sc, IWM_DEBUG_INTR,
5371                     "rx packet qid=%d idx=%d type=%x %d %d\n",
5372                     pkt->hdr.qid & ~0x80, pkt->hdr.idx, code, ring->cur, hw);
5373
5374                 /*
5375                  * randomly get these from the firmware, no idea why.
5376                  * they at least seem harmless, so just ignore them for now
5377                  */
5378                 if (__predict_false((pkt->hdr.code == 0 && qid == 0 && idx == 0)
5379                     || pkt->len_n_flags == htole32(0x55550000))) {
5380                         ADVANCE_RXQ(sc);
5381                         continue;
5382                 }
5383
5384                 iwm_notification_wait_notify(sc->sc_notif_wait, code, pkt);
5385
5386                 switch (code) {
5387                 case IWM_REPLY_RX_PHY_CMD:
5388                         iwm_mvm_rx_rx_phy_cmd(sc, pkt, data);
5389                         break;
5390
5391                 case IWM_REPLY_RX_MPDU_CMD:
5392                         iwm_mvm_rx_rx_mpdu(sc, data->m);
5393                         break;
5394
5395                 case IWM_TX_CMD:
5396                         iwm_mvm_rx_tx_cmd(sc, pkt, data);
5397                         break;
5398
5399                 case IWM_MISSED_BEACONS_NOTIFICATION: {
5400                         struct iwm_missed_beacons_notif *resp;
5401                         int missed;
5402
5403                         /* XXX look at mac_id to determine interface ID */
5404                         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
5405
5406                         resp = (void *)pkt->data;
5407                         missed = le32toh(resp->consec_missed_beacons);
5408
5409                         IWM_DPRINTF(sc, IWM_DEBUG_BEACON | IWM_DEBUG_STATE,
5410                             "%s: MISSED_BEACON: mac_id=%d, "
5411                             "consec_since_last_rx=%d, consec=%d, num_expect=%d "
5412                             "num_rx=%d\n",
5413                             __func__,
5414                             le32toh(resp->mac_id),
5415                             le32toh(resp->consec_missed_beacons_since_last_rx),
5416                             le32toh(resp->consec_missed_beacons),
5417                             le32toh(resp->num_expected_beacons),
5418                             le32toh(resp->num_recvd_beacons));
5419
5420                         /* Be paranoid */
5421                         if (vap == NULL)
5422                                 break;
5423
5424                         /* XXX no net80211 locking? */
5425                         if (vap->iv_state == IEEE80211_S_RUN &&
5426                             (ic->ic_flags & IEEE80211_F_SCAN) == 0) {
5427                                 if (missed > vap->iv_bmissthreshold) {
5428                                         /* XXX bad locking; turn into task */
5429                                         IWM_UNLOCK(sc);
5430                                         ieee80211_beacon_miss(ic);
5431                                         IWM_LOCK(sc);
5432                                 }
5433                         }
5434
5435                         break;
5436                 }
5437
5438                 case IWM_MFUART_LOAD_NOTIFICATION:
5439                         break;
5440
5441                 case IWM_MVM_ALIVE:
5442                         break;
5443
5444                 case IWM_CALIB_RES_NOTIF_PHY_DB:
5445                         break;
5446
5447                 case IWM_STATISTICS_NOTIFICATION: {
5448                         struct iwm_notif_statistics *stats;
5449                         stats = (void *)pkt->data;
5450                         memcpy(&sc->sc_stats, stats, sizeof(sc->sc_stats));
5451                         sc->sc_noise = iwm_get_noise(sc, &stats->rx.general);
5452                         break;
5453                 }
5454
5455                 case IWM_NVM_ACCESS_CMD:
5456                 case IWM_MCC_UPDATE_CMD:
5457                         if (sc->sc_wantresp == ((qid << 16) | idx)) {
5458                                 memcpy(sc->sc_cmd_resp,
5459                                     pkt, sizeof(sc->sc_cmd_resp));
5460                         }
5461                         break;
5462
5463                 case IWM_MCC_CHUB_UPDATE_CMD: {
5464                         struct iwm_mcc_chub_notif *notif;
5465                         notif = (void *)pkt->data;
5466
5467                         sc->sc_fw_mcc[0] = (notif->mcc & 0xff00) >> 8;
5468                         sc->sc_fw_mcc[1] = notif->mcc & 0xff;
5469                         sc->sc_fw_mcc[2] = '\0';
5470                         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
5471                             "fw source %d sent CC '%s'\n",
5472                             notif->source_id, sc->sc_fw_mcc);
5473                         break;
5474                 }
5475
5476                 case IWM_DTS_MEASUREMENT_NOTIFICATION:
5477                 case IWM_WIDE_ID(IWM_PHY_OPS_GROUP,
5478                                  IWM_DTS_MEASUREMENT_NOTIF_WIDE): {
5479                         struct iwm_dts_measurement_notif_v1 *notif;
5480
5481                         if (iwm_rx_packet_payload_len(pkt) < sizeof(*notif)) {
5482                                 device_printf(sc->sc_dev,
5483                                     "Invalid DTS_MEASUREMENT_NOTIFICATION\n");
5484                                 break;
5485                         }
5486                         notif = (void *)pkt->data;
5487                         IWM_DPRINTF(sc, IWM_DEBUG_TEMP,
5488                             "IWM_DTS_MEASUREMENT_NOTIFICATION - %d\n",
5489                             notif->temp);
5490                         break;
5491                 }
5492
5493                 case IWM_PHY_CONFIGURATION_CMD:
5494                 case IWM_TX_ANT_CONFIGURATION_CMD:
5495                 case IWM_ADD_STA:
5496                 case IWM_MAC_CONTEXT_CMD:
5497                 case IWM_REPLY_SF_CFG_CMD:
5498                 case IWM_POWER_TABLE_CMD:
5499                 case IWM_PHY_CONTEXT_CMD:
5500                 case IWM_BINDING_CONTEXT_CMD:
5501                 case IWM_TIME_EVENT_CMD:
5502                 case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_CFG_CMD):
5503                 case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_REQ_UMAC):
5504                 case IWM_SCAN_ABORT_UMAC:
5505                 case IWM_SCAN_OFFLOAD_REQUEST_CMD:
5506                 case IWM_SCAN_OFFLOAD_ABORT_CMD:
5507                 case IWM_REPLY_BEACON_FILTERING_CMD:
5508                 case IWM_MAC_PM_POWER_TABLE:
5509                 case IWM_TIME_QUOTA_CMD:
5510                 case IWM_REMOVE_STA:
5511                 case IWM_TXPATH_FLUSH:
5512                 case IWM_LQ_CMD:
5513                 case IWM_FW_PAGING_BLOCK_CMD:
5514                 case IWM_BT_CONFIG:
5515                 case IWM_REPLY_THERMAL_MNG_BACKOFF:
5516                         cresp = (void *)pkt->data;
5517                         if (sc->sc_wantresp == ((qid << 16) | idx)) {
5518                                 memcpy(sc->sc_cmd_resp,
5519                                     pkt, sizeof(*pkt)+sizeof(*cresp));
5520                         }
5521                         break;
5522
5523                 /* ignore */
5524                 case 0x6c: /* IWM_PHY_DB_CMD, no idea why it's not in fw-api.h */
5525                         break;
5526
5527                 case IWM_INIT_COMPLETE_NOTIF:
5528                         break;
5529
5530                 case IWM_SCAN_OFFLOAD_COMPLETE: {
5531                         struct iwm_periodic_scan_complete *notif;
5532                         notif = (void *)pkt->data;
5533                         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
5534                                 sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
5535                                 ieee80211_runtask(ic, &sc->sc_es_task);
5536                         }
5537                         break;
5538                 }
5539
5540                 case IWM_SCAN_ITERATION_COMPLETE: {
5541                         struct iwm_lmac_scan_complete_notif *notif;
5542                         notif = (void *)pkt->data;
5543                         ieee80211_runtask(&sc->sc_ic, &sc->sc_es_task);
5544                         break;
5545                 }
5546
5547                 case IWM_SCAN_COMPLETE_UMAC: {
5548                         struct iwm_umac_scan_complete *notif;
5549                         notif = (void *)pkt->data;
5550
5551                         IWM_DPRINTF(sc, IWM_DEBUG_SCAN,
5552                             "UMAC scan complete, status=0x%x\n",
5553                             notif->status);
5554                         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
5555                                 sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
5556                                 ieee80211_runtask(ic, &sc->sc_es_task);
5557                         }
5558                         break;
5559                 }
5560
5561                 case IWM_SCAN_ITERATION_COMPLETE_UMAC: {
5562                         struct iwm_umac_scan_iter_complete_notif *notif;
5563                         notif = (void *)pkt->data;
5564
5565                         IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "UMAC scan iteration "
5566                             "complete, status=0x%x, %d channels scanned\n",
5567                             notif->status, notif->scanned_channels);
5568                         ieee80211_runtask(&sc->sc_ic, &sc->sc_es_task);
5569                         break;
5570                 }
5571
5572                 case IWM_REPLY_ERROR: {
5573                         struct iwm_error_resp *resp;
5574                         resp = (void *)pkt->data;
5575
5576                         device_printf(sc->sc_dev,
5577                             "firmware error 0x%x, cmd 0x%x\n",
5578                             le32toh(resp->error_type),
5579                             resp->cmd_id);
5580                         break;
5581                 }
5582
5583                 case IWM_TIME_EVENT_NOTIFICATION: {
5584                         struct iwm_time_event_notif *notif;
5585                         notif = (void *)pkt->data;
5586
5587                         IWM_DPRINTF(sc, IWM_DEBUG_INTR,
5588                             "TE notif status = 0x%x action = 0x%x\n",
5589                             notif->status, notif->action);
5590                         break;
5591                 }
5592
5593                 case IWM_MCAST_FILTER_CMD:
5594                         break;
5595
5596                 case IWM_SCD_QUEUE_CFG: {
5597                         struct iwm_scd_txq_cfg_rsp *rsp;
5598                         rsp = (void *)pkt->data;
5599
5600                         IWM_DPRINTF(sc, IWM_DEBUG_CMD,
5601                             "queue cfg token=0x%x sta_id=%d "
5602                             "tid=%d scd_queue=%d\n",
5603                             rsp->token, rsp->sta_id, rsp->tid,
5604                             rsp->scd_queue);
5605                         break;
5606                 }
5607
5608                 default:
5609                         device_printf(sc->sc_dev,
5610                             "frame %d/%d %x UNHANDLED (this should "
5611                             "not happen)\n", qid, idx,
5612                             pkt->len_n_flags);
5613                         break;
5614                 }
5615
5616                 /*
5617                  * Why test bit 0x80?  The Linux driver:
5618                  *
5619                  * There is one exception:  uCode sets bit 15 when it
5620                  * originates the response/notification, i.e. when the
5621                  * response/notification is not a direct response to a
5622                  * command sent by the driver.  For example, uCode issues
5623                  * IWM_REPLY_RX when it sends a received frame to the driver;
5624                  * it is not a direct response to any driver command.
5625                  *
5626                  * Ok, so since when is 7 == 15?  Well, the Linux driver
5627                  * uses a slightly different format for pkt->hdr, and "qid"
5628                  * is actually the upper byte of a two-byte field.
5629                  */
5630                 if (!(pkt->hdr.qid & (1 << 7))) {
5631                         iwm_cmd_done(sc, pkt);
5632                 }
5633
5634                 ADVANCE_RXQ(sc);
5635         }
5636
5637         /*
5638          * Tell the firmware what we have processed.
5639          * Seems like the hardware gets upset unless we align
5640          * the write by 8??
5641          */
5642         hw = (hw == 0) ? IWM_RX_RING_COUNT - 1 : hw - 1;
5643         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_WPTR, hw & ~7);
5644 }
5645
5646 static void
5647 iwm_intr(void *arg)
5648 {
5649         struct iwm_softc *sc = arg;
5650         int handled = 0;
5651         int r1, r2, rv = 0;
5652         int isperiodic = 0;
5653
5654         IWM_LOCK(sc);
5655         IWM_WRITE(sc, IWM_CSR_INT_MASK, 0);
5656
5657         if (sc->sc_flags & IWM_FLAG_USE_ICT) {
5658                 uint32_t *ict = sc->ict_dma.vaddr;
5659                 int tmp;
5660
5661                 tmp = htole32(ict[sc->ict_cur]);
5662                 if (!tmp)
5663                         goto out_ena;
5664
5665                 /*
5666                  * ok, there was something.  keep plowing until we have all.
5667                  */
5668                 r1 = r2 = 0;
5669                 while (tmp) {
5670                         r1 |= tmp;
5671                         ict[sc->ict_cur] = 0;
5672                         sc->ict_cur = (sc->ict_cur+1) % IWM_ICT_COUNT;
5673                         tmp = htole32(ict[sc->ict_cur]);
5674                 }
5675
5676                 /* this is where the fun begins.  don't ask */
5677                 if (r1 == 0xffffffff)
5678                         r1 = 0;
5679
5680                 /* i am not expected to understand this */
5681                 if (r1 & 0xc0000)
5682                         r1 |= 0x8000;
5683                 r1 = (0xff & r1) | ((0xff00 & r1) << 16);
5684         } else {
5685                 r1 = IWM_READ(sc, IWM_CSR_INT);
5686                 /* "hardware gone" (where, fishing?) */
5687                 if (r1 == 0xffffffff || (r1 & 0xfffffff0) == 0xa5a5a5a0)
5688                         goto out;
5689                 r2 = IWM_READ(sc, IWM_CSR_FH_INT_STATUS);
5690         }
5691         if (r1 == 0 && r2 == 0) {
5692                 goto out_ena;
5693         }
5694
5695         IWM_WRITE(sc, IWM_CSR_INT, r1 | ~sc->sc_intmask);
5696
5697         /* Safely ignore these bits for debug checks below */
5698         r1 &= ~(IWM_CSR_INT_BIT_ALIVE | IWM_CSR_INT_BIT_SCD);
5699
5700         if (r1 & IWM_CSR_INT_BIT_SW_ERR) {
5701                 int i;
5702                 struct ieee80211com *ic = &sc->sc_ic;
5703                 struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
5704
5705 #ifdef IWM_DEBUG
5706                 iwm_nic_error(sc);
5707 #endif
5708                 /* Dump driver status (TX and RX rings) while we're here. */
5709                 device_printf(sc->sc_dev, "driver status:\n");
5710                 for (i = 0; i < IWM_MVM_MAX_QUEUES; i++) {
5711                         struct iwm_tx_ring *ring = &sc->txq[i];
5712                         device_printf(sc->sc_dev,
5713                             "  tx ring %2d: qid=%-2d cur=%-3d "
5714                             "queued=%-3d\n",
5715                             i, ring->qid, ring->cur, ring->queued);
5716                 }
5717                 device_printf(sc->sc_dev,
5718                     "  rx ring: cur=%d\n", sc->rxq.cur);
5719                 device_printf(sc->sc_dev,
5720                     "  802.11 state %d\n", (vap == NULL) ? -1 : vap->iv_state);
5721
5722                 /* Don't stop the device; just do a VAP restart */
5723                 IWM_UNLOCK(sc);
5724
5725                 if (vap == NULL) {
5726                         printf("%s: null vap\n", __func__);
5727                         return;
5728                 }
5729
5730                 device_printf(sc->sc_dev, "%s: controller panicked, iv_state = %d; "
5731                     "restarting\n", __func__, vap->iv_state);
5732
5733                 /* XXX TODO: turn this into a callout/taskqueue */
5734                 ieee80211_restart_all(ic);
5735                 return;
5736         }
5737
5738         if (r1 & IWM_CSR_INT_BIT_HW_ERR) {
5739                 handled |= IWM_CSR_INT_BIT_HW_ERR;
5740                 device_printf(sc->sc_dev, "hardware error, stopping device\n");
5741                 iwm_stop(sc);
5742                 rv = 1;
5743                 goto out;
5744         }
5745
5746         /* firmware chunk loaded */
5747         if (r1 & IWM_CSR_INT_BIT_FH_TX) {
5748                 IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, IWM_CSR_FH_INT_TX_MASK);
5749                 handled |= IWM_CSR_INT_BIT_FH_TX;
5750                 sc->sc_fw_chunk_done = 1;
5751                 wakeup(&sc->sc_fw);
5752         }
5753
5754         if (r1 & IWM_CSR_INT_BIT_RF_KILL) {
5755                 handled |= IWM_CSR_INT_BIT_RF_KILL;
5756                 if (iwm_check_rfkill(sc)) {
5757                         device_printf(sc->sc_dev,
5758                             "%s: rfkill switch, disabling interface\n",
5759                             __func__);
5760                         iwm_stop(sc);
5761                 }
5762         }
5763
5764         /*
5765          * The Linux driver uses periodic interrupts to avoid races.
5766          * We cargo-cult like it's going out of fashion.
5767          */
5768         if (r1 & IWM_CSR_INT_BIT_RX_PERIODIC) {
5769                 handled |= IWM_CSR_INT_BIT_RX_PERIODIC;
5770                 IWM_WRITE(sc, IWM_CSR_INT, IWM_CSR_INT_BIT_RX_PERIODIC);
5771                 if ((r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX)) == 0)
5772                         IWM_WRITE_1(sc,
5773                             IWM_CSR_INT_PERIODIC_REG, IWM_CSR_INT_PERIODIC_DIS);
5774                 isperiodic = 1;
5775         }
5776
5777         if ((r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX)) || isperiodic) {
5778                 handled |= (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX);
5779                 IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, IWM_CSR_FH_INT_RX_MASK);
5780
5781                 iwm_notif_intr(sc);
5782
5783                 /* enable periodic interrupt, see above */
5784                 if (r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX) && !isperiodic)
5785                         IWM_WRITE_1(sc, IWM_CSR_INT_PERIODIC_REG,
5786                             IWM_CSR_INT_PERIODIC_ENA);
5787         }
5788
5789         if (__predict_false(r1 & ~handled))
5790                 IWM_DPRINTF(sc, IWM_DEBUG_INTR,
5791                     "%s: unhandled interrupts: %x\n", __func__, r1);
5792         rv = 1;
5793
5794  out_ena:
5795         iwm_restore_interrupts(sc);
5796  out:
5797         IWM_UNLOCK(sc);
5798         return;
5799 }
5800
5801 /*
5802  * Autoconf glue-sniffing
5803  */
5804 #define PCI_VENDOR_INTEL                0x8086
5805 #define PCI_PRODUCT_INTEL_WL_3160_1     0x08b3
5806 #define PCI_PRODUCT_INTEL_WL_3160_2     0x08b4
5807 #define PCI_PRODUCT_INTEL_WL_3165_1     0x3165
5808 #define PCI_PRODUCT_INTEL_WL_3165_2     0x3166
5809 #define PCI_PRODUCT_INTEL_WL_7260_1     0x08b1
5810 #define PCI_PRODUCT_INTEL_WL_7260_2     0x08b2
5811 #define PCI_PRODUCT_INTEL_WL_7265_1     0x095a
5812 #define PCI_PRODUCT_INTEL_WL_7265_2     0x095b
5813 #define PCI_PRODUCT_INTEL_WL_8260_1     0x24f3
5814 #define PCI_PRODUCT_INTEL_WL_8260_2     0x24f4
5815
5816 static const struct iwm_devices {
5817         uint16_t                device;
5818         const struct iwm_cfg    *cfg;
5819 } iwm_devices[] = {
5820         { PCI_PRODUCT_INTEL_WL_3160_1, &iwm3160_cfg },
5821         { PCI_PRODUCT_INTEL_WL_3160_2, &iwm3160_cfg },
5822         { PCI_PRODUCT_INTEL_WL_3165_1, &iwm3165_cfg },
5823         { PCI_PRODUCT_INTEL_WL_3165_2, &iwm3165_cfg },
5824         { PCI_PRODUCT_INTEL_WL_7260_1, &iwm7260_cfg },
5825         { PCI_PRODUCT_INTEL_WL_7260_2, &iwm7260_cfg },
5826         { PCI_PRODUCT_INTEL_WL_7265_1, &iwm7265_cfg },
5827         { PCI_PRODUCT_INTEL_WL_7265_2, &iwm7265_cfg },
5828         { PCI_PRODUCT_INTEL_WL_8260_1, &iwm8260_cfg },
5829         { PCI_PRODUCT_INTEL_WL_8260_2, &iwm8260_cfg },
5830 };
5831
5832 static int
5833 iwm_probe(device_t dev)
5834 {
5835         int i;
5836
5837         for (i = 0; i < nitems(iwm_devices); i++) {
5838                 if (pci_get_vendor(dev) == PCI_VENDOR_INTEL &&
5839                     pci_get_device(dev) == iwm_devices[i].device) {
5840                         device_set_desc(dev, iwm_devices[i].cfg->name);
5841                         return (BUS_PROBE_DEFAULT);
5842                 }
5843         }
5844
5845         return (ENXIO);
5846 }
5847
5848 static int
5849 iwm_dev_check(device_t dev)
5850 {
5851         struct iwm_softc *sc;
5852         uint16_t devid;
5853         int i;
5854
5855         sc = device_get_softc(dev);
5856
5857         devid = pci_get_device(dev);
5858         for (i = 0; i < nitems(iwm_devices); i++) {
5859                 if (iwm_devices[i].device == devid) {
5860                         sc->cfg = iwm_devices[i].cfg;
5861                         return (0);
5862                 }
5863         }
5864         device_printf(dev, "unknown adapter type\n");
5865         return ENXIO;
5866 }
5867
5868 /* PCI registers */
5869 #define PCI_CFG_RETRY_TIMEOUT   0x041
5870
5871 static int
5872 iwm_pci_attach(device_t dev)
5873 {
5874         struct iwm_softc *sc;
5875         int count, error, rid;
5876         uint16_t reg;
5877
5878         sc = device_get_softc(dev);
5879
5880         /* We disable the RETRY_TIMEOUT register (0x41) to keep
5881          * PCI Tx retries from interfering with C3 CPU state */
5882         pci_write_config(dev, PCI_CFG_RETRY_TIMEOUT, 0x00, 1);
5883
5884         /* Enable bus-mastering and hardware bug workaround. */
5885         pci_enable_busmaster(dev);
5886         reg = pci_read_config(dev, PCIR_STATUS, sizeof(reg));
5887         /* if !MSI */
5888         if (reg & PCIM_STATUS_INTxSTATE) {
5889                 reg &= ~PCIM_STATUS_INTxSTATE;
5890         }
5891         pci_write_config(dev, PCIR_STATUS, reg, sizeof(reg));
5892
5893         rid = PCIR_BAR(0);
5894         sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
5895             RF_ACTIVE);
5896         if (sc->sc_mem == NULL) {
5897                 device_printf(sc->sc_dev, "can't map mem space\n");
5898                 return (ENXIO);
5899         }
5900         sc->sc_st = rman_get_bustag(sc->sc_mem);
5901         sc->sc_sh = rman_get_bushandle(sc->sc_mem);
5902
5903         /* Install interrupt handler. */
5904         count = 1;
5905         rid = 0;
5906         if (pci_alloc_msi(dev, &count) == 0)
5907                 rid = 1;
5908         sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE |
5909             (rid != 0 ? 0 : RF_SHAREABLE));
5910         if (sc->sc_irq == NULL) {
5911                 device_printf(dev, "can't map interrupt\n");
5912                         return (ENXIO);
5913         }
5914         error = bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
5915             NULL, iwm_intr, sc, &sc->sc_ih);
5916         if (sc->sc_ih == NULL) {
5917                 device_printf(dev, "can't establish interrupt");
5918                         return (ENXIO);
5919         }
5920         sc->sc_dmat = bus_get_dma_tag(sc->sc_dev);
5921
5922         return (0);
5923 }
5924
5925 static void
5926 iwm_pci_detach(device_t dev)
5927 {
5928         struct iwm_softc *sc = device_get_softc(dev);
5929
5930         if (sc->sc_irq != NULL) {
5931                 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
5932                 bus_release_resource(dev, SYS_RES_IRQ,
5933                     rman_get_rid(sc->sc_irq), sc->sc_irq);
5934                 pci_release_msi(dev);
5935         }
5936         if (sc->sc_mem != NULL)
5937                 bus_release_resource(dev, SYS_RES_MEMORY,
5938                     rman_get_rid(sc->sc_mem), sc->sc_mem);
5939 }
5940
5941
5942
5943 static int
5944 iwm_attach(device_t dev)
5945 {
5946         struct iwm_softc *sc = device_get_softc(dev);
5947         struct ieee80211com *ic = &sc->sc_ic;
5948         int error;
5949         int txq_i, i;
5950
5951         sc->sc_dev = dev;
5952         sc->sc_attached = 1;
5953         IWM_LOCK_INIT(sc);
5954         mbufq_init(&sc->sc_snd, ifqmaxlen);
5955         callout_init_mtx(&sc->sc_watchdog_to, &sc->sc_mtx, 0);
5956         callout_init_mtx(&sc->sc_led_blink_to, &sc->sc_mtx, 0);
5957         TASK_INIT(&sc->sc_es_task, 0, iwm_endscan_cb, sc);
5958
5959         sc->sc_notif_wait = iwm_notification_wait_init(sc);
5960         if (sc->sc_notif_wait == NULL) {
5961                 device_printf(dev, "failed to init notification wait struct\n");
5962                 goto fail;
5963         }
5964
5965         /* Init phy db */
5966         sc->sc_phy_db = iwm_phy_db_init(sc);
5967         if (!sc->sc_phy_db) {
5968                 device_printf(dev, "Cannot init phy_db\n");
5969                 goto fail;
5970         }
5971
5972         /* PCI attach */
5973         error = iwm_pci_attach(dev);
5974         if (error != 0)
5975                 goto fail;
5976
5977         sc->sc_wantresp = -1;
5978
5979         /* Check device type */
5980         error = iwm_dev_check(dev);
5981         if (error != 0)
5982                 goto fail;
5983
5984         sc->sc_hw_rev = IWM_READ(sc, IWM_CSR_HW_REV);
5985         /*
5986          * In the 8000 HW family the format of the 4 bytes of CSR_HW_REV have
5987          * changed, and now the revision step also includes bit 0-1 (no more
5988          * "dash" value). To keep hw_rev backwards compatible - we'll store it
5989          * in the old format.
5990          */
5991         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000)
5992                 sc->sc_hw_rev = (sc->sc_hw_rev & 0xfff0) |
5993                                 (IWM_CSR_HW_REV_STEP(sc->sc_hw_rev << 2) << 2);
5994
5995         if (iwm_prepare_card_hw(sc) != 0) {
5996                 device_printf(dev, "could not initialize hardware\n");
5997                 goto fail;
5998         }
5999
6000         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) {
6001                 int ret;
6002                 uint32_t hw_step;
6003
6004                 /*
6005                  * In order to recognize C step the driver should read the
6006                  * chip version id located at the AUX bus MISC address.
6007                  */
6008                 IWM_SETBITS(sc, IWM_CSR_GP_CNTRL,
6009                             IWM_CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
6010                 DELAY(2);
6011
6012                 ret = iwm_poll_bit(sc, IWM_CSR_GP_CNTRL,
6013                                    IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
6014                                    IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
6015                                    25000);
6016                 if (!ret) {
6017                         device_printf(sc->sc_dev,
6018                             "Failed to wake up the nic\n");
6019                         goto fail;
6020                 }
6021
6022                 if (iwm_nic_lock(sc)) {
6023                         hw_step = iwm_read_prph(sc, IWM_WFPM_CTRL_REG);
6024                         hw_step |= IWM_ENABLE_WFPM;
6025                         iwm_write_prph(sc, IWM_WFPM_CTRL_REG, hw_step);
6026                         hw_step = iwm_read_prph(sc, IWM_AUX_MISC_REG);
6027                         hw_step = (hw_step >> IWM_HW_STEP_LOCATION_BITS) & 0xF;
6028                         if (hw_step == 0x3)
6029                                 sc->sc_hw_rev = (sc->sc_hw_rev & 0xFFFFFFF3) |
6030                                                 (IWM_SILICON_C_STEP << 2);
6031                         iwm_nic_unlock(sc);
6032                 } else {
6033                         device_printf(sc->sc_dev, "Failed to lock the nic\n");
6034                         goto fail;
6035                 }
6036         }
6037
6038         /* special-case 7265D, it has the same PCI IDs. */
6039         if (sc->cfg == &iwm7265_cfg &&
6040             (sc->sc_hw_rev & IWM_CSR_HW_REV_TYPE_MSK) == IWM_CSR_HW_REV_TYPE_7265D) {
6041                 sc->cfg = &iwm7265d_cfg;
6042         }
6043
6044         /* Allocate DMA memory for firmware transfers. */
6045         if ((error = iwm_alloc_fwmem(sc)) != 0) {
6046                 device_printf(dev, "could not allocate memory for firmware\n");
6047                 goto fail;
6048         }
6049
6050         /* Allocate "Keep Warm" page. */
6051         if ((error = iwm_alloc_kw(sc)) != 0) {
6052                 device_printf(dev, "could not allocate keep warm page\n");
6053                 goto fail;
6054         }
6055
6056         /* We use ICT interrupts */
6057         if ((error = iwm_alloc_ict(sc)) != 0) {
6058                 device_printf(dev, "could not allocate ICT table\n");
6059                 goto fail;
6060         }
6061
6062         /* Allocate TX scheduler "rings". */
6063         if ((error = iwm_alloc_sched(sc)) != 0) {
6064                 device_printf(dev, "could not allocate TX scheduler rings\n");
6065                 goto fail;
6066         }
6067
6068         /* Allocate TX rings */
6069         for (txq_i = 0; txq_i < nitems(sc->txq); txq_i++) {
6070                 if ((error = iwm_alloc_tx_ring(sc,
6071                     &sc->txq[txq_i], txq_i)) != 0) {
6072                         device_printf(dev,
6073                             "could not allocate TX ring %d\n",
6074                             txq_i);
6075                         goto fail;
6076                 }
6077         }
6078
6079         /* Allocate RX ring. */
6080         if ((error = iwm_alloc_rx_ring(sc, &sc->rxq)) != 0) {
6081                 device_printf(dev, "could not allocate RX ring\n");
6082                 goto fail;
6083         }
6084
6085         /* Clear pending interrupts. */
6086         IWM_WRITE(sc, IWM_CSR_INT, 0xffffffff);
6087
6088         ic->ic_softc = sc;
6089         ic->ic_name = device_get_nameunit(sc->sc_dev);
6090         ic->ic_phytype = IEEE80211_T_OFDM;      /* not only, but not used */
6091         ic->ic_opmode = IEEE80211_M_STA;        /* default to BSS mode */
6092
6093         /* Set device capabilities. */
6094         ic->ic_caps =
6095             IEEE80211_C_STA |
6096             IEEE80211_C_WPA |           /* WPA/RSN */
6097             IEEE80211_C_WME |
6098             IEEE80211_C_SHSLOT |        /* short slot time supported */
6099             IEEE80211_C_SHPREAMBLE      /* short preamble supported */
6100 //          IEEE80211_C_BGSCAN          /* capable of bg scanning */
6101             ;
6102         /* Advertise full-offload scanning */
6103         ic->ic_flags_ext = IEEE80211_FEXT_SCAN_OFFLOAD;
6104         for (i = 0; i < nitems(sc->sc_phyctxt); i++) {
6105                 sc->sc_phyctxt[i].id = i;
6106                 sc->sc_phyctxt[i].color = 0;
6107                 sc->sc_phyctxt[i].ref = 0;
6108                 sc->sc_phyctxt[i].channel = NULL;
6109         }
6110
6111         /* Default noise floor */
6112         sc->sc_noise = -96;
6113
6114         /* Max RSSI */
6115         sc->sc_max_rssi = IWM_MAX_DBM - IWM_MIN_DBM;
6116
6117         sc->sc_preinit_hook.ich_func = iwm_preinit;
6118         sc->sc_preinit_hook.ich_arg = sc;
6119         if (config_intrhook_establish(&sc->sc_preinit_hook) != 0) {
6120                 device_printf(dev, "config_intrhook_establish failed\n");
6121                 goto fail;
6122         }
6123
6124 #ifdef IWM_DEBUG
6125         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
6126             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug",
6127             CTLFLAG_RW, &sc->sc_debug, 0, "control debugging");
6128 #endif
6129
6130         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6131             "<-%s\n", __func__);
6132
6133         return 0;
6134
6135         /* Free allocated memory if something failed during attachment. */
6136 fail:
6137         iwm_detach_local(sc, 0);
6138
6139         return ENXIO;
6140 }
6141
6142 static int
6143 iwm_is_valid_ether_addr(uint8_t *addr)
6144 {
6145         char zero_addr[IEEE80211_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 };
6146
6147         if ((addr[0] & 1) || IEEE80211_ADDR_EQ(zero_addr, addr))
6148                 return (FALSE);
6149
6150         return (TRUE);
6151 }
6152
6153 static int
6154 iwm_update_edca(struct ieee80211com *ic)
6155 {
6156         struct iwm_softc *sc = ic->ic_softc;
6157
6158         device_printf(sc->sc_dev, "%s: called\n", __func__);
6159         return (0);
6160 }
6161
6162 static void
6163 iwm_preinit(void *arg)
6164 {
6165         struct iwm_softc *sc = arg;
6166         device_t dev = sc->sc_dev;
6167         struct ieee80211com *ic = &sc->sc_ic;
6168         int error;
6169
6170         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6171             "->%s\n", __func__);
6172
6173         IWM_LOCK(sc);
6174         if ((error = iwm_start_hw(sc)) != 0) {
6175                 device_printf(dev, "could not initialize hardware\n");
6176                 IWM_UNLOCK(sc);
6177                 goto fail;
6178         }
6179
6180         error = iwm_run_init_mvm_ucode(sc, 1);
6181         iwm_stop_device(sc);
6182         if (error) {
6183                 IWM_UNLOCK(sc);
6184                 goto fail;
6185         }
6186         device_printf(dev,
6187             "hw rev 0x%x, fw ver %s, address %s\n",
6188             sc->sc_hw_rev & IWM_CSR_HW_REV_TYPE_MSK,
6189             sc->sc_fwver, ether_sprintf(sc->nvm_data->hw_addr));
6190
6191         /* not all hardware can do 5GHz band */
6192         if (!sc->nvm_data->sku_cap_band_52GHz_enable)
6193                 memset(&ic->ic_sup_rates[IEEE80211_MODE_11A], 0,
6194                     sizeof(ic->ic_sup_rates[IEEE80211_MODE_11A]));
6195         IWM_UNLOCK(sc);
6196
6197         iwm_init_channel_map(ic, IEEE80211_CHAN_MAX, &ic->ic_nchans,
6198             ic->ic_channels);
6199
6200         /*
6201          * At this point we've committed - if we fail to do setup,
6202          * we now also have to tear down the net80211 state.
6203          */
6204         ieee80211_ifattach(ic);
6205         ic->ic_vap_create = iwm_vap_create;
6206         ic->ic_vap_delete = iwm_vap_delete;
6207         ic->ic_raw_xmit = iwm_raw_xmit;
6208         ic->ic_node_alloc = iwm_node_alloc;
6209         ic->ic_scan_start = iwm_scan_start;
6210         ic->ic_scan_end = iwm_scan_end;
6211         ic->ic_update_mcast = iwm_update_mcast;
6212         ic->ic_getradiocaps = iwm_init_channel_map;
6213         ic->ic_set_channel = iwm_set_channel;
6214         ic->ic_scan_curchan = iwm_scan_curchan;
6215         ic->ic_scan_mindwell = iwm_scan_mindwell;
6216         ic->ic_wme.wme_update = iwm_update_edca;
6217         ic->ic_parent = iwm_parent;
6218         ic->ic_transmit = iwm_transmit;
6219         iwm_radiotap_attach(sc);
6220         if (bootverbose)
6221                 ieee80211_announce(ic);
6222
6223         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6224             "<-%s\n", __func__);
6225         config_intrhook_disestablish(&sc->sc_preinit_hook);
6226
6227         return;
6228 fail:
6229         config_intrhook_disestablish(&sc->sc_preinit_hook);
6230         iwm_detach_local(sc, 0);
6231 }
6232
6233 /*
6234  * Attach the interface to 802.11 radiotap.
6235  */
6236 static void
6237 iwm_radiotap_attach(struct iwm_softc *sc)
6238 {
6239         struct ieee80211com *ic = &sc->sc_ic;
6240
6241         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6242             "->%s begin\n", __func__);
6243         ieee80211_radiotap_attach(ic,
6244             &sc->sc_txtap.wt_ihdr, sizeof(sc->sc_txtap),
6245                 IWM_TX_RADIOTAP_PRESENT,
6246             &sc->sc_rxtap.wr_ihdr, sizeof(sc->sc_rxtap),
6247                 IWM_RX_RADIOTAP_PRESENT);
6248         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6249             "->%s end\n", __func__);
6250 }
6251
6252 static struct ieee80211vap *
6253 iwm_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], int unit,
6254     enum ieee80211_opmode opmode, int flags,
6255     const uint8_t bssid[IEEE80211_ADDR_LEN],
6256     const uint8_t mac[IEEE80211_ADDR_LEN])
6257 {
6258         struct iwm_vap *ivp;
6259         struct ieee80211vap *vap;
6260
6261         if (!TAILQ_EMPTY(&ic->ic_vaps))         /* only one at a time */
6262                 return NULL;
6263         ivp = malloc(sizeof(struct iwm_vap), M_80211_VAP, M_WAITOK | M_ZERO);
6264         vap = &ivp->iv_vap;
6265         ieee80211_vap_setup(ic, vap, name, unit, opmode, flags, bssid);
6266         vap->iv_bmissthreshold = 10;            /* override default */
6267         /* Override with driver methods. */
6268         ivp->iv_newstate = vap->iv_newstate;
6269         vap->iv_newstate = iwm_newstate;
6270
6271         ieee80211_ratectl_init(vap);
6272         /* Complete setup. */
6273         ieee80211_vap_attach(vap, iwm_media_change, ieee80211_media_status,
6274             mac);
6275         ic->ic_opmode = opmode;
6276
6277         return vap;
6278 }
6279
6280 static void
6281 iwm_vap_delete(struct ieee80211vap *vap)
6282 {
6283         struct iwm_vap *ivp = IWM_VAP(vap);
6284
6285         ieee80211_ratectl_deinit(vap);
6286         ieee80211_vap_detach(vap);
6287         free(ivp, M_80211_VAP);
6288 }
6289
6290 static void
6291 iwm_scan_start(struct ieee80211com *ic)
6292 {
6293         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
6294         struct iwm_softc *sc = ic->ic_softc;
6295         int error;
6296
6297         IWM_LOCK(sc);
6298         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
6299                 /* This should not be possible */
6300                 device_printf(sc->sc_dev,
6301                     "%s: Previous scan not completed yet\n", __func__);
6302         }
6303         if (fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_UMAC_SCAN))
6304                 error = iwm_mvm_umac_scan(sc);
6305         else
6306                 error = iwm_mvm_lmac_scan(sc);
6307         if (error != 0) {
6308                 device_printf(sc->sc_dev, "could not initiate scan\n");
6309                 IWM_UNLOCK(sc);
6310                 ieee80211_cancel_scan(vap);
6311         } else {
6312                 sc->sc_flags |= IWM_FLAG_SCAN_RUNNING;
6313                 iwm_led_blink_start(sc);
6314                 IWM_UNLOCK(sc);
6315         }
6316 }
6317
6318 static void
6319 iwm_scan_end(struct ieee80211com *ic)
6320 {
6321         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
6322         struct iwm_softc *sc = ic->ic_softc;
6323
6324         IWM_LOCK(sc);
6325         iwm_led_blink_stop(sc);
6326         if (vap->iv_state == IEEE80211_S_RUN)
6327                 iwm_mvm_led_enable(sc);
6328         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
6329                 /*
6330                  * Removing IWM_FLAG_SCAN_RUNNING now, is fine because
6331                  * both iwm_scan_end and iwm_scan_start run in the ic->ic_tq
6332                  * taskqueue.
6333                  */
6334                 sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
6335                 iwm_mvm_scan_stop_wait(sc);
6336         }
6337         IWM_UNLOCK(sc);
6338
6339         /*
6340          * Make sure we don't race, if sc_es_task is still enqueued here.
6341          * This is to make sure that it won't call ieee80211_scan_done
6342          * when we have already started the next scan.
6343          */
6344         taskqueue_cancel(ic->ic_tq, &sc->sc_es_task, NULL);
6345 }
6346
6347 static void
6348 iwm_update_mcast(struct ieee80211com *ic)
6349 {
6350 }
6351
6352 static void
6353 iwm_set_channel(struct ieee80211com *ic)
6354 {
6355 }
6356
6357 static void
6358 iwm_scan_curchan(struct ieee80211_scan_state *ss, unsigned long maxdwell)
6359 {
6360 }
6361
6362 static void
6363 iwm_scan_mindwell(struct ieee80211_scan_state *ss)
6364 {
6365         return;
6366 }
6367
6368 void
6369 iwm_init_task(void *arg1)
6370 {
6371         struct iwm_softc *sc = arg1;
6372
6373         IWM_LOCK(sc);
6374         while (sc->sc_flags & IWM_FLAG_BUSY)
6375                 msleep(&sc->sc_flags, &sc->sc_mtx, 0, "iwmpwr", 0);
6376         sc->sc_flags |= IWM_FLAG_BUSY;
6377         iwm_stop(sc);
6378         if (sc->sc_ic.ic_nrunning > 0)
6379                 iwm_init(sc);
6380         sc->sc_flags &= ~IWM_FLAG_BUSY;
6381         wakeup(&sc->sc_flags);
6382         IWM_UNLOCK(sc);
6383 }
6384
6385 static int
6386 iwm_resume(device_t dev)
6387 {
6388         struct iwm_softc *sc = device_get_softc(dev);
6389         int do_reinit = 0;
6390
6391         /*
6392          * We disable the RETRY_TIMEOUT register (0x41) to keep
6393          * PCI Tx retries from interfering with C3 CPU state.
6394          */
6395         pci_write_config(dev, PCI_CFG_RETRY_TIMEOUT, 0x00, 1);
6396         iwm_init_task(device_get_softc(dev));
6397
6398         IWM_LOCK(sc);
6399         if (sc->sc_flags & IWM_FLAG_SCANNING) {
6400                 sc->sc_flags &= ~IWM_FLAG_SCANNING;
6401                 do_reinit = 1;
6402         }
6403         IWM_UNLOCK(sc);
6404
6405         if (do_reinit)
6406                 ieee80211_resume_all(&sc->sc_ic);
6407
6408         return 0;
6409 }
6410
6411 static int
6412 iwm_suspend(device_t dev)
6413 {
6414         int do_stop = 0;
6415         struct iwm_softc *sc = device_get_softc(dev);
6416
6417         do_stop = !! (sc->sc_ic.ic_nrunning > 0);
6418
6419         ieee80211_suspend_all(&sc->sc_ic);
6420
6421         if (do_stop) {
6422                 IWM_LOCK(sc);
6423                 iwm_stop(sc);
6424                 sc->sc_flags |= IWM_FLAG_SCANNING;
6425                 IWM_UNLOCK(sc);
6426         }
6427
6428         return (0);
6429 }
6430
6431 static int
6432 iwm_detach_local(struct iwm_softc *sc, int do_net80211)
6433 {
6434         struct iwm_fw_info *fw = &sc->sc_fw;
6435         device_t dev = sc->sc_dev;
6436         int i;
6437
6438         if (!sc->sc_attached)
6439                 return 0;
6440         sc->sc_attached = 0;
6441
6442         if (do_net80211)
6443                 ieee80211_draintask(&sc->sc_ic, &sc->sc_es_task);
6444
6445         callout_drain(&sc->sc_led_blink_to);
6446         callout_drain(&sc->sc_watchdog_to);
6447         iwm_stop_device(sc);
6448         if (do_net80211) {
6449                 ieee80211_ifdetach(&sc->sc_ic);
6450         }
6451
6452         iwm_phy_db_free(sc->sc_phy_db);
6453         sc->sc_phy_db = NULL;
6454
6455         iwm_free_nvm_data(sc->nvm_data);
6456
6457         /* Free descriptor rings */
6458         iwm_free_rx_ring(sc, &sc->rxq);
6459         for (i = 0; i < nitems(sc->txq); i++)
6460                 iwm_free_tx_ring(sc, &sc->txq[i]);
6461
6462         /* Free firmware */
6463         if (fw->fw_fp != NULL)
6464                 iwm_fw_info_free(fw);
6465
6466         /* Free scheduler */
6467         iwm_dma_contig_free(&sc->sched_dma);
6468         iwm_dma_contig_free(&sc->ict_dma);
6469         iwm_dma_contig_free(&sc->kw_dma);
6470         iwm_dma_contig_free(&sc->fw_dma);
6471
6472         iwm_free_fw_paging(sc);
6473
6474         /* Finished with the hardware - detach things */
6475         iwm_pci_detach(dev);
6476
6477         if (sc->sc_notif_wait != NULL) {
6478                 iwm_notification_wait_free(sc->sc_notif_wait);
6479                 sc->sc_notif_wait = NULL;
6480         }
6481
6482         mbufq_drain(&sc->sc_snd);
6483         IWM_LOCK_DESTROY(sc);
6484
6485         return (0);
6486 }
6487
6488 static int
6489 iwm_detach(device_t dev)
6490 {
6491         struct iwm_softc *sc = device_get_softc(dev);
6492
6493         return (iwm_detach_local(sc, 1));
6494 }
6495
6496 static device_method_t iwm_pci_methods[] = {
6497         /* Device interface */
6498         DEVMETHOD(device_probe,         iwm_probe),
6499         DEVMETHOD(device_attach,        iwm_attach),
6500         DEVMETHOD(device_detach,        iwm_detach),
6501         DEVMETHOD(device_suspend,       iwm_suspend),
6502         DEVMETHOD(device_resume,        iwm_resume),
6503
6504         DEVMETHOD_END
6505 };
6506
6507 static driver_t iwm_pci_driver = {
6508         "iwm",
6509         iwm_pci_methods,
6510         sizeof (struct iwm_softc)
6511 };
6512
6513 static devclass_t iwm_devclass;
6514
6515 DRIVER_MODULE(iwm, pci, iwm_pci_driver, iwm_devclass, NULL, NULL);
6516 MODULE_DEPEND(iwm, firmware, 1, 1, 1);
6517 MODULE_DEPEND(iwm, pci, 1, 1, 1);
6518 MODULE_DEPEND(iwm, wlan, 1, 1, 1);