sys/dev/iwm/if_iwm.c

   1 /*      $OpenBSD: if_iwm.c,v 1.42 2015/05/30 02:49:23 deraadt Exp $     */
   2
   3 /*
   4  * Copyright (c) 2014 genua mbh <info@genua.de>
   5  * Copyright (c) 2014 Fixup Software Ltd.
   6  *
   7  * Permission to use, copy, modify, and distribute this software for any
   8  * purpose with or without fee is hereby granted, provided that the above
   9  * copyright notice and this permission notice appear in all copies.
  10  *
  11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  18  */
  19
  20 /*-
  21  * Based on BSD-licensed source modules in the Linux iwlwifi driver,
  22  * which were used as the reference documentation for this implementation.
  23  *
  24  * Driver version we are currently based off of is
  25  * Linux 3.14.3 (tag id a2df521e42b1d9a23f620ac79dbfe8655a8391dd)
  26  *
  27  ***********************************************************************
  28  *
  29  * This file is provided under a dual BSD/GPLv2 license.  When using or
  30  * redistributing this file, you may do so under either license.
  31  *
  32  * GPL LICENSE SUMMARY
  33  *
  34  * Copyright(c) 2007 - 2013 Intel Corporation. All rights reserved.
  35  *
  36  * This program is free software; you can redistribute it and/or modify
  37  * it under the terms of version 2 of the GNU General Public License as
  38  * published by the Free Software Foundation.
  39  *
  40  * This program is distributed in the hope that it will be useful, but
  41  * WITHOUT ANY WARRANTY; without even the implied warranty of
  42  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  43  * General Public License for more details.
  44  *
  45  * You should have received a copy of the GNU General Public License
  46  * along with this program; if not, write to the Free Software
  47  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
  48  * USA
  49  *
  50  * The full GNU General Public License is included in this distribution
  51  * in the file called COPYING.
  52  *
  53  * Contact Information:
  54  *  Intel Linux Wireless <ilw@linux.intel.com>
  55  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  56  *
  57  *
  58  * BSD LICENSE
  59  *
  60  * Copyright(c) 2005 - 2013 Intel Corporation. All rights reserved.
  61  * All rights reserved.
  62  *
  63  * Redistribution and use in source and binary forms, with or without
  64  * modification, are permitted provided that the following conditions
  65  * are met:
  66  *
  67  *  * Redistributions of source code must retain the above copyright
  68  *    notice, this list of conditions and the following disclaimer.
  69  *  * Redistributions in binary form must reproduce the above copyright
  70  *    notice, this list of conditions and the following disclaimer in
  71  *    the documentation and/or other materials provided with the
  72  *    distribution.
  73  *  * Neither the name Intel Corporation nor the names of its
  74  *    contributors may be used to endorse or promote products derived
  75  *    from this software without specific prior written permission.
  76  *
  77  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  78  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  79  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  80  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  81  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  82  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  83  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  84  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  85  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  86  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  87  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  88  */
  89
  90 /*-
  91  * Copyright (c) 2007-2010 Damien Bergamini <damien.bergamini@free.fr>
  92  *
  93  * Permission to use, copy, modify, and distribute this software for any
  94  * purpose with or without fee is hereby granted, provided that the above
  95  * copyright notice and this permission notice appear in all copies.
  96  *
  97  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  98  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  99  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 100  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 101  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 102  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 103  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 104  */
 105 #include <sys/cdefs.h>
 106 __FBSDID("$FreeBSD$");
 107
 108 #include "opt_wlan.h"
 109
 110 #include <sys/param.h>
 111 #include <sys/bus.h>
 112 #include <sys/conf.h>
 113 #include <sys/endian.h>
 114 #include <sys/firmware.h>
 115 #include <sys/kernel.h>
 116 #include <sys/malloc.h>
 117 #include <sys/mbuf.h>
 118 #include <sys/mutex.h>
 119 #include <sys/module.h>
 120 #include <sys/proc.h>
 121 #include <sys/rman.h>
 122 #include <sys/socket.h>
 123 #include <sys/sockio.h>
 124 #include <sys/sysctl.h>
 125 #include <sys/linker.h>
 126
 127 #include <machine/bus.h>
 128 #include <machine/endian.h>
 129 #include <machine/resource.h>
 130
 131 #include <dev/pci/pcivar.h>
 132 #include <dev/pci/pcireg.h>
 133
 134 #include <net/bpf.h>
 135
 136 #include <net/if.h>
 137 #include <net/if_var.h>
 138 #include <net/if_arp.h>
 139 #include <net/if_dl.h>
 140 #include <net/if_media.h>
 141 #include <net/if_types.h>
 142
 143 #include <netinet/in.h>
 144 #include <netinet/in_systm.h>
 145 #include <netinet/if_ether.h>
 146 #include <netinet/ip.h>
 147
 148 #include <net80211/ieee80211_var.h>
 149 #include <net80211/ieee80211_regdomain.h>
 150 #include <net80211/ieee80211_ratectl.h>
 151 #include <net80211/ieee80211_radiotap.h>
 152
 153 #include <dev/iwm/if_iwmreg.h>
 154 #include <dev/iwm/if_iwmvar.h>
 155 #include <dev/iwm/if_iwm_config.h>
 156 #include <dev/iwm/if_iwm_debug.h>
 157 #include <dev/iwm/if_iwm_notif_wait.h>
 158 #include <dev/iwm/if_iwm_util.h>
 159 #include <dev/iwm/if_iwm_binding.h>
 160 #include <dev/iwm/if_iwm_phy_db.h>
 161 #include <dev/iwm/if_iwm_mac_ctxt.h>
 162 #include <dev/iwm/if_iwm_phy_ctxt.h>
 163 #include <dev/iwm/if_iwm_time_event.h>
 164 #include <dev/iwm/if_iwm_power.h>
 165 #include <dev/iwm/if_iwm_scan.h>
 166
 167 #include <dev/iwm/if_iwm_pcie_trans.h>
 168 #include <dev/iwm/if_iwm_led.h>
 169 #include <dev/iwm/if_iwm_fw.h>
 170
 171 const uint8_t iwm_nvm_channels[] = {
 172         /* 2.4 GHz */
 173         1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
 174         /* 5 GHz */
 175         36, 40, 44, 48, 52, 56, 60, 64,
 176         100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144,
 177         149, 153, 157, 161, 165
 178 };
 179 _Static_assert(nitems(iwm_nvm_channels) <= IWM_NUM_CHANNELS,
 180     "IWM_NUM_CHANNELS is too small");
 181
 182 const uint8_t iwm_nvm_channels_8000[] = {
 183         /* 2.4 GHz */
 184         1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
 185         /* 5 GHz */
 186         36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92,
 187         96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144,
 188         149, 153, 157, 161, 165, 169, 173, 177, 181
 189 };
 190 _Static_assert(nitems(iwm_nvm_channels_8000) <= IWM_NUM_CHANNELS_8000,
 191     "IWM_NUM_CHANNELS_8000 is too small");
 192
 193 #define IWM_NUM_2GHZ_CHANNELS   14
 194 #define IWM_N_HW_ADDR_MASK      0xF
 195
 196 /*
 197  * XXX For now, there's simply a fixed set of rate table entries
 198  * that are populated.
 199  */
 200 const struct iwm_rate {
 201         uint8_t rate;
 202         uint8_t plcp;
 203 } iwm_rates[] = {
 204         {   2,  IWM_RATE_1M_PLCP  },
 205         {   4,  IWM_RATE_2M_PLCP  },
 206         {  11,  IWM_RATE_5M_PLCP  },
 207         {  22,  IWM_RATE_11M_PLCP },
 208         {  12,  IWM_RATE_6M_PLCP  },
 209         {  18,  IWM_RATE_9M_PLCP  },
 210         {  24,  IWM_RATE_12M_PLCP },
 211         {  36,  IWM_RATE_18M_PLCP },
 212         {  48,  IWM_RATE_24M_PLCP },
 213         {  72,  IWM_RATE_36M_PLCP },
 214         {  96,  IWM_RATE_48M_PLCP },
 215         { 108,  IWM_RATE_54M_PLCP },
 216 };
 217 #define IWM_RIDX_CCK    0
 218 #define IWM_RIDX_OFDM   4
 219 #define IWM_RIDX_MAX    (nitems(iwm_rates)-1)
 220 #define IWM_RIDX_IS_CCK(_i_) ((_i_) < IWM_RIDX_OFDM)
 221 #define IWM_RIDX_IS_OFDM(_i_) ((_i_) >= IWM_RIDX_OFDM)
 222
 223 struct iwm_nvm_section {
 224         uint16_t length;
 225         uint8_t *data;
 226 };
 227
 228 #define IWM_MVM_UCODE_ALIVE_TIMEOUT     hz
 229 #define IWM_MVM_UCODE_CALIB_TIMEOUT     (2*hz)
 230
 231 struct iwm_mvm_alive_data {
 232         int valid;
 233         uint32_t scd_base_addr;
 234 };
 235
 236 static int      iwm_store_cscheme(struct iwm_softc *, const uint8_t *, size_t);
 237 static int      iwm_firmware_store_section(struct iwm_softc *,
 238                                            enum iwm_ucode_type,
 239                                            const uint8_t *, size_t);
 240 static int      iwm_set_default_calib(struct iwm_softc *, const void *);
 241 static void     iwm_fw_info_free(struct iwm_fw_info *);
 242 static int      iwm_read_firmware(struct iwm_softc *, enum iwm_ucode_type);
 243 static int      iwm_alloc_fwmem(struct iwm_softc *);
 244 static int      iwm_alloc_sched(struct iwm_softc *);
 245 static int      iwm_alloc_kw(struct iwm_softc *);
 246 static int      iwm_alloc_ict(struct iwm_softc *);
 247 static int      iwm_alloc_rx_ring(struct iwm_softc *, struct iwm_rx_ring *);
 248 static void     iwm_reset_rx_ring(struct iwm_softc *, struct iwm_rx_ring *);
 249 static void     iwm_free_rx_ring(struct iwm_softc *, struct iwm_rx_ring *);
 250 static int      iwm_alloc_tx_ring(struct iwm_softc *, struct iwm_tx_ring *,
 251                                   int);
 252 static void     iwm_reset_tx_ring(struct iwm_softc *, struct iwm_tx_ring *);
 253 static void     iwm_free_tx_ring(struct iwm_softc *, struct iwm_tx_ring *);
 254 static void     iwm_enable_interrupts(struct iwm_softc *);
 255 static void     iwm_restore_interrupts(struct iwm_softc *);
 256 static void     iwm_disable_interrupts(struct iwm_softc *);
 257 static void     iwm_ict_reset(struct iwm_softc *);
 258 static int      iwm_allow_mcast(struct ieee80211vap *, struct iwm_softc *);
 259 static void     iwm_stop_device(struct iwm_softc *);
 260 static void     iwm_mvm_nic_config(struct iwm_softc *);
 261 static int      iwm_nic_rx_init(struct iwm_softc *);
 262 static int      iwm_nic_tx_init(struct iwm_softc *);
 263 static int      iwm_nic_init(struct iwm_softc *);
 264 static int      iwm_enable_txq(struct iwm_softc *, int, int, int);
 265 static int      iwm_trans_pcie_fw_alive(struct iwm_softc *, uint32_t);
 266 static int      iwm_nvm_read_chunk(struct iwm_softc *, uint16_t, uint16_t,
 267                                    uint16_t, uint8_t *, uint16_t *);
 268 static int      iwm_nvm_read_section(struct iwm_softc *, uint16_t, uint8_t *,
 269                                      uint16_t *, uint32_t);
 270 static uint32_t iwm_eeprom_channel_flags(uint16_t);
 271 static void     iwm_add_channel_band(struct iwm_softc *,
 272                     struct ieee80211_channel[], int, int *, int, size_t,
 273                     const uint8_t[]);
 274 static void     iwm_init_channel_map(struct ieee80211com *, int, int *,
 275                     struct ieee80211_channel[]);
 276 static struct iwm_nvm_data *
 277         iwm_parse_nvm_data(struct iwm_softc *, const uint16_t *,
 278                            const uint16_t *, const uint16_t *,
 279                            const uint16_t *, const uint16_t *,
 280                            const uint16_t *);
 281 static void     iwm_free_nvm_data(struct iwm_nvm_data *);
 282 static void     iwm_set_hw_address_family_8000(struct iwm_softc *,
 283                                                struct iwm_nvm_data *,
 284                                                const uint16_t *,
 285                                                const uint16_t *);
 286 static int      iwm_get_sku(const struct iwm_softc *, const uint16_t *,
 287                             const uint16_t *);
 288 static int      iwm_get_nvm_version(const struct iwm_softc *, const uint16_t *);
 289 static int      iwm_get_radio_cfg(const struct iwm_softc *, const uint16_t *,
 290                                   const uint16_t *);
 291 static int      iwm_get_n_hw_addrs(const struct iwm_softc *,
 292                                    const uint16_t *);
 293 static void     iwm_set_radio_cfg(const struct iwm_softc *,
 294                                   struct iwm_nvm_data *, uint32_t);
 295 static struct iwm_nvm_data *
 296         iwm_parse_nvm_sections(struct iwm_softc *, struct iwm_nvm_section *);
 297 static int      iwm_nvm_init(struct iwm_softc *);
 298 static int      iwm_pcie_load_section(struct iwm_softc *, uint8_t,
 299                                       const struct iwm_fw_desc *);
 300 static int      iwm_pcie_load_firmware_chunk(struct iwm_softc *, uint32_t,
 301                                              bus_addr_t, uint32_t);
 302 static int      iwm_pcie_load_cpu_sections_8000(struct iwm_softc *sc,
 303                                                 const struct iwm_fw_sects *,
 304                                                 int, int *);
 305 static int      iwm_pcie_load_cpu_sections(struct iwm_softc *,
 306                                            const struct iwm_fw_sects *,
 307                                            int, int *);
 308 static int      iwm_pcie_load_given_ucode_8000(struct iwm_softc *,
 309                                                const struct iwm_fw_sects *);
 310 static int      iwm_pcie_load_given_ucode(struct iwm_softc *,
 311                                           const struct iwm_fw_sects *);
 312 static int      iwm_start_fw(struct iwm_softc *, const struct iwm_fw_sects *);
 313 static int      iwm_send_tx_ant_cfg(struct iwm_softc *, uint8_t);
 314 static int      iwm_send_phy_cfg_cmd(struct iwm_softc *);
 315 static int      iwm_mvm_load_ucode_wait_alive(struct iwm_softc *,
 316                                               enum iwm_ucode_type);
 317 static int      iwm_run_init_mvm_ucode(struct iwm_softc *, int);
 318 static int      iwm_rx_addbuf(struct iwm_softc *, int, int);
 319 static int      iwm_mvm_get_signal_strength(struct iwm_softc *,
 320                                             struct iwm_rx_phy_info *);
 321 static void     iwm_mvm_rx_rx_phy_cmd(struct iwm_softc *,
 322                                       struct iwm_rx_packet *,
 323                                       struct iwm_rx_data *);
 324 static int      iwm_get_noise(struct iwm_softc *sc,
 325                     const struct iwm_mvm_statistics_rx_non_phy *);
 326 static void     iwm_mvm_rx_rx_mpdu(struct iwm_softc *, struct mbuf *);
 327 static int      iwm_mvm_rx_tx_cmd_single(struct iwm_softc *,
 328                                          struct iwm_rx_packet *,
 329                                          struct iwm_node *);
 330 static void     iwm_mvm_rx_tx_cmd(struct iwm_softc *, struct iwm_rx_packet *,
 331                                   struct iwm_rx_data *);
 332 static void     iwm_cmd_done(struct iwm_softc *, struct iwm_rx_packet *);
 333 #if 0
 334 static void     iwm_update_sched(struct iwm_softc *, int, int, uint8_t,
 335                                  uint16_t);
 336 #endif
 337 static const struct iwm_rate *
 338         iwm_tx_fill_cmd(struct iwm_softc *, struct iwm_node *,
 339                         struct mbuf *, struct iwm_tx_cmd *);
 340 static int      iwm_tx(struct iwm_softc *, struct mbuf *,
 341                        struct ieee80211_node *, int);
 342 static int      iwm_raw_xmit(struct ieee80211_node *, struct mbuf *,
 343                              const struct ieee80211_bpf_params *);
 344 static int      iwm_mvm_flush_tx_path(struct iwm_softc *sc,
 345                                       uint32_t tfd_msk, uint32_t flags);
 346 static int      iwm_mvm_send_add_sta_cmd_status(struct iwm_softc *,
 347                                                 struct iwm_mvm_add_sta_cmd *,
 348                                                 int *);
 349 static int      iwm_mvm_sta_send_to_fw(struct iwm_softc *, struct iwm_node *,
 350                                        int);
 351 static int      iwm_mvm_add_sta(struct iwm_softc *, struct iwm_node *);
 352 static int      iwm_mvm_update_sta(struct iwm_softc *, struct iwm_node *);
 353 static int      iwm_mvm_add_int_sta_common(struct iwm_softc *,
 354                                            struct iwm_int_sta *,
 355                                            const uint8_t *, uint16_t, uint16_t);
 356 static int      iwm_mvm_add_aux_sta(struct iwm_softc *);
 357 static int      iwm_mvm_update_quotas(struct iwm_softc *, struct iwm_node *);
 358 static int      iwm_auth(struct ieee80211vap *, struct iwm_softc *);
 359 static int      iwm_assoc(struct ieee80211vap *, struct iwm_softc *);
 360 static int      iwm_release(struct iwm_softc *, struct iwm_node *);
 361 static struct ieee80211_node *
 362                 iwm_node_alloc(struct ieee80211vap *,
 363                                const uint8_t[IEEE80211_ADDR_LEN]);
 364 static void     iwm_setrates(struct iwm_softc *, struct iwm_node *);
 365 static int      iwm_media_change(struct ifnet *);
 366 static int      iwm_newstate(struct ieee80211vap *, enum ieee80211_state, int);
 367 static void     iwm_endscan_cb(void *, int);
 368 static void     iwm_mvm_fill_sf_command(struct iwm_softc *,
 369                                         struct iwm_sf_cfg_cmd *,
 370                                         struct ieee80211_node *);
 371 static int      iwm_mvm_sf_config(struct iwm_softc *, enum iwm_sf_state);
 372 static int      iwm_send_bt_init_conf(struct iwm_softc *);
 373 static int      iwm_send_update_mcc_cmd(struct iwm_softc *, const char *);
 374 static void     iwm_mvm_tt_tx_backoff(struct iwm_softc *, uint32_t);
 375 static int      iwm_init_hw(struct iwm_softc *);
 376 static void     iwm_init(struct iwm_softc *);
 377 static void     iwm_start(struct iwm_softc *);
 378 static void     iwm_stop(struct iwm_softc *);
 379 static void     iwm_watchdog(void *);
 380 static void     iwm_parent(struct ieee80211com *);
 381 #ifdef IWM_DEBUG
 382 static const char *
 383                 iwm_desc_lookup(uint32_t);
 384 static void     iwm_nic_error(struct iwm_softc *);
 385 static void     iwm_nic_umac_error(struct iwm_softc *);
 386 #endif
 387 static void     iwm_notif_intr(struct iwm_softc *);
 388 static void     iwm_intr(void *);
 389 static int      iwm_attach(device_t);
 390 static int      iwm_is_valid_ether_addr(uint8_t *);
 391 static void     iwm_preinit(void *);
 392 static int      iwm_detach_local(struct iwm_softc *sc, int);
 393 static void     iwm_init_task(void *);
 394 static void     iwm_radiotap_attach(struct iwm_softc *);
 395 static struct ieee80211vap *
 396                 iwm_vap_create(struct ieee80211com *,
 397                                const char [IFNAMSIZ], int,
 398                                enum ieee80211_opmode, int,
 399                                const uint8_t [IEEE80211_ADDR_LEN],
 400                                const uint8_t [IEEE80211_ADDR_LEN]);
 401 static void     iwm_vap_delete(struct ieee80211vap *);
 402 static void     iwm_scan_start(struct ieee80211com *);
 403 static void     iwm_scan_end(struct ieee80211com *);
 404 static void     iwm_update_mcast(struct ieee80211com *);
 405 static void     iwm_set_channel(struct ieee80211com *);
 406 static void     iwm_scan_curchan(struct ieee80211_scan_state *, unsigned long);
 407 static void     iwm_scan_mindwell(struct ieee80211_scan_state *);
 408 static int      iwm_detach(device_t);
 409
 410 /*
 411  * Firmware parser.
 412  */
 413
 414 static int
 415 iwm_store_cscheme(struct iwm_softc *sc, const uint8_t *data, size_t dlen)
 416 {
 417         const struct iwm_fw_cscheme_list *l = (const void *)data;
 418
 419         if (dlen < sizeof(*l) ||
 420             dlen < sizeof(l->size) + l->size * sizeof(*l->cs))
 421                 return EINVAL;
 422
 423         /* we don't actually store anything for now, always use s/w crypto */
 424
 425         return 0;
 426 }
 427
 428 static int
 429 iwm_firmware_store_section(struct iwm_softc *sc,
 430     enum iwm_ucode_type type, const uint8_t *data, size_t dlen)
 431 {
 432         struct iwm_fw_sects *fws;
 433         struct iwm_fw_desc *fwone;
 434
 435         if (type >= IWM_UCODE_TYPE_MAX)
 436                 return EINVAL;
 437         if (dlen < sizeof(uint32_t))
 438                 return EINVAL;
 439
 440         fws = &sc->sc_fw.fw_sects[type];
 441         if (fws->fw_count >= IWM_UCODE_SECTION_MAX)
 442                 return EINVAL;
 443
 444         fwone = &fws->fw_sect[fws->fw_count];
 445
 446         /* first 32bit are device load offset */
 447         memcpy(&fwone->offset, data, sizeof(uint32_t));
 448
 449         /* rest is data */
 450         fwone->data = data + sizeof(uint32_t);
 451         fwone->len = dlen - sizeof(uint32_t);
 452
 453         fws->fw_count++;
 454
 455         return 0;
 456 }
 457
 458 #define IWM_DEFAULT_SCAN_CHANNELS 40
 459
 460 /* iwlwifi: iwl-drv.c */
 461 struct iwm_tlv_calib_data {
 462         uint32_t ucode_type;
 463         struct iwm_tlv_calib_ctrl calib;
 464 } __packed;
 465
 466 static int
 467 iwm_set_default_calib(struct iwm_softc *sc, const void *data)
 468 {
 469         const struct iwm_tlv_calib_data *def_calib = data;
 470         uint32_t ucode_type = le32toh(def_calib->ucode_type);
 471
 472         if (ucode_type >= IWM_UCODE_TYPE_MAX) {
 473                 device_printf(sc->sc_dev,
 474                     "Wrong ucode_type %u for default "
 475                     "calibration.\n", ucode_type);
 476                 return EINVAL;
 477         }
 478
 479         sc->sc_default_calib[ucode_type].flow_trigger =
 480             def_calib->calib.flow_trigger;
 481         sc->sc_default_calib[ucode_type].event_trigger =
 482             def_calib->calib.event_trigger;
 483
 484         return 0;
 485 }
 486
 487 static int
 488 iwm_set_ucode_api_flags(struct iwm_softc *sc, const uint8_t *data,
 489                         struct iwm_ucode_capabilities *capa)
 490 {
 491         const struct iwm_ucode_api *ucode_api = (const void *)data;
 492         uint32_t api_index = le32toh(ucode_api->api_index);
 493         uint32_t api_flags = le32toh(ucode_api->api_flags);
 494         int i;
 495
 496         if (api_index >= howmany(IWM_NUM_UCODE_TLV_API, 32)) {
 497                 device_printf(sc->sc_dev,
 498                     "api flags index %d larger than supported by driver\n",
 499                     api_index);
 500                 /* don't return an error so we can load FW that has more bits */
 501                 return 0;
 502         }
 503
 504         for (i = 0; i < 32; i++) {
 505                 if (api_flags & (1U << i))
 506                         setbit(capa->enabled_api, i + 32 * api_index);
 507         }
 508
 509         return 0;
 510 }
 511
 512 static int
 513 iwm_set_ucode_capabilities(struct iwm_softc *sc, const uint8_t *data,
 514                            struct iwm_ucode_capabilities *capa)
 515 {
 516         const struct iwm_ucode_capa *ucode_capa = (const void *)data;
 517         uint32_t api_index = le32toh(ucode_capa->api_index);
 518         uint32_t api_flags = le32toh(ucode_capa->api_capa);
 519         int i;
 520
 521         if (api_index >= howmany(IWM_NUM_UCODE_TLV_CAPA, 32)) {
 522                 device_printf(sc->sc_dev,
 523                     "capa flags index %d larger than supported by driver\n",
 524                     api_index);
 525                 /* don't return an error so we can load FW that has more bits */
 526                 return 0;
 527         }
 528
 529         for (i = 0; i < 32; i++) {
 530                 if (api_flags & (1U << i))
 531                         setbit(capa->enabled_capa, i + 32 * api_index);
 532         }
 533
 534         return 0;
 535 }
 536
 537 static void
 538 iwm_fw_info_free(struct iwm_fw_info *fw)
 539 {
 540         firmware_put(fw->fw_fp, FIRMWARE_UNLOAD);
 541         fw->fw_fp = NULL;
 542         /* don't touch fw->fw_status */
 543         memset(fw->fw_sects, 0, sizeof(fw->fw_sects));
 544 }
 545
 546 static int
 547 iwm_read_firmware(struct iwm_softc *sc, enum iwm_ucode_type ucode_type)
 548 {
 549         struct iwm_fw_info *fw = &sc->sc_fw;
 550         const struct iwm_tlv_ucode_header *uhdr;
 551         struct iwm_ucode_tlv tlv;
 552         struct iwm_ucode_capabilities *capa = &sc->ucode_capa;
 553         enum iwm_ucode_tlv_type tlv_type;
 554         const struct firmware *fwp;
 555         const uint8_t *data;
 556         uint32_t usniffer_img;
 557         uint32_t paging_mem_size;
 558         int num_of_cpus;
 559         int error = 0;
 560         size_t len;
 561
 562         if (fw->fw_status == IWM_FW_STATUS_DONE &&
 563             ucode_type != IWM_UCODE_INIT)
 564                 return 0;
 565
 566         while (fw->fw_status == IWM_FW_STATUS_INPROGRESS)
 567                 msleep(&sc->sc_fw, &sc->sc_mtx, 0, "iwmfwp", 0);
 568         fw->fw_status = IWM_FW_STATUS_INPROGRESS;
 569
 570         if (fw->fw_fp != NULL)
 571                 iwm_fw_info_free(fw);
 572
 573         /*
 574          * Load firmware into driver memory.
 575          * fw_fp will be set.
 576          */
 577         IWM_UNLOCK(sc);
 578         fwp = firmware_get(sc->cfg->fw_name);
 579         IWM_LOCK(sc);
 580         if (fwp == NULL) {
 581                 device_printf(sc->sc_dev,
 582                     "could not read firmware %s (error %d)\n",
 583                     sc->cfg->fw_name, error);
 584                 goto out;
 585         }
 586         fw->fw_fp = fwp;
 587
 588         /* (Re-)Initialize default values. */
 589         capa->flags = 0;
 590         capa->max_probe_length = IWM_DEFAULT_MAX_PROBE_LENGTH;
 591         capa->n_scan_channels = IWM_DEFAULT_SCAN_CHANNELS;
 592         memset(capa->enabled_capa, 0, sizeof(capa->enabled_capa));
 593         memset(capa->enabled_api, 0, sizeof(capa->enabled_api));
 594         memset(sc->sc_fw_mcc, 0, sizeof(sc->sc_fw_mcc));
 595
 596         /*
 597          * Parse firmware contents
 598          */
 599
 600         uhdr = (const void *)fw->fw_fp->data;
 601         if (*(const uint32_t *)fw->fw_fp->data != 0
 602             || le32toh(uhdr->magic) != IWM_TLV_UCODE_MAGIC) {
 603                 device_printf(sc->sc_dev, "invalid firmware %s\n",
 604                     sc->cfg->fw_name);
 605                 error = EINVAL;
 606                 goto out;
 607         }
 608
 609         snprintf(sc->sc_fwver, sizeof(sc->sc_fwver), "%d.%d (API ver %d)",
 610             IWM_UCODE_MAJOR(le32toh(uhdr->ver)),
 611             IWM_UCODE_MINOR(le32toh(uhdr->ver)),
 612             IWM_UCODE_API(le32toh(uhdr->ver)));
 613         data = uhdr->data;
 614         len = fw->fw_fp->datasize - sizeof(*uhdr);
 615
 616         while (len >= sizeof(tlv)) {
 617                 size_t tlv_len;
 618                 const void *tlv_data;
 619
 620                 memcpy(&tlv, data, sizeof(tlv));
 621                 tlv_len = le32toh(tlv.length);
 622                 tlv_type = le32toh(tlv.type);
 623
 624                 len -= sizeof(tlv);
 625                 data += sizeof(tlv);
 626                 tlv_data = data;
 627
 628                 if (len < tlv_len) {
 629                         device_printf(sc->sc_dev,
 630                             "firmware too short: %zu bytes\n",
 631                             len);
 632                         error = EINVAL;
 633                         goto parse_out;
 634                 }
 635
 636                 switch ((int)tlv_type) {
 637                 case IWM_UCODE_TLV_PROBE_MAX_LEN:
 638                         if (tlv_len < sizeof(uint32_t)) {
 639                                 device_printf(sc->sc_dev,
 640                                     "%s: PROBE_MAX_LEN (%d) < sizeof(uint32_t)\n",
 641                                     __func__,
 642                                     (int) tlv_len);
 643                                 error = EINVAL;
 644                                 goto parse_out;
 645                         }
 646                         capa->max_probe_length =
 647                             le32toh(*(const uint32_t *)tlv_data);
 648                         /* limit it to something sensible */
 649                         if (capa->max_probe_length >
 650                             IWM_SCAN_OFFLOAD_PROBE_REQ_SIZE) {
 651                                 IWM_DPRINTF(sc, IWM_DEBUG_FIRMWARE_TLV,
 652                                     "%s: IWM_UCODE_TLV_PROBE_MAX_LEN "
 653                                     "ridiculous\n", __func__);
 654                                 error = EINVAL;
 655                                 goto parse_out;
 656                         }
 657                         break;
 658                 case IWM_UCODE_TLV_PAN:
 659                         if (tlv_len) {
 660                                 device_printf(sc->sc_dev,
 661                                     "%s: IWM_UCODE_TLV_PAN: tlv_len (%d) > 0\n",
 662                                     __func__,
 663                                     (int) tlv_len);
 664                                 error = EINVAL;
 665                                 goto parse_out;
 666                         }
 667                         capa->flags |= IWM_UCODE_TLV_FLAGS_PAN;
 668                         break;
 669                 case IWM_UCODE_TLV_FLAGS:
 670                         if (tlv_len < sizeof(uint32_t)) {
 671                                 device_printf(sc->sc_dev,
 672                                     "%s: IWM_UCODE_TLV_FLAGS: tlv_len (%d) < sizeof(uint32_t)\n",
 673                                     __func__,
 674                                     (int) tlv_len);
 675                                 error = EINVAL;
 676                                 goto parse_out;
 677                         }
 678                         /*
 679                          * Apparently there can be many flags, but Linux driver
 680                          * parses only the first one, and so do we.
 681                          *
 682                          * XXX: why does this override IWM_UCODE_TLV_PAN?
 683                          * Intentional or a bug?  Observations from
 684                          * current firmware file:
 685                          *  1) TLV_PAN is parsed first
 686                          *  2) TLV_FLAGS contains TLV_FLAGS_PAN
 687                          * ==> this resets TLV_PAN to itself... hnnnk
 688                          */
 689                         capa->flags = le32toh(*(const uint32_t *)tlv_data);
 690                         break;
 691                 case IWM_UCODE_TLV_CSCHEME:
 692                         if ((error = iwm_store_cscheme(sc,
 693                             tlv_data, tlv_len)) != 0) {
 694                                 device_printf(sc->sc_dev,
 695                                     "%s: iwm_store_cscheme(): returned %d\n",
 696                                     __func__,
 697                                     error);
 698                                 goto parse_out;
 699                         }
 700                         break;
 701                 case IWM_UCODE_TLV_NUM_OF_CPU:
 702                         if (tlv_len != sizeof(uint32_t)) {
 703                                 device_printf(sc->sc_dev,
 704                                     "%s: IWM_UCODE_TLV_NUM_OF_CPU: tlv_len (%d) != sizeof(uint32_t)\n",
 705                                     __func__,
 706                                     (int) tlv_len);
 707                                 error = EINVAL;
 708                                 goto parse_out;
 709                         }
 710                         num_of_cpus = le32toh(*(const uint32_t *)tlv_data);
 711                         if (num_of_cpus == 2) {
 712                                 fw->fw_sects[IWM_UCODE_REGULAR].is_dual_cpus =
 713                                         TRUE;
 714                                 fw->fw_sects[IWM_UCODE_INIT].is_dual_cpus =
 715                                         TRUE;
 716                                 fw->fw_sects[IWM_UCODE_WOWLAN].is_dual_cpus =
 717                                         TRUE;
 718                         } else if ((num_of_cpus > 2) || (num_of_cpus < 1)) {
 719                                 device_printf(sc->sc_dev,
 720                                     "%s: Driver supports only 1 or 2 CPUs\n",
 721                                     __func__);
 722                                 error = EINVAL;
 723                                 goto parse_out;
 724                         }
 725                         break;
 726                 case IWM_UCODE_TLV_SEC_RT:
 727                         if ((error = iwm_firmware_store_section(sc,
 728                             IWM_UCODE_REGULAR, tlv_data, tlv_len)) != 0) {
 729                                 device_printf(sc->sc_dev,
 730                                     "%s: IWM_UCODE_REGULAR: iwm_firmware_store_section() failed; %d\n",
 731                                     __func__,
 732                                     error);
 733                                 goto parse_out;
 734                         }
 735                         break;
 736                 case IWM_UCODE_TLV_SEC_INIT:
 737                         if ((error = iwm_firmware_store_section(sc,
 738                             IWM_UCODE_INIT, tlv_data, tlv_len)) != 0) {
 739                                 device_printf(sc->sc_dev,
 740                                     "%s: IWM_UCODE_INIT: iwm_firmware_store_section() failed; %d\n",
 741                                     __func__,
 742                                     error);
 743                                 goto parse_out;
 744                         }
 745                         break;
 746                 case IWM_UCODE_TLV_SEC_WOWLAN:
 747                         if ((error = iwm_firmware_store_section(sc,
 748                             IWM_UCODE_WOWLAN, tlv_data, tlv_len)) != 0) {
 749                                 device_printf(sc->sc_dev,
 750                                     "%s: IWM_UCODE_WOWLAN: iwm_firmware_store_section() failed; %d\n",
 751                                     __func__,
 752                                     error);
 753                                 goto parse_out;
 754                         }
 755                         break;
 756                 case IWM_UCODE_TLV_DEF_CALIB:
 757                         if (tlv_len != sizeof(struct iwm_tlv_calib_data)) {
 758                                 device_printf(sc->sc_dev,
 759                                     "%s: IWM_UCODE_TLV_DEV_CALIB: tlv_len (%d) < sizeof(iwm_tlv_calib_data) (%d)\n",
 760                                     __func__,
 761                                     (int) tlv_len,
 762                                     (int) sizeof(struct iwm_tlv_calib_data));
 763                                 error = EINVAL;
 764                                 goto parse_out;
 765                         }
 766                         if ((error = iwm_set_default_calib(sc, tlv_data)) != 0) {
 767                                 device_printf(sc->sc_dev,
 768                                     "%s: iwm_set_default_calib() failed: %d\n",
 769                                     __func__,
 770                                     error);
 771                                 goto parse_out;
 772                         }
 773                         break;
 774                 case IWM_UCODE_TLV_PHY_SKU:
 775                         if (tlv_len != sizeof(uint32_t)) {
 776                                 error = EINVAL;
 777                                 device_printf(sc->sc_dev,
 778                                     "%s: IWM_UCODE_TLV_PHY_SKU: tlv_len (%d) < sizeof(uint32_t)\n",
 779                                     __func__,
 780                                     (int) tlv_len);
 781                                 goto parse_out;
 782                         }
 783                         sc->sc_fw.phy_config =
 784                             le32toh(*(const uint32_t *)tlv_data);
 785                         sc->sc_fw.valid_tx_ant = (sc->sc_fw.phy_config &
 786                                                   IWM_FW_PHY_CFG_TX_CHAIN) >>
 787                                                   IWM_FW_PHY_CFG_TX_CHAIN_POS;
 788                         sc->sc_fw.valid_rx_ant = (sc->sc_fw.phy_config &
 789                                                   IWM_FW_PHY_CFG_RX_CHAIN) >>
 790                                                   IWM_FW_PHY_CFG_RX_CHAIN_POS;
 791                         break;
 792
 793                 case IWM_UCODE_TLV_API_CHANGES_SET: {
 794                         if (tlv_len != sizeof(struct iwm_ucode_api)) {
 795                                 error = EINVAL;
 796                                 goto parse_out;
 797                         }
 798                         if (iwm_set_ucode_api_flags(sc, tlv_data, capa)) {
 799                                 error = EINVAL;
 800                                 goto parse_out;
 801                         }
 802                         break;
 803                 }
 804
 805                 case IWM_UCODE_TLV_ENABLED_CAPABILITIES: {
 806                         if (tlv_len != sizeof(struct iwm_ucode_capa)) {
 807                                 error = EINVAL;
 808                                 goto parse_out;
 809                         }
 810                         if (iwm_set_ucode_capabilities(sc, tlv_data, capa)) {
 811                                 error = EINVAL;
 812                                 goto parse_out;
 813                         }
 814                         break;
 815                 }
 816
 817                 case 48: /* undocumented TLV */
 818                 case IWM_UCODE_TLV_SDIO_ADMA_ADDR:
 819                 case IWM_UCODE_TLV_FW_GSCAN_CAPA:
 820                         /* ignore, not used by current driver */
 821                         break;
 822
 823                 case IWM_UCODE_TLV_SEC_RT_USNIFFER:
 824                         if ((error = iwm_firmware_store_section(sc,
 825                             IWM_UCODE_REGULAR_USNIFFER, tlv_data,
 826                             tlv_len)) != 0)
 827                                 goto parse_out;
 828                         break;
 829
 830                 case IWM_UCODE_TLV_PAGING:
 831                         if (tlv_len != sizeof(uint32_t)) {
 832                                 error = EINVAL;
 833                                 goto parse_out;
 834                         }
 835                         paging_mem_size = le32toh(*(const uint32_t *)tlv_data);
 836
 837                         IWM_DPRINTF(sc, IWM_DEBUG_FIRMWARE_TLV,
 838                             "%s: Paging: paging enabled (size = %u bytes)\n",
 839                             __func__, paging_mem_size);
 840                         if (paging_mem_size > IWM_MAX_PAGING_IMAGE_SIZE) {
 841                                 device_printf(sc->sc_dev,
 842                                         "%s: Paging: driver supports up to %u bytes for paging image\n",
 843                                         __func__, IWM_MAX_PAGING_IMAGE_SIZE);
 844                                 error = EINVAL;
 845                                 goto out;
 846                         }
 847                         if (paging_mem_size & (IWM_FW_PAGING_SIZE - 1)) {
 848                                 device_printf(sc->sc_dev,
 849                                     "%s: Paging: image isn't multiple %u\n",
 850                                     __func__, IWM_FW_PAGING_SIZE);
 851                                 error = EINVAL;
 852                                 goto out;
 853                         }
 854
 855                         sc->sc_fw.fw_sects[IWM_UCODE_REGULAR].paging_mem_size =
 856                             paging_mem_size;
 857                         usniffer_img = IWM_UCODE_REGULAR_USNIFFER;
 858                         sc->sc_fw.fw_sects[usniffer_img].paging_mem_size =
 859                             paging_mem_size;
 860                         break;
 861
 862                 case IWM_UCODE_TLV_N_SCAN_CHANNELS:
 863                         if (tlv_len != sizeof(uint32_t)) {
 864                                 error = EINVAL;
 865                                 goto parse_out;
 866                         }
 867                         capa->n_scan_channels =
 868                             le32toh(*(const uint32_t *)tlv_data);
 869                         break;
 870
 871                 case IWM_UCODE_TLV_FW_VERSION:
 872                         if (tlv_len != sizeof(uint32_t) * 3) {
 873                                 error = EINVAL;
 874                                 goto parse_out;
 875                         }
 876                         snprintf(sc->sc_fwver, sizeof(sc->sc_fwver),
 877                             "%d.%d.%d",
 878                             le32toh(((const uint32_t *)tlv_data)[0]),
 879                             le32toh(((const uint32_t *)tlv_data)[1]),
 880                             le32toh(((const uint32_t *)tlv_data)[2]));
 881                         break;
 882
 883                 case IWM_UCODE_TLV_FW_MEM_SEG:
 884                         break;
 885
 886                 default:
 887                         device_printf(sc->sc_dev,
 888                             "%s: unknown firmware section %d, abort\n",
 889                             __func__, tlv_type);
 890                         error = EINVAL;
 891                         goto parse_out;
 892                 }
 893
 894                 len -= roundup(tlv_len, 4);
 895                 data += roundup(tlv_len, 4);
 896         }
 897
 898         KASSERT(error == 0, ("unhandled error"));
 899
 900  parse_out:
 901         if (error) {
 902                 device_printf(sc->sc_dev, "firmware parse error %d, "
 903                     "section type %d\n", error, tlv_type);
 904         }
 905
 906  out:
 907         if (error) {
 908                 fw->fw_status = IWM_FW_STATUS_NONE;
 909                 if (fw->fw_fp != NULL)
 910                         iwm_fw_info_free(fw);
 911         } else
 912                 fw->fw_status = IWM_FW_STATUS_DONE;
 913         wakeup(&sc->sc_fw);
 914
 915         return error;
 916 }
 917
 918 /*
 919  * DMA resource routines
 920  */
 921
 922 /* fwmem is used to load firmware onto the card */
 923 static int
 924 iwm_alloc_fwmem(struct iwm_softc *sc)
 925 {
 926         /* Must be aligned on a 16-byte boundary. */
 927         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->fw_dma,
 928             IWM_FH_MEM_TB_MAX_LENGTH, 16);
 929 }
 930
 931 /* tx scheduler rings.  not used? */
 932 static int
 933 iwm_alloc_sched(struct iwm_softc *sc)
 934 {
 935         /* TX scheduler rings must be aligned on a 1KB boundary. */
 936         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->sched_dma,
 937             nitems(sc->txq) * sizeof(struct iwm_agn_scd_bc_tbl), 1024);
 938 }
 939
 940 /* keep-warm page is used internally by the card.  see iwl-fh.h for more info */
 941 static int
 942 iwm_alloc_kw(struct iwm_softc *sc)
 943 {
 944         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->kw_dma, 4096, 4096);
 945 }
 946
 947 /* interrupt cause table */
 948 static int
 949 iwm_alloc_ict(struct iwm_softc *sc)
 950 {
 951         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->ict_dma,
 952             IWM_ICT_SIZE, 1<<IWM_ICT_PADDR_SHIFT);
 953 }
 954
 955 static int
 956 iwm_alloc_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring)
 957 {
 958         bus_size_t size;
 959         int i, error;
 960
 961         ring->cur = 0;
 962
 963         /* Allocate RX descriptors (256-byte aligned). */
 964         size = IWM_RX_RING_COUNT * sizeof(uint32_t);
 965         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->desc_dma, size, 256);
 966         if (error != 0) {
 967                 device_printf(sc->sc_dev,
 968                     "could not allocate RX ring DMA memory\n");
 969                 goto fail;
 970         }
 971         ring->desc = ring->desc_dma.vaddr;
 972
 973         /* Allocate RX status area (16-byte aligned). */
 974         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->stat_dma,
 975             sizeof(*ring->stat), 16);
 976         if (error != 0) {
 977                 device_printf(sc->sc_dev,
 978                     "could not allocate RX status DMA memory\n");
 979                 goto fail;
 980         }
 981         ring->stat = ring->stat_dma.vaddr;
 982
 983         /* Create RX buffer DMA tag. */
 984         error = bus_dma_tag_create(sc->sc_dmat, 1, 0,
 985             BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
 986             IWM_RBUF_SIZE, 1, IWM_RBUF_SIZE, 0, NULL, NULL, &ring->data_dmat);
 987         if (error != 0) {
 988                 device_printf(sc->sc_dev,
 989                     "%s: could not create RX buf DMA tag, error %d\n",
 990                     __func__, error);
 991                 goto fail;
 992         }
 993
 994         /* Allocate spare bus_dmamap_t for iwm_rx_addbuf() */
 995         error = bus_dmamap_create(ring->data_dmat, 0, &ring->spare_map);
 996         if (error != 0) {
 997                 device_printf(sc->sc_dev,
 998                     "%s: could not create RX buf DMA map, error %d\n",
 999                     __func__, error);
1000                 goto fail;
1001         }
1002         /*
1003          * Allocate and map RX buffers.
1004          */
1005         for (i = 0; i < IWM_RX_RING_COUNT; i++) {
1006                 struct iwm_rx_data *data = &ring->data[i];
1007                 error = bus_dmamap_create(ring->data_dmat, 0, &data->map);
1008                 if (error != 0) {
1009                         device_printf(sc->sc_dev,
1010                             "%s: could not create RX buf DMA map, error %d\n",
1011                             __func__, error);
1012                         goto fail;
1013                 }
1014                 data->m = NULL;
1015
1016                 if ((error = iwm_rx_addbuf(sc, IWM_RBUF_SIZE, i)) != 0) {
1017                         goto fail;
1018                 }
1019         }
1020         return 0;
1021
1022 fail:   iwm_free_rx_ring(sc, ring);
1023         return error;
1024 }
1025
1026 static void
1027 iwm_reset_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring)
1028 {
1029         /* Reset the ring state */
1030         ring->cur = 0;
1031
1032         /*
1033          * The hw rx ring index in shared memory must also be cleared,
1034          * otherwise the discrepancy can cause reprocessing chaos.
1035          */
1036         memset(sc->rxq.stat, 0, sizeof(*sc->rxq.stat));
1037 }
1038
1039 static void
1040 iwm_free_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring)
1041 {
1042         int i;
1043
1044         iwm_dma_contig_free(&ring->desc_dma);
1045         iwm_dma_contig_free(&ring->stat_dma);
1046
1047         for (i = 0; i < IWM_RX_RING_COUNT; i++) {
1048                 struct iwm_rx_data *data = &ring->data[i];
1049
1050                 if (data->m != NULL) {
1051                         bus_dmamap_sync(ring->data_dmat, data->map,
1052                             BUS_DMASYNC_POSTREAD);
1053                         bus_dmamap_unload(ring->data_dmat, data->map);
1054                         m_freem(data->m);
1055                         data->m = NULL;
1056                 }
1057                 if (data->map != NULL) {
1058                         bus_dmamap_destroy(ring->data_dmat, data->map);
1059                         data->map = NULL;
1060                 }
1061         }
1062         if (ring->spare_map != NULL) {
1063                 bus_dmamap_destroy(ring->data_dmat, ring->spare_map);
1064                 ring->spare_map = NULL;
1065         }
1066         if (ring->data_dmat != NULL) {
1067                 bus_dma_tag_destroy(ring->data_dmat);
1068                 ring->data_dmat = NULL;
1069         }
1070 }
1071
1072 static int
1073 iwm_alloc_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring, int qid)
1074 {
1075         bus_addr_t paddr;
1076         bus_size_t size;
1077         size_t maxsize;
1078         int nsegments;
1079         int i, error;
1080
1081         ring->qid = qid;
1082         ring->queued = 0;
1083         ring->cur = 0;
1084
1085         /* Allocate TX descriptors (256-byte aligned). */
1086         size = IWM_TX_RING_COUNT * sizeof (struct iwm_tfd);
1087         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->desc_dma, size, 256);
1088         if (error != 0) {
1089                 device_printf(sc->sc_dev,
1090                     "could not allocate TX ring DMA memory\n");
1091                 goto fail;
1092         }
1093         ring->desc = ring->desc_dma.vaddr;
1094
1095         /*
1096          * We only use rings 0 through 9 (4 EDCA + cmd) so there is no need
1097          * to allocate commands space for other rings.
1098          */
1099         if (qid > IWM_MVM_CMD_QUEUE)
1100                 return 0;
1101
1102         size = IWM_TX_RING_COUNT * sizeof(struct iwm_device_cmd);
1103         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->cmd_dma, size, 4);
1104         if (error != 0) {
1105                 device_printf(sc->sc_dev,
1106                     "could not allocate TX cmd DMA memory\n");
1107                 goto fail;
1108         }
1109         ring->cmd = ring->cmd_dma.vaddr;
1110
1111         /* FW commands may require more mapped space than packets. */
1112         if (qid == IWM_MVM_CMD_QUEUE) {
1113                 maxsize = IWM_RBUF_SIZE;
1114                 nsegments = 1;
1115         } else {
1116                 maxsize = MCLBYTES;
1117                 nsegments = IWM_MAX_SCATTER - 2;
1118         }
1119
1120         error = bus_dma_tag_create(sc->sc_dmat, 1, 0,
1121             BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, maxsize,
1122             nsegments, maxsize, 0, NULL, NULL, &ring->data_dmat);
1123         if (error != 0) {
1124                 device_printf(sc->sc_dev, "could not create TX buf DMA tag\n");
1125                 goto fail;
1126         }
1127
1128         paddr = ring->cmd_dma.paddr;
1129         for (i = 0; i < IWM_TX_RING_COUNT; i++) {
1130                 struct iwm_tx_data *data = &ring->data[i];
1131
1132                 data->cmd_paddr = paddr;
1133                 data->scratch_paddr = paddr + sizeof(struct iwm_cmd_header)
1134                     + offsetof(struct iwm_tx_cmd, scratch);
1135                 paddr += sizeof(struct iwm_device_cmd);
1136
1137                 error = bus_dmamap_create(ring->data_dmat, 0, &data->map);
1138                 if (error != 0) {
1139                         device_printf(sc->sc_dev,
1140                             "could not create TX buf DMA map\n");
1141                         goto fail;
1142                 }
1143         }
1144         KASSERT(paddr == ring->cmd_dma.paddr + size,
1145             ("invalid physical address"));
1146         return 0;
1147
1148 fail:   iwm_free_tx_ring(sc, ring);
1149         return error;
1150 }
1151
1152 static void
1153 iwm_reset_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring)
1154 {
1155         int i;
1156
1157         for (i = 0; i < IWM_TX_RING_COUNT; i++) {
1158                 struct iwm_tx_data *data = &ring->data[i];
1159
1160                 if (data->m != NULL) {
1161                         bus_dmamap_sync(ring->data_dmat, data->map,
1162                             BUS_DMASYNC_POSTWRITE);
1163                         bus_dmamap_unload(ring->data_dmat, data->map);
1164                         m_freem(data->m);
1165                         data->m = NULL;
1166                 }
1167         }
1168         /* Clear TX descriptors. */
1169         memset(ring->desc, 0, ring->desc_dma.size);
1170         bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map,
1171             BUS_DMASYNC_PREWRITE);
1172         sc->qfullmsk &= ~(1 << ring->qid);
1173         ring->queued = 0;
1174         ring->cur = 0;
1175
1176         if (ring->qid == IWM_MVM_CMD_QUEUE && sc->cmd_hold_nic_awake)
1177                 iwm_pcie_clear_cmd_in_flight(sc);
1178 }
1179
1180 static void
1181 iwm_free_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring)
1182 {
1183         int i;
1184
1185         iwm_dma_contig_free(&ring->desc_dma);
1186         iwm_dma_contig_free(&ring->cmd_dma);
1187
1188         for (i = 0; i < IWM_TX_RING_COUNT; i++) {
1189                 struct iwm_tx_data *data = &ring->data[i];
1190
1191                 if (data->m != NULL) {
1192                         bus_dmamap_sync(ring->data_dmat, data->map,
1193                             BUS_DMASYNC_POSTWRITE);
1194                         bus_dmamap_unload(ring->data_dmat, data->map);
1195                         m_freem(data->m);
1196                         data->m = NULL;
1197                 }
1198                 if (data->map != NULL) {
1199                         bus_dmamap_destroy(ring->data_dmat, data->map);
1200                         data->map = NULL;
1201                 }
1202         }
1203         if (ring->data_dmat != NULL) {
1204                 bus_dma_tag_destroy(ring->data_dmat);
1205                 ring->data_dmat = NULL;
1206         }
1207 }
1208
1209 /*
1210  * High-level hardware frobbing routines
1211  */
1212
1213 static void
1214 iwm_enable_interrupts(struct iwm_softc *sc)
1215 {
1216         sc->sc_intmask = IWM_CSR_INI_SET_MASK;
1217         IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask);
1218 }
1219
1220 static void
1221 iwm_restore_interrupts(struct iwm_softc *sc)
1222 {
1223         IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask);
1224 }
1225
1226 static void
1227 iwm_disable_interrupts(struct iwm_softc *sc)
1228 {
1229         /* disable interrupts */
1230         IWM_WRITE(sc, IWM_CSR_INT_MASK, 0);
1231
1232         /* acknowledge all interrupts */
1233         IWM_WRITE(sc, IWM_CSR_INT, ~0);
1234         IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, ~0);
1235 }
1236
1237 static void
1238 iwm_ict_reset(struct iwm_softc *sc)
1239 {
1240         iwm_disable_interrupts(sc);
1241
1242         /* Reset ICT table. */
1243         memset(sc->ict_dma.vaddr, 0, IWM_ICT_SIZE);
1244         sc->ict_cur = 0;
1245
1246         /* Set physical address of ICT table (4KB aligned). */
1247         IWM_WRITE(sc, IWM_CSR_DRAM_INT_TBL_REG,
1248             IWM_CSR_DRAM_INT_TBL_ENABLE
1249             | IWM_CSR_DRAM_INIT_TBL_WRITE_POINTER
1250             | IWM_CSR_DRAM_INIT_TBL_WRAP_CHECK
1251             | sc->ict_dma.paddr >> IWM_ICT_PADDR_SHIFT);
1252
1253         /* Switch to ICT interrupt mode in driver. */
1254         sc->sc_flags |= IWM_FLAG_USE_ICT;
1255
1256         /* Re-enable interrupts. */
1257         IWM_WRITE(sc, IWM_CSR_INT, ~0);
1258         iwm_enable_interrupts(sc);
1259 }
1260
1261 /* iwlwifi pcie/trans.c */
1262
1263 /*
1264  * Since this .. hard-resets things, it's time to actually
1265  * mark the first vap (if any) as having no mac context.
1266  * It's annoying, but since the driver is potentially being
1267  * stop/start'ed whilst active (thanks openbsd port!) we
1268  * have to correctly track this.
1269  */
1270 static void
1271 iwm_stop_device(struct iwm_softc *sc)
1272 {
1273         struct ieee80211com *ic = &sc->sc_ic;
1274         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
1275         int chnl, qid;
1276         uint32_t mask = 0;
1277
1278         /* tell the device to stop sending interrupts */
1279         iwm_disable_interrupts(sc);
1280
1281         /*
1282          * FreeBSD-local: mark the first vap as not-uploaded,
1283          * so the next transition through auth/assoc
1284          * will correctly populate the MAC context.
1285          */
1286         if (vap) {
1287                 struct iwm_vap *iv = IWM_VAP(vap);
1288                 iv->is_uploaded = 0;
1289         }
1290
1291         /* device going down, Stop using ICT table */
1292         sc->sc_flags &= ~IWM_FLAG_USE_ICT;
1293
1294         /* stop tx and rx.  tx and rx bits, as usual, are from if_iwn */
1295
1296         iwm_write_prph(sc, IWM_SCD_TXFACT, 0);
1297
1298         if (iwm_nic_lock(sc)) {
1299                 /* Stop each Tx DMA channel */
1300                 for (chnl = 0; chnl < IWM_FH_TCSR_CHNL_NUM; chnl++) {
1301                         IWM_WRITE(sc,
1302                             IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl), 0);
1303                         mask |= IWM_FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(chnl);
1304                 }
1305
1306                 /* Wait for DMA channels to be idle */
1307                 if (!iwm_poll_bit(sc, IWM_FH_TSSR_TX_STATUS_REG, mask, mask,
1308                     5000)) {
1309                         device_printf(sc->sc_dev,
1310                             "Failing on timeout while stopping DMA channel: [0x%08x]\n",
1311                             IWM_READ(sc, IWM_FH_TSSR_TX_STATUS_REG));
1312                 }
1313                 iwm_nic_unlock(sc);
1314         }
1315         iwm_pcie_rx_stop(sc);
1316
1317         /* Stop RX ring. */
1318         iwm_reset_rx_ring(sc, &sc->rxq);
1319
1320         /* Reset all TX rings. */
1321         for (qid = 0; qid < nitems(sc->txq); qid++)
1322                 iwm_reset_tx_ring(sc, &sc->txq[qid]);
1323
1324         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
1325                 /* Power-down device's busmaster DMA clocks */
1326                 iwm_write_prph(sc, IWM_APMG_CLK_DIS_REG,
1327                     IWM_APMG_CLK_VAL_DMA_CLK_RQT);
1328                 DELAY(5);
1329         }
1330
1331         /* Make sure (redundant) we've released our request to stay awake */
1332         IWM_CLRBITS(sc, IWM_CSR_GP_CNTRL,
1333             IWM_CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1334
1335         /* Stop the device, and put it in low power state */
1336         iwm_apm_stop(sc);
1337
1338         /* Upon stop, the APM issues an interrupt if HW RF kill is set.
1339          * Clean again the interrupt here
1340          */
1341         iwm_disable_interrupts(sc);
1342         /* stop and reset the on-board processor */
1343         IWM_WRITE(sc, IWM_CSR_RESET, IWM_CSR_RESET_REG_FLAG_SW_RESET);
1344
1345         /*
1346          * Even if we stop the HW, we still want the RF kill
1347          * interrupt
1348          */
1349         iwm_enable_rfkill_int(sc);
1350         iwm_check_rfkill(sc);
1351 }
1352
1353 /* iwlwifi: mvm/ops.c */
1354 static void
1355 iwm_mvm_nic_config(struct iwm_softc *sc)
1356 {
1357         uint8_t radio_cfg_type, radio_cfg_step, radio_cfg_dash;
1358         uint32_t reg_val = 0;
1359         uint32_t phy_config = iwm_mvm_get_phy_config(sc);
1360
1361         radio_cfg_type = (phy_config & IWM_FW_PHY_CFG_RADIO_TYPE) >>
1362             IWM_FW_PHY_CFG_RADIO_TYPE_POS;
1363         radio_cfg_step = (phy_config & IWM_FW_PHY_CFG_RADIO_STEP) >>
1364             IWM_FW_PHY_CFG_RADIO_STEP_POS;
1365         radio_cfg_dash = (phy_config & IWM_FW_PHY_CFG_RADIO_DASH) >>
1366             IWM_FW_PHY_CFG_RADIO_DASH_POS;
1367
1368         /* SKU control */
1369         reg_val |= IWM_CSR_HW_REV_STEP(sc->sc_hw_rev) <<
1370             IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_STEP;
1371         reg_val |= IWM_CSR_HW_REV_DASH(sc->sc_hw_rev) <<
1372             IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_DASH;
1373
1374         /* radio configuration */
1375         reg_val |= radio_cfg_type << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_TYPE;
1376         reg_val |= radio_cfg_step << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_STEP;
1377         reg_val |= radio_cfg_dash << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_DASH;
1378
1379         IWM_WRITE(sc, IWM_CSR_HW_IF_CONFIG_REG, reg_val);
1380
1381         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
1382             "Radio type=0x%x-0x%x-0x%x\n", radio_cfg_type,
1383             radio_cfg_step, radio_cfg_dash);
1384
1385         /*
1386          * W/A : NIC is stuck in a reset state after Early PCIe power off
1387          * (PCIe power is lost before PERST# is asserted), causing ME FW
1388          * to lose ownership and not being able to obtain it back.
1389          */
1390         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
1391                 iwm_set_bits_mask_prph(sc, IWM_APMG_PS_CTRL_REG,
1392                     IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS,
1393                     ~IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS);
1394         }
1395 }
1396
1397 static int
1398 iwm_nic_rx_init(struct iwm_softc *sc)
1399 {
1400         /*
1401          * Initialize RX ring.  This is from the iwn driver.
1402          */
1403         memset(sc->rxq.stat, 0, sizeof(*sc->rxq.stat));
1404
1405         /* Stop Rx DMA */
1406         iwm_pcie_rx_stop(sc);
1407
1408         if (!iwm_nic_lock(sc))
1409                 return EBUSY;
1410
1411         /* reset and flush pointers */
1412         IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_RBDCB_WPTR, 0);
1413         IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_FLUSH_RB_REQ, 0);
1414         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_RDPTR, 0);
1415         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_RBDCB_WPTR_REG, 0);
1416
1417         /* Set physical address of RX ring (256-byte aligned). */
1418         IWM_WRITE(sc,
1419             IWM_FH_RSCSR_CHNL0_RBDCB_BASE_REG, sc->rxq.desc_dma.paddr >> 8);
1420
1421         /* Set physical address of RX status (16-byte aligned). */
1422         IWM_WRITE(sc,
1423             IWM_FH_RSCSR_CHNL0_STTS_WPTR_REG, sc->rxq.stat_dma.paddr >> 4);
1424
1425         /* Enable RX. */
1426         IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_CONFIG_REG,
1427             IWM_FH_RCSR_RX_CONFIG_CHNL_EN_ENABLE_VAL            |
1428             IWM_FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY               |  /* HW bug */
1429             IWM_FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_INT_HOST_VAL   |
1430             IWM_FH_RCSR_CHNL0_RX_CONFIG_SINGLE_FRAME_MSK        |
1431             (IWM_RX_RB_TIMEOUT << IWM_FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS) |
1432             IWM_FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K            |
1433             IWM_RX_QUEUE_SIZE_LOG << IWM_FH_RCSR_RX_CONFIG_RBDCB_SIZE_POS);
1434
1435         IWM_WRITE_1(sc, IWM_CSR_INT_COALESCING, IWM_HOST_INT_TIMEOUT_DEF);
1436
1437         /* W/A for interrupt coalescing bug in 7260 and 3160 */
1438         if (sc->cfg->host_interrupt_operation_mode)
1439                 IWM_SETBITS(sc, IWM_CSR_INT_COALESCING, IWM_HOST_INT_OPER_MODE);
1440
1441         /*
1442          * Thus sayeth el jefe (iwlwifi) via a comment:
1443          *
1444          * This value should initially be 0 (before preparing any
1445          * RBs), should be 8 after preparing the first 8 RBs (for example)
1446          */
1447         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_WPTR, 8);
1448
1449         iwm_nic_unlock(sc);
1450
1451         return 0;
1452 }
1453
1454 static int
1455 iwm_nic_tx_init(struct iwm_softc *sc)
1456 {
1457         int qid;
1458
1459         if (!iwm_nic_lock(sc))
1460                 return EBUSY;
1461
1462         /* Deactivate TX scheduler. */
1463         iwm_write_prph(sc, IWM_SCD_TXFACT, 0);
1464
1465         /* Set physical address of "keep warm" page (16-byte aligned). */
1466         IWM_WRITE(sc, IWM_FH_KW_MEM_ADDR_REG, sc->kw_dma.paddr >> 4);
1467
1468         /* Initialize TX rings. */
1469         for (qid = 0; qid < nitems(sc->txq); qid++) {
1470                 struct iwm_tx_ring *txq = &sc->txq[qid];
1471
1472                 /* Set physical address of TX ring (256-byte aligned). */
1473                 IWM_WRITE(sc, IWM_FH_MEM_CBBC_QUEUE(qid),
1474                     txq->desc_dma.paddr >> 8);
1475                 IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
1476                     "%s: loading ring %d descriptors (%p) at %lx\n",
1477                     __func__,
1478                     qid, txq->desc,
1479                     (unsigned long) (txq->desc_dma.paddr >> 8));
1480         }
1481
1482         iwm_write_prph(sc, IWM_SCD_GP_CTRL, IWM_SCD_GP_CTRL_AUTO_ACTIVE_MODE);
1483
1484         iwm_nic_unlock(sc);
1485
1486         return 0;
1487 }
1488
1489 static int
1490 iwm_nic_init(struct iwm_softc *sc)
1491 {
1492         int error;
1493
1494         iwm_apm_init(sc);
1495         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
1496                 iwm_set_pwr(sc);
1497
1498         iwm_mvm_nic_config(sc);
1499
1500         if ((error = iwm_nic_rx_init(sc)) != 0)
1501                 return error;
1502
1503         /*
1504          * Ditto for TX, from iwn
1505          */
1506         if ((error = iwm_nic_tx_init(sc)) != 0)
1507                 return error;
1508
1509         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
1510             "%s: shadow registers enabled\n", __func__);
1511         IWM_SETBITS(sc, IWM_CSR_MAC_SHADOW_REG_CTRL, 0x800fffff);
1512
1513         return 0;
1514 }
1515
1516 const uint8_t iwm_mvm_ac_to_tx_fifo[] = {
1517         IWM_MVM_TX_FIFO_VO,
1518         IWM_MVM_TX_FIFO_VI,
1519         IWM_MVM_TX_FIFO_BE,
1520         IWM_MVM_TX_FIFO_BK,
1521 };
1522
1523 static int
1524 iwm_enable_txq(struct iwm_softc *sc, int sta_id, int qid, int fifo)
1525 {
1526         if (!iwm_nic_lock(sc)) {
1527                 device_printf(sc->sc_dev,
1528                     "%s: cannot enable txq %d\n",
1529                     __func__,
1530                     qid);
1531                 return EBUSY;
1532         }
1533
1534         IWM_WRITE(sc, IWM_HBUS_TARG_WRPTR, qid << 8 | 0);
1535
1536         if (qid == IWM_MVM_CMD_QUEUE) {
1537                 /* unactivate before configuration */
1538                 iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid),
1539                     (0 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE)
1540                     | (1 << IWM_SCD_QUEUE_STTS_REG_POS_SCD_ACT_EN));
1541
1542                 iwm_nic_unlock(sc);
1543
1544                 iwm_clear_bits_prph(sc, IWM_SCD_AGGR_SEL, (1 << qid));
1545
1546                 if (!iwm_nic_lock(sc)) {
1547                         device_printf(sc->sc_dev,
1548                             "%s: cannot enable txq %d\n", __func__, qid);
1549                         return EBUSY;
1550                 }
1551                 iwm_write_prph(sc, IWM_SCD_QUEUE_RDPTR(qid), 0);
1552                 iwm_nic_unlock(sc);
1553
1554                 iwm_write_mem32(sc, sc->scd_base_addr + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid), 0);
1555                 /* Set scheduler window size and frame limit. */
1556                 iwm_write_mem32(sc,
1557                     sc->scd_base_addr + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid) +
1558                     sizeof(uint32_t),
1559                     ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_POS) &
1560                     IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_MSK) |
1561                     ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_POS) &
1562                     IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_MSK));
1563
1564                 if (!iwm_nic_lock(sc)) {
1565                         device_printf(sc->sc_dev,
1566                             "%s: cannot enable txq %d\n", __func__, qid);
1567                         return EBUSY;
1568                 }
1569                 iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid),
1570                     (1 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE) |
1571                     (fifo << IWM_SCD_QUEUE_STTS_REG_POS_TXF) |
1572                     (1 << IWM_SCD_QUEUE_STTS_REG_POS_WSL) |
1573                     IWM_SCD_QUEUE_STTS_REG_MSK);
1574         } else {
1575                 struct iwm_scd_txq_cfg_cmd cmd;
1576                 int error;
1577
1578                 iwm_nic_unlock(sc);
1579
1580                 memset(&cmd, 0, sizeof(cmd));
1581                 cmd.scd_queue = qid;
1582                 cmd.enable = 1;
1583                 cmd.sta_id = sta_id;
1584                 cmd.tx_fifo = fifo;
1585                 cmd.aggregate = 0;
1586                 cmd.window = IWM_FRAME_LIMIT;
1587
1588                 error = iwm_mvm_send_cmd_pdu(sc, IWM_SCD_QUEUE_CFG, IWM_CMD_SYNC,
1589                     sizeof(cmd), &cmd);
1590                 if (error) {
1591                         device_printf(sc->sc_dev,
1592                             "cannot enable txq %d\n", qid);
1593                         return error;
1594                 }
1595
1596                 if (!iwm_nic_lock(sc))
1597                         return EBUSY;
1598         }
1599
1600         iwm_write_prph(sc, IWM_SCD_EN_CTRL,
1601             iwm_read_prph(sc, IWM_SCD_EN_CTRL) | qid);
1602
1603         iwm_nic_unlock(sc);
1604
1605         IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "%s: enabled txq %d FIFO %d\n",
1606             __func__, qid, fifo);
1607
1608         return 0;
1609 }
1610
1611 static int
1612 iwm_trans_pcie_fw_alive(struct iwm_softc *sc, uint32_t scd_base_addr)
1613 {
1614         int error, chnl;
1615
1616         int clear_dwords = (IWM_SCD_TRANS_TBL_MEM_UPPER_BOUND -
1617             IWM_SCD_CONTEXT_MEM_LOWER_BOUND) / sizeof(uint32_t);
1618
1619         if (!iwm_nic_lock(sc))
1620                 return EBUSY;
1621
1622         iwm_ict_reset(sc);
1623
1624         iwm_nic_unlock(sc);
1625
1626         sc->scd_base_addr = iwm_read_prph(sc, IWM_SCD_SRAM_BASE_ADDR);
1627         if (scd_base_addr != 0 &&
1628             scd_base_addr != sc->scd_base_addr) {
1629                 device_printf(sc->sc_dev,
1630                     "%s: sched addr mismatch: alive: 0x%x prph: 0x%x\n",
1631                     __func__, sc->scd_base_addr, scd_base_addr);
1632         }
1633
1634         /* reset context data, TX status and translation data */
1635         error = iwm_write_mem(sc,
1636             sc->scd_base_addr + IWM_SCD_CONTEXT_MEM_LOWER_BOUND,
1637             NULL, clear_dwords);
1638         if (error)
1639                 return EBUSY;
1640
1641         if (!iwm_nic_lock(sc))
1642                 return EBUSY;
1643
1644         /* Set physical address of TX scheduler rings (1KB aligned). */
1645         iwm_write_prph(sc, IWM_SCD_DRAM_BASE_ADDR, sc->sched_dma.paddr >> 10);
1646
1647         iwm_write_prph(sc, IWM_SCD_CHAINEXT_EN, 0);
1648
1649         iwm_nic_unlock(sc);
1650
1651         /* enable command channel */
1652         error = iwm_enable_txq(sc, 0 /* unused */, IWM_MVM_CMD_QUEUE, 7);
1653         if (error)
1654                 return error;
1655
1656         if (!iwm_nic_lock(sc))
1657                 return EBUSY;
1658
1659         iwm_write_prph(sc, IWM_SCD_TXFACT, 0xff);
1660
1661         /* Enable DMA channels. */
1662         for (chnl = 0; chnl < IWM_FH_TCSR_CHNL_NUM; chnl++) {
1663                 IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl),
1664                     IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE |
1665                     IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE);
1666         }
1667
1668         IWM_SETBITS(sc, IWM_FH_TX_CHICKEN_BITS_REG,
1669             IWM_FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN);
1670
1671         iwm_nic_unlock(sc);
1672
1673         /* Enable L1-Active */
1674         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
1675                 iwm_clear_bits_prph(sc, IWM_APMG_PCIDEV_STT_REG,
1676                     IWM_APMG_PCIDEV_STT_VAL_L1_ACT_DIS);
1677         }
1678
1679         return error;
1680 }
1681
1682 /*
1683  * NVM read access and content parsing.  We do not support
1684  * external NVM or writing NVM.
1685  * iwlwifi/mvm/nvm.c
1686  */
1687
1688 /* Default NVM size to read */
1689 #define IWM_NVM_DEFAULT_CHUNK_SIZE      (2*1024)
1690
1691 #define IWM_NVM_WRITE_OPCODE 1
1692 #define IWM_NVM_READ_OPCODE 0
1693
1694 /* load nvm chunk response */
1695 enum {
1696         IWM_READ_NVM_CHUNK_SUCCEED = 0,
1697         IWM_READ_NVM_CHUNK_NOT_VALID_ADDRESS = 1
1698 };
1699
1700 static int
1701 iwm_nvm_read_chunk(struct iwm_softc *sc, uint16_t section,
1702         uint16_t offset, uint16_t length, uint8_t *data, uint16_t *len)
1703 {
1704         struct iwm_nvm_access_cmd nvm_access_cmd = {
1705                 .offset = htole16(offset),
1706                 .length = htole16(length),
1707                 .type = htole16(section),
1708                 .op_code = IWM_NVM_READ_OPCODE,
1709         };
1710         struct iwm_nvm_access_resp *nvm_resp;
1711         struct iwm_rx_packet *pkt;
1712         struct iwm_host_cmd cmd = {
1713                 .id = IWM_NVM_ACCESS_CMD,
1714                 .flags = IWM_CMD_WANT_SKB | IWM_CMD_SEND_IN_RFKILL,
1715                 .data = { &nvm_access_cmd, },
1716         };
1717         int ret, bytes_read, offset_read;
1718         uint8_t *resp_data;
1719
1720         cmd.len[0] = sizeof(struct iwm_nvm_access_cmd);
1721
1722         ret = iwm_send_cmd(sc, &cmd);
1723         if (ret) {
1724                 device_printf(sc->sc_dev,
1725                     "Could not send NVM_ACCESS command (error=%d)\n", ret);
1726                 return ret;
1727         }
1728
1729         pkt = cmd.resp_pkt;
1730
1731         /* Extract NVM response */
1732         nvm_resp = (void *)pkt->data;
1733         ret = le16toh(nvm_resp->status);
1734         bytes_read = le16toh(nvm_resp->length);
1735         offset_read = le16toh(nvm_resp->offset);
1736         resp_data = nvm_resp->data;
1737         if (ret) {
1738                 if ((offset != 0) &&
1739                     (ret == IWM_READ_NVM_CHUNK_NOT_VALID_ADDRESS)) {
1740                         /*
1741                          * meaning of NOT_VALID_ADDRESS:
1742                          * driver try to read chunk from address that is
1743                          * multiple of 2K and got an error since addr is empty.
1744                          * meaning of (offset != 0): driver already
1745                          * read valid data from another chunk so this case
1746                          * is not an error.
1747                          */
1748                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1749                                     "NVM access command failed on offset 0x%x since that section size is multiple 2K\n",
1750                                     offset);
1751                         *len = 0;
1752                         ret = 0;
1753                 } else {
1754                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1755                                     "NVM access command failed with status %d\n", ret);
1756                         ret = EIO;
1757                 }
1758                 goto exit;
1759         }
1760
1761         if (offset_read != offset) {
1762                 device_printf(sc->sc_dev,
1763                     "NVM ACCESS response with invalid offset %d\n",
1764                     offset_read);
1765                 ret = EINVAL;
1766                 goto exit;
1767         }
1768
1769         if (bytes_read > length) {
1770                 device_printf(sc->sc_dev,
1771                     "NVM ACCESS response with too much data "
1772                     "(%d bytes requested, %d bytes received)\n",
1773                     length, bytes_read);
1774                 ret = EINVAL;
1775                 goto exit;
1776         }
1777
1778         /* Write data to NVM */
1779         memcpy(data + offset, resp_data, bytes_read);
1780         *len = bytes_read;
1781
1782  exit:
1783         iwm_free_resp(sc, &cmd);
1784         return ret;
1785 }
1786
1787 /*
1788  * Reads an NVM section completely.
1789  * NICs prior to 7000 family don't have a real NVM, but just read
1790  * section 0 which is the EEPROM. Because the EEPROM reading is unlimited
1791  * by uCode, we need to manually check in this case that we don't
1792  * overflow and try to read more than the EEPROM size.
1793  * For 7000 family NICs, we supply the maximal size we can read, and
1794  * the uCode fills the response with as much data as we can,
1795  * without overflowing, so no check is needed.
1796  */
1797 static int
1798 iwm_nvm_read_section(struct iwm_softc *sc,
1799         uint16_t section, uint8_t *data, uint16_t *len, uint32_t size_read)
1800 {
1801         uint16_t seglen, length, offset = 0;
1802         int ret;
1803
1804         /* Set nvm section read length */
1805         length = IWM_NVM_DEFAULT_CHUNK_SIZE;
1806
1807         seglen = length;
1808
1809         /* Read the NVM until exhausted (reading less than requested) */
1810         while (seglen == length) {
1811                 /* Check no memory assumptions fail and cause an overflow */
1812                 if ((size_read + offset + length) >
1813                     sc->cfg->eeprom_size) {
1814                         device_printf(sc->sc_dev,
1815                             "EEPROM size is too small for NVM\n");
1816                         return ENOBUFS;
1817                 }
1818
1819                 ret = iwm_nvm_read_chunk(sc, section, offset, length, data, &seglen);
1820                 if (ret) {
1821                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1822                                     "Cannot read NVM from section %d offset %d, length %d\n",
1823                                     section, offset, length);
1824                         return ret;
1825                 }
1826                 offset += seglen;
1827         }
1828
1829         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1830                     "NVM section %d read completed\n", section);
1831         *len = offset;
1832         return 0;
1833 }
1834
1835 /*
1836  * BEGIN IWM_NVM_PARSE
1837  */
1838
1839 /* iwlwifi/iwl-nvm-parse.c */
1840
1841 /* NVM offsets (in words) definitions */
1842 enum iwm_nvm_offsets {
1843         /* NVM HW-Section offset (in words) definitions */
1844         IWM_HW_ADDR = 0x15,
1845
1846 /* NVM SW-Section offset (in words) definitions */
1847         IWM_NVM_SW_SECTION = 0x1C0,
1848         IWM_NVM_VERSION = 0,
1849         IWM_RADIO_CFG = 1,
1850         IWM_SKU = 2,
1851         IWM_N_HW_ADDRS = 3,
1852         IWM_NVM_CHANNELS = 0x1E0 - IWM_NVM_SW_SECTION,
1853
1854 /* NVM calibration section offset (in words) definitions */
1855         IWM_NVM_CALIB_SECTION = 0x2B8,
1856         IWM_XTAL_CALIB = 0x316 - IWM_NVM_CALIB_SECTION
1857 };
1858
1859 enum iwm_8000_nvm_offsets {
1860         /* NVM HW-Section offset (in words) definitions */
1861         IWM_HW_ADDR0_WFPM_8000 = 0x12,
1862         IWM_HW_ADDR1_WFPM_8000 = 0x16,
1863         IWM_HW_ADDR0_PCIE_8000 = 0x8A,
1864         IWM_HW_ADDR1_PCIE_8000 = 0x8E,
1865         IWM_MAC_ADDRESS_OVERRIDE_8000 = 1,
1866
1867         /* NVM SW-Section offset (in words) definitions */
1868         IWM_NVM_SW_SECTION_8000 = 0x1C0,
1869         IWM_NVM_VERSION_8000 = 0,
1870         IWM_RADIO_CFG_8000 = 0,
1871         IWM_SKU_8000 = 2,
1872         IWM_N_HW_ADDRS_8000 = 3,
1873
1874         /* NVM REGULATORY -Section offset (in words) definitions */
1875         IWM_NVM_CHANNELS_8000 = 0,
1876         IWM_NVM_LAR_OFFSET_8000_OLD = 0x4C7,
1877         IWM_NVM_LAR_OFFSET_8000 = 0x507,
1878         IWM_NVM_LAR_ENABLED_8000 = 0x7,
1879
1880         /* NVM calibration section offset (in words) definitions */
1881         IWM_NVM_CALIB_SECTION_8000 = 0x2B8,
1882         IWM_XTAL_CALIB_8000 = 0x316 - IWM_NVM_CALIB_SECTION_8000
1883 };
1884
1885 /* SKU Capabilities (actual values from NVM definition) */
1886 enum nvm_sku_bits {
1887         IWM_NVM_SKU_CAP_BAND_24GHZ      = (1 << 0),
1888         IWM_NVM_SKU_CAP_BAND_52GHZ      = (1 << 1),
1889         IWM_NVM_SKU_CAP_11N_ENABLE      = (1 << 2),
1890         IWM_NVM_SKU_CAP_11AC_ENABLE     = (1 << 3),
1891 };
1892
1893 /* radio config bits (actual values from NVM definition) */
1894 #define IWM_NVM_RF_CFG_DASH_MSK(x)   (x & 0x3)         /* bits 0-1   */
1895 #define IWM_NVM_RF_CFG_STEP_MSK(x)   ((x >> 2)  & 0x3) /* bits 2-3   */
1896 #define IWM_NVM_RF_CFG_TYPE_MSK(x)   ((x >> 4)  & 0x3) /* bits 4-5   */
1897 #define IWM_NVM_RF_CFG_PNUM_MSK(x)   ((x >> 6)  & 0x3) /* bits 6-7   */
1898 #define IWM_NVM_RF_CFG_TX_ANT_MSK(x) ((x >> 8)  & 0xF) /* bits 8-11  */
1899 #define IWM_NVM_RF_CFG_RX_ANT_MSK(x) ((x >> 12) & 0xF) /* bits 12-15 */
1900
1901 #define IWM_NVM_RF_CFG_FLAVOR_MSK_8000(x)       (x & 0xF)
1902 #define IWM_NVM_RF_CFG_DASH_MSK_8000(x)         ((x >> 4) & 0xF)
1903 #define IWM_NVM_RF_CFG_STEP_MSK_8000(x)         ((x >> 8) & 0xF)
1904 #define IWM_NVM_RF_CFG_TYPE_MSK_8000(x)         ((x >> 12) & 0xFFF)
1905 #define IWM_NVM_RF_CFG_TX_ANT_MSK_8000(x)       ((x >> 24) & 0xF)
1906 #define IWM_NVM_RF_CFG_RX_ANT_MSK_8000(x)       ((x >> 28) & 0xF)
1907
1908 #define DEFAULT_MAX_TX_POWER 16
1909
1910 /**
1911  * enum iwm_nvm_channel_flags - channel flags in NVM
1912  * @IWM_NVM_CHANNEL_VALID: channel is usable for this SKU/geo
1913  * @IWM_NVM_CHANNEL_IBSS: usable as an IBSS channel
1914  * @IWM_NVM_CHANNEL_ACTIVE: active scanning allowed
1915  * @IWM_NVM_CHANNEL_RADAR: radar detection required
1916  * XXX cannot find this (DFS) flag in iwm-nvm-parse.c
1917  * @IWM_NVM_CHANNEL_DFS: dynamic freq selection candidate
1918  * @IWM_NVM_CHANNEL_WIDE: 20 MHz channel okay (?)
1919  * @IWM_NVM_CHANNEL_40MHZ: 40 MHz channel okay (?)
1920  * @IWM_NVM_CHANNEL_80MHZ: 80 MHz channel okay (?)
1921  * @IWM_NVM_CHANNEL_160MHZ: 160 MHz channel okay (?)
1922  */
1923 enum iwm_nvm_channel_flags {
1924         IWM_NVM_CHANNEL_VALID = (1 << 0),
1925         IWM_NVM_CHANNEL_IBSS = (1 << 1),
1926         IWM_NVM_CHANNEL_ACTIVE = (1 << 3),
1927         IWM_NVM_CHANNEL_RADAR = (1 << 4),
1928         IWM_NVM_CHANNEL_DFS = (1 << 7),
1929         IWM_NVM_CHANNEL_WIDE = (1 << 8),
1930         IWM_NVM_CHANNEL_40MHZ = (1 << 9),
1931         IWM_NVM_CHANNEL_80MHZ = (1 << 10),
1932         IWM_NVM_CHANNEL_160MHZ = (1 << 11),
1933 };
1934
1935 /*
1936  * Translate EEPROM flags to net80211.
1937  */
1938 static uint32_t
1939 iwm_eeprom_channel_flags(uint16_t ch_flags)
1940 {
1941         uint32_t nflags;
1942
1943         nflags = 0;
1944         if ((ch_flags & IWM_NVM_CHANNEL_ACTIVE) == 0)
1945                 nflags |= IEEE80211_CHAN_PASSIVE;
1946         if ((ch_flags & IWM_NVM_CHANNEL_IBSS) == 0)
1947                 nflags |= IEEE80211_CHAN_NOADHOC;
1948         if (ch_flags & IWM_NVM_CHANNEL_RADAR) {
1949                 nflags |= IEEE80211_CHAN_DFS;
1950                 /* Just in case. */
1951                 nflags |= IEEE80211_CHAN_NOADHOC;
1952         }
1953
1954         return (nflags);
1955 }
1956
1957 static void
1958 iwm_add_channel_band(struct iwm_softc *sc, struct ieee80211_channel chans[],
1959     int maxchans, int *nchans, int ch_idx, size_t ch_num,
1960     const uint8_t bands[])
1961 {
1962         const uint16_t * const nvm_ch_flags = sc->nvm_data->nvm_ch_flags;
1963         uint32_t nflags;
1964         uint16_t ch_flags;
1965         uint8_t ieee;
1966         int error;
1967
1968         for (; ch_idx < ch_num; ch_idx++) {
1969                 ch_flags = le16_to_cpup(nvm_ch_flags + ch_idx);
1970                 if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
1971                         ieee = iwm_nvm_channels[ch_idx];
1972                 else
1973                         ieee = iwm_nvm_channels_8000[ch_idx];
1974
1975                 if (!(ch_flags & IWM_NVM_CHANNEL_VALID)) {
1976                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM,
1977                             "Ch. %d Flags %x [%sGHz] - No traffic\n",
1978                             ieee, ch_flags,
1979                             (ch_idx >= IWM_NUM_2GHZ_CHANNELS) ?
1980                             "5.2" : "2.4");
1981                         continue;
1982                 }
1983
1984                 nflags = iwm_eeprom_channel_flags(ch_flags);
1985                 error = ieee80211_add_channel(chans, maxchans, nchans,
1986                     ieee, 0, 0, nflags, bands);
1987                 if (error != 0)
1988                         break;
1989
1990                 IWM_DPRINTF(sc, IWM_DEBUG_EEPROM,
1991                     "Ch. %d Flags %x [%sGHz] - Added\n",
1992                     ieee, ch_flags,
1993                     (ch_idx >= IWM_NUM_2GHZ_CHANNELS) ?
1994                     "5.2" : "2.4");
1995         }
1996 }
1997
1998 static void
1999 iwm_init_channel_map(struct ieee80211com *ic, int maxchans, int *nchans,
2000     struct ieee80211_channel chans[])
2001 {
2002         struct iwm_softc *sc = ic->ic_softc;
2003         struct iwm_nvm_data *data = sc->nvm_data;
2004         uint8_t bands[IEEE80211_MODE_BYTES];
2005         size_t ch_num;
2006
2007         memset(bands, 0, sizeof(bands));
2008         /* 1-13: 11b/g channels. */
2009         setbit(bands, IEEE80211_MODE_11B);
2010         setbit(bands, IEEE80211_MODE_11G);
2011         iwm_add_channel_band(sc, chans, maxchans, nchans, 0,
2012             IWM_NUM_2GHZ_CHANNELS - 1, bands);
2013
2014         /* 14: 11b channel only. */
2015         clrbit(bands, IEEE80211_MODE_11G);
2016         iwm_add_channel_band(sc, chans, maxchans, nchans,
2017             IWM_NUM_2GHZ_CHANNELS - 1, IWM_NUM_2GHZ_CHANNELS, bands);
2018
2019         if (data->sku_cap_band_52GHz_enable) {
2020                 if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
2021                         ch_num = nitems(iwm_nvm_channels);
2022                 else
2023                         ch_num = nitems(iwm_nvm_channels_8000);
2024                 memset(bands, 0, sizeof(bands));
2025                 setbit(bands, IEEE80211_MODE_11A);
2026                 iwm_add_channel_band(sc, chans, maxchans, nchans,
2027                     IWM_NUM_2GHZ_CHANNELS, ch_num, bands);
2028         }
2029 }
2030
2031 static void
2032 iwm_set_hw_address_family_8000(struct iwm_softc *sc, struct iwm_nvm_data *data,
2033         const uint16_t *mac_override, const uint16_t *nvm_hw)
2034 {
2035         const uint8_t *hw_addr;
2036
2037         if (mac_override) {
2038                 static const uint8_t reserved_mac[] = {
2039                         0x02, 0xcc, 0xaa, 0xff, 0xee, 0x00
2040                 };
2041
2042                 hw_addr = (const uint8_t *)(mac_override +
2043                                  IWM_MAC_ADDRESS_OVERRIDE_8000);
2044
2045                 /*
2046                  * Store the MAC address from MAO section.
2047                  * No byte swapping is required in MAO section
2048                  */
2049                 IEEE80211_ADDR_COPY(data->hw_addr, hw_addr);
2050
2051                 /*
2052                  * Force the use of the OTP MAC address in case of reserved MAC
2053                  * address in the NVM, or if address is given but invalid.
2054                  */
2055                 if (!IEEE80211_ADDR_EQ(reserved_mac, hw_addr) &&
2056                     !IEEE80211_ADDR_EQ(ieee80211broadcastaddr, data->hw_addr) &&
2057                     iwm_is_valid_ether_addr(data->hw_addr) &&
2058                     !IEEE80211_IS_MULTICAST(data->hw_addr))
2059                         return;
2060
2061                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2062                     "%s: mac address from nvm override section invalid\n",
2063                     __func__);
2064         }
2065
2066         if (nvm_hw) {
2067                 /* read the mac address from WFMP registers */
2068                 uint32_t mac_addr0 =
2069                     htole32(iwm_read_prph(sc, IWM_WFMP_MAC_ADDR_0));
2070                 uint32_t mac_addr1 =
2071                     htole32(iwm_read_prph(sc, IWM_WFMP_MAC_ADDR_1));
2072
2073                 hw_addr = (const uint8_t *)&mac_addr0;
2074                 data->hw_addr[0] = hw_addr[3];
2075                 data->hw_addr[1] = hw_addr[2];
2076                 data->hw_addr[2] = hw_addr[1];
2077                 data->hw_addr[3] = hw_addr[0];
2078
2079                 hw_addr = (const uint8_t *)&mac_addr1;
2080                 data->hw_addr[4] = hw_addr[1];
2081                 data->hw_addr[5] = hw_addr[0];
2082
2083                 return;
2084         }
2085
2086         device_printf(sc->sc_dev, "%s: mac address not found\n", __func__);
2087         memset(data->hw_addr, 0, sizeof(data->hw_addr));
2088 }
2089
2090 static int
2091 iwm_get_sku(const struct iwm_softc *sc, const uint16_t *nvm_sw,
2092             const uint16_t *phy_sku)
2093 {
2094         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2095                 return le16_to_cpup(nvm_sw + IWM_SKU);
2096
2097         return le32_to_cpup((const uint32_t *)(phy_sku + IWM_SKU_8000));
2098 }
2099
2100 static int
2101 iwm_get_nvm_version(const struct iwm_softc *sc, const uint16_t *nvm_sw)
2102 {
2103         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2104                 return le16_to_cpup(nvm_sw + IWM_NVM_VERSION);
2105         else
2106                 return le32_to_cpup((const uint32_t *)(nvm_sw +
2107                                                 IWM_NVM_VERSION_8000));
2108 }
2109
2110 static int
2111 iwm_get_radio_cfg(const struct iwm_softc *sc, const uint16_t *nvm_sw,
2112                   const uint16_t *phy_sku)
2113 {
2114         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2115                 return le16_to_cpup(nvm_sw + IWM_RADIO_CFG);
2116
2117         return le32_to_cpup((const uint32_t *)(phy_sku + IWM_RADIO_CFG_8000));
2118 }
2119
2120 static int
2121 iwm_get_n_hw_addrs(const struct iwm_softc *sc, const uint16_t *nvm_sw)
2122 {
2123         int n_hw_addr;
2124
2125         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2126                 return le16_to_cpup(nvm_sw + IWM_N_HW_ADDRS);
2127
2128         n_hw_addr = le32_to_cpup((const uint32_t *)(nvm_sw + IWM_N_HW_ADDRS_8000));
2129
2130         return n_hw_addr & IWM_N_HW_ADDR_MASK;
2131 }
2132
2133 static void
2134 iwm_set_radio_cfg(const struct iwm_softc *sc, struct iwm_nvm_data *data,
2135                   uint32_t radio_cfg)
2136 {
2137         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
2138                 data->radio_cfg_type = IWM_NVM_RF_CFG_TYPE_MSK(radio_cfg);
2139                 data->radio_cfg_step = IWM_NVM_RF_CFG_STEP_MSK(radio_cfg);
2140                 data->radio_cfg_dash = IWM_NVM_RF_CFG_DASH_MSK(radio_cfg);
2141                 data->radio_cfg_pnum = IWM_NVM_RF_CFG_PNUM_MSK(radio_cfg);
2142                 return;
2143         }
2144
2145         /* set the radio configuration for family 8000 */
2146         data->radio_cfg_type = IWM_NVM_RF_CFG_TYPE_MSK_8000(radio_cfg);
2147         data->radio_cfg_step = IWM_NVM_RF_CFG_STEP_MSK_8000(radio_cfg);
2148         data->radio_cfg_dash = IWM_NVM_RF_CFG_DASH_MSK_8000(radio_cfg);
2149         data->radio_cfg_pnum = IWM_NVM_RF_CFG_FLAVOR_MSK_8000(radio_cfg);
2150         data->valid_tx_ant = IWM_NVM_RF_CFG_TX_ANT_MSK_8000(radio_cfg);
2151         data->valid_rx_ant = IWM_NVM_RF_CFG_RX_ANT_MSK_8000(radio_cfg);
2152 }
2153
2154 static int
2155 iwm_set_hw_address(struct iwm_softc *sc, struct iwm_nvm_data *data,
2156                    const uint16_t *nvm_hw, const uint16_t *mac_override)
2157 {
2158 #ifdef notyet /* for FAMILY 9000 */
2159         if (cfg->mac_addr_from_csr) {
2160                 iwm_set_hw_address_from_csr(sc, data);
2161         } else
2162 #endif
2163         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
2164                 const uint8_t *hw_addr = (const uint8_t *)(nvm_hw + IWM_HW_ADDR);
2165
2166                 /* The byte order is little endian 16 bit, meaning 214365 */
2167                 data->hw_addr[0] = hw_addr[1];
2168                 data->hw_addr[1] = hw_addr[0];
2169                 data->hw_addr[2] = hw_addr[3];
2170                 data->hw_addr[3] = hw_addr[2];
2171                 data->hw_addr[4] = hw_addr[5];
2172                 data->hw_addr[5] = hw_addr[4];
2173         } else {
2174                 iwm_set_hw_address_family_8000(sc, data, mac_override, nvm_hw);
2175         }
2176
2177         if (!iwm_is_valid_ether_addr(data->hw_addr)) {
2178                 device_printf(sc->sc_dev, "no valid mac address was found\n");
2179                 return EINVAL;
2180         }
2181
2182         return 0;
2183 }
2184
2185 static struct iwm_nvm_data *
2186 iwm_parse_nvm_data(struct iwm_softc *sc,
2187                    const uint16_t *nvm_hw, const uint16_t *nvm_sw,
2188                    const uint16_t *nvm_calib, const uint16_t *mac_override,
2189                    const uint16_t *phy_sku, const uint16_t *regulatory)
2190 {
2191         struct iwm_nvm_data *data;
2192         uint32_t sku, radio_cfg;
2193
2194         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
2195                 data = malloc(sizeof(*data) +
2196                     IWM_NUM_CHANNELS * sizeof(uint16_t),
2197                     M_DEVBUF, M_NOWAIT | M_ZERO);
2198         } else {
2199                 data = malloc(sizeof(*data) +
2200                     IWM_NUM_CHANNELS_8000 * sizeof(uint16_t),
2201                     M_DEVBUF, M_NOWAIT | M_ZERO);
2202         }
2203         if (!data)
2204                 return NULL;
2205
2206         data->nvm_version = iwm_get_nvm_version(sc, nvm_sw);
2207
2208         radio_cfg = iwm_get_radio_cfg(sc, nvm_sw, phy_sku);
2209         iwm_set_radio_cfg(sc, data, radio_cfg);
2210
2211         sku = iwm_get_sku(sc, nvm_sw, phy_sku);
2212         data->sku_cap_band_24GHz_enable = sku & IWM_NVM_SKU_CAP_BAND_24GHZ;
2213         data->sku_cap_band_52GHz_enable = sku & IWM_NVM_SKU_CAP_BAND_52GHZ;
2214         data->sku_cap_11n_enable = 0;
2215
2216         data->n_hw_addrs = iwm_get_n_hw_addrs(sc, nvm_sw);
2217
2218         /* If no valid mac address was found - bail out */
2219         if (iwm_set_hw_address(sc, data, nvm_hw, mac_override)) {
2220                 free(data, M_DEVBUF);
2221                 return NULL;
2222         }
2223
2224         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
2225                 memcpy(data->nvm_ch_flags, &nvm_sw[IWM_NVM_CHANNELS],
2226                     IWM_NUM_CHANNELS * sizeof(uint16_t));
2227         } else {
2228                 memcpy(data->nvm_ch_flags, &regulatory[IWM_NVM_CHANNELS_8000],
2229                     IWM_NUM_CHANNELS_8000 * sizeof(uint16_t));
2230         }
2231
2232         return data;
2233 }
2234
2235 static void
2236 iwm_free_nvm_data(struct iwm_nvm_data *data)
2237 {
2238         if (data != NULL)
2239                 free(data, M_DEVBUF);
2240 }
2241
2242 static struct iwm_nvm_data *
2243 iwm_parse_nvm_sections(struct iwm_softc *sc, struct iwm_nvm_section *sections)
2244 {
2245         const uint16_t *hw, *sw, *calib, *regulatory, *mac_override, *phy_sku;
2246
2247         /* Checking for required sections */
2248         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
2249                 if (!sections[IWM_NVM_SECTION_TYPE_SW].data ||
2250                     !sections[sc->cfg->nvm_hw_section_num].data) {
2251                         device_printf(sc->sc_dev,
2252                             "Can't parse empty OTP/NVM sections\n");
2253                         return NULL;
2254                 }
2255         } else if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) {
2256                 /* SW and REGULATORY sections are mandatory */
2257                 if (!sections[IWM_NVM_SECTION_TYPE_SW].data ||
2258                     !sections[IWM_NVM_SECTION_TYPE_REGULATORY].data) {
2259                         device_printf(sc->sc_dev,
2260                             "Can't parse empty OTP/NVM sections\n");
2261                         return NULL;
2262                 }
2263                 /* MAC_OVERRIDE or at least HW section must exist */
2264                 if (!sections[sc->cfg->nvm_hw_section_num].data &&
2265                     !sections[IWM_NVM_SECTION_TYPE_MAC_OVERRIDE].data) {
2266                         device_printf(sc->sc_dev,
2267                             "Can't parse mac_address, empty sections\n");
2268                         return NULL;
2269                 }
2270
2271                 /* PHY_SKU section is mandatory in B0 */
2272                 if (!sections[IWM_NVM_SECTION_TYPE_PHY_SKU].data) {
2273                         device_printf(sc->sc_dev,
2274                             "Can't parse phy_sku in B0, empty sections\n");
2275                         return NULL;
2276                 }
2277         } else {
2278                 panic("unknown device family %d\n", sc->cfg->device_family);
2279         }
2280
2281         hw = (const uint16_t *) sections[sc->cfg->nvm_hw_section_num].data;
2282         sw = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_SW].data;
2283         calib = (const uint16_t *)
2284             sections[IWM_NVM_SECTION_TYPE_CALIBRATION].data;
2285         regulatory = (const uint16_t *)
2286             sections[IWM_NVM_SECTION_TYPE_REGULATORY].data;
2287         mac_override = (const uint16_t *)
2288             sections[IWM_NVM_SECTION_TYPE_MAC_OVERRIDE].data;
2289         phy_sku = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_PHY_SKU].data;
2290
2291         return iwm_parse_nvm_data(sc, hw, sw, calib, mac_override,
2292             phy_sku, regulatory);
2293 }
2294
2295 static int
2296 iwm_nvm_init(struct iwm_softc *sc)
2297 {
2298         struct iwm_nvm_section nvm_sections[IWM_NVM_MAX_NUM_SECTIONS];
2299         int i, ret, section;
2300         uint32_t size_read = 0;
2301         uint8_t *nvm_buffer, *temp;
2302         uint16_t len;
2303
2304         memset(nvm_sections, 0, sizeof(nvm_sections));
2305
2306         if (sc->cfg->nvm_hw_section_num >= IWM_NVM_MAX_NUM_SECTIONS)
2307                 return EINVAL;
2308
2309         /* load NVM values from nic */
2310         /* Read From FW NVM */
2311         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM, "Read from NVM\n");
2312
2313         nvm_buffer = malloc(sc->cfg->eeprom_size, M_DEVBUF, M_NOWAIT | M_ZERO);
2314         if (!nvm_buffer)
2315                 return ENOMEM;
2316         for (section = 0; section < IWM_NVM_MAX_NUM_SECTIONS; section++) {
2317                 /* we override the constness for initial read */
2318                 ret = iwm_nvm_read_section(sc, section, nvm_buffer,
2319                                            &len, size_read);
2320                 if (ret)
2321                         continue;
2322                 size_read += len;
2323                 temp = malloc(len, M_DEVBUF, M_NOWAIT);
2324                 if (!temp) {
2325                         ret = ENOMEM;
2326                         break;
2327                 }
2328                 memcpy(temp, nvm_buffer, len);
2329
2330                 nvm_sections[section].data = temp;
2331                 nvm_sections[section].length = len;
2332         }
2333         if (!size_read)
2334                 device_printf(sc->sc_dev, "OTP is blank\n");
2335         free(nvm_buffer, M_DEVBUF);
2336
2337         sc->nvm_data = iwm_parse_nvm_sections(sc, nvm_sections);
2338         if (!sc->nvm_data)
2339                 return EINVAL;
2340         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
2341                     "nvm version = %x\n", sc->nvm_data->nvm_version);
2342
2343         for (i = 0; i < IWM_NVM_MAX_NUM_SECTIONS; i++) {
2344                 if (nvm_sections[i].data != NULL)
2345                         free(nvm_sections[i].data, M_DEVBUF);
2346         }
2347
2348         return 0;
2349 }
2350
2351 static int
2352 iwm_pcie_load_section(struct iwm_softc *sc, uint8_t section_num,
2353         const struct iwm_fw_desc *section)
2354 {
2355         struct iwm_dma_info *dma = &sc->fw_dma;
2356         uint8_t *v_addr;
2357         bus_addr_t p_addr;
2358         uint32_t offset, chunk_sz = MIN(IWM_FH_MEM_TB_MAX_LENGTH, section->len);
2359         int ret = 0;
2360
2361         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2362                     "%s: [%d] uCode section being loaded...\n",
2363                     __func__, section_num);
2364
2365         v_addr = dma->vaddr;
2366         p_addr = dma->paddr;
2367
2368         for (offset = 0; offset < section->len; offset += chunk_sz) {
2369                 uint32_t copy_size, dst_addr;
2370                 int extended_addr = FALSE;
2371
2372                 copy_size = MIN(chunk_sz, section->len - offset);
2373                 dst_addr = section->offset + offset;
2374
2375                 if (dst_addr >= IWM_FW_MEM_EXTENDED_START &&
2376                     dst_addr <= IWM_FW_MEM_EXTENDED_END)
2377                         extended_addr = TRUE;
2378
2379                 if (extended_addr)
2380                         iwm_set_bits_prph(sc, IWM_LMPM_CHICK,
2381                                           IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE);
2382
2383                 memcpy(v_addr, (const uint8_t *)section->data + offset,
2384                     copy_size);
2385                 bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREWRITE);
2386                 ret = iwm_pcie_load_firmware_chunk(sc, dst_addr, p_addr,
2387                                                    copy_size);
2388
2389                 if (extended_addr)
2390                         iwm_clear_bits_prph(sc, IWM_LMPM_CHICK,
2391                                             IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE);
2392
2393                 if (ret) {
2394                         device_printf(sc->sc_dev,
2395                             "%s: Could not load the [%d] uCode section\n",
2396                             __func__, section_num);
2397                         break;
2398                 }
2399         }
2400
2401         return ret;
2402 }
2403
2404 /*
2405  * ucode
2406  */
2407 static int
2408 iwm_pcie_load_firmware_chunk(struct iwm_softc *sc, uint32_t dst_addr,
2409                              bus_addr_t phy_addr, uint32_t byte_cnt)
2410 {
2411         int ret;
2412
2413         sc->sc_fw_chunk_done = 0;
2414
2415         if (!iwm_nic_lock(sc))
2416                 return EBUSY;
2417
2418         IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(IWM_FH_SRVC_CHNL),
2419             IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE);
2420
2421         IWM_WRITE(sc, IWM_FH_SRVC_CHNL_SRAM_ADDR_REG(IWM_FH_SRVC_CHNL),
2422             dst_addr);
2423
2424         IWM_WRITE(sc, IWM_FH_TFDIB_CTRL0_REG(IWM_FH_SRVC_CHNL),
2425             phy_addr & IWM_FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK);
2426
2427         IWM_WRITE(sc, IWM_FH_TFDIB_CTRL1_REG(IWM_FH_SRVC_CHNL),
2428             (iwm_get_dma_hi_addr(phy_addr)
2429              << IWM_FH_MEM_TFDIB_REG1_ADDR_BITSHIFT) | byte_cnt);
2430
2431         IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_BUF_STS_REG(IWM_FH_SRVC_CHNL),
2432             1 << IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM |
2433             1 << IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX |
2434             IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID);
2435
2436         IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(IWM_FH_SRVC_CHNL),
2437             IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE    |
2438             IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE |
2439             IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD);
2440
2441         iwm_nic_unlock(sc);
2442
2443         /* wait up to 5s for this segment to load */
2444         ret = 0;
2445         while (!sc->sc_fw_chunk_done) {
2446                 ret = msleep(&sc->sc_fw, &sc->sc_mtx, 0, "iwmfw", hz);
2447                 if (ret)
2448                         break;
2449         }
2450
2451         if (ret != 0) {
2452                 device_printf(sc->sc_dev,
2453                     "fw chunk addr 0x%x len %d failed to load\n",
2454                     dst_addr, byte_cnt);
2455                 return ETIMEDOUT;
2456         }
2457
2458         return 0;
2459 }
2460
2461 static int
2462 iwm_pcie_load_cpu_sections_8000(struct iwm_softc *sc,
2463         const struct iwm_fw_sects *image, int cpu, int *first_ucode_section)
2464 {
2465         int shift_param;
2466         int i, ret = 0, sec_num = 0x1;
2467         uint32_t val, last_read_idx = 0;
2468
2469         if (cpu == 1) {
2470                 shift_param = 0;
2471                 *first_ucode_section = 0;
2472         } else {
2473                 shift_param = 16;
2474                 (*first_ucode_section)++;
2475         }
2476
2477         for (i = *first_ucode_section; i < IWM_UCODE_SECTION_MAX; i++) {
2478                 last_read_idx = i;
2479
2480                 /*
2481                  * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
2482                  * CPU1 to CPU2.
2483                  * PAGING_SEPARATOR_SECTION delimiter - separate between
2484                  * CPU2 non paged to CPU2 paging sec.
2485                  */
2486                 if (!image->fw_sect[i].data ||
2487                     image->fw_sect[i].offset == IWM_CPU1_CPU2_SEPARATOR_SECTION ||
2488                     image->fw_sect[i].offset == IWM_PAGING_SEPARATOR_SECTION) {
2489                         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2490                                     "Break since Data not valid or Empty section, sec = %d\n",
2491                                     i);
2492                         break;
2493                 }
2494                 ret = iwm_pcie_load_section(sc, i, &image->fw_sect[i]);
2495                 if (ret)
2496                         return ret;
2497
2498                 /* Notify the ucode of the loaded section number and status */
2499                 if (iwm_nic_lock(sc)) {
2500                         val = IWM_READ(sc, IWM_FH_UCODE_LOAD_STATUS);
2501                         val = val | (sec_num << shift_param);
2502                         IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, val);
2503                         sec_num = (sec_num << 1) | 0x1;
2504                         iwm_nic_unlock(sc);
2505                 }
2506         }
2507
2508         *first_ucode_section = last_read_idx;
2509
2510         iwm_enable_interrupts(sc);
2511
2512         if (iwm_nic_lock(sc)) {
2513                 if (cpu == 1)
2514                         IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, 0xFFFF);
2515                 else
2516                         IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, 0xFFFFFFFF);
2517                 iwm_nic_unlock(sc);
2518         }
2519
2520         return 0;
2521 }
2522
2523 static int
2524 iwm_pcie_load_cpu_sections(struct iwm_softc *sc,
2525         const struct iwm_fw_sects *image, int cpu, int *first_ucode_section)
2526 {
2527         int shift_param;
2528         int i, ret = 0;
2529         uint32_t last_read_idx = 0;
2530
2531         if (cpu == 1) {
2532                 shift_param = 0;
2533                 *first_ucode_section = 0;
2534         } else {
2535                 shift_param = 16;
2536                 (*first_ucode_section)++;
2537         }
2538
2539         for (i = *first_ucode_section; i < IWM_UCODE_SECTION_MAX; i++) {
2540                 last_read_idx = i;
2541
2542                 /*
2543                  * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
2544                  * CPU1 to CPU2.
2545                  * PAGING_SEPARATOR_SECTION delimiter - separate between
2546                  * CPU2 non paged to CPU2 paging sec.
2547                  */
2548                 if (!image->fw_sect[i].data ||
2549                     image->fw_sect[i].offset == IWM_CPU1_CPU2_SEPARATOR_SECTION ||
2550                     image->fw_sect[i].offset == IWM_PAGING_SEPARATOR_SECTION) {
2551                         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2552                                     "Break since Data not valid or Empty section, sec = %d\n",
2553                                      i);
2554                         break;
2555                 }
2556
2557                 ret = iwm_pcie_load_section(sc, i, &image->fw_sect[i]);
2558                 if (ret)
2559                         return ret;
2560         }
2561
2562         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000)
2563                 iwm_set_bits_prph(sc,
2564                                   IWM_CSR_UCODE_LOAD_STATUS_ADDR,
2565                                   (IWM_LMPM_CPU_UCODE_LOADING_COMPLETED |
2566                                    IWM_LMPM_CPU_HDRS_LOADING_COMPLETED |
2567                                    IWM_LMPM_CPU_UCODE_LOADING_STARTED) <<
2568                                         shift_param);
2569
2570         *first_ucode_section = last_read_idx;
2571
2572         return 0;
2573
2574 }
2575
2576 static int
2577 iwm_pcie_load_given_ucode(struct iwm_softc *sc,
2578         const struct iwm_fw_sects *image)
2579 {
2580         int ret = 0;
2581         int first_ucode_section;
2582
2583         IWM_DPRINTF(sc, IWM_DEBUG_RESET, "working with %s CPU\n",
2584                      image->is_dual_cpus ? "Dual" : "Single");
2585
2586         /* load to FW the binary non secured sections of CPU1 */
2587         ret = iwm_pcie_load_cpu_sections(sc, image, 1, &first_ucode_section);
2588         if (ret)
2589                 return ret;
2590
2591         if (image->is_dual_cpus) {
2592                 /* set CPU2 header address */
2593                 iwm_write_prph(sc,
2594                                IWM_LMPM_SECURE_UCODE_LOAD_CPU2_HDR_ADDR,
2595                                IWM_LMPM_SECURE_CPU2_HDR_MEM_SPACE);
2596
2597                 /* load to FW the binary sections of CPU2 */
2598                 ret = iwm_pcie_load_cpu_sections(sc, image, 2,
2599                                                  &first_ucode_section);
2600                 if (ret)
2601                         return ret;
2602         }
2603
2604         iwm_enable_interrupts(sc);
2605
2606         /* release CPU reset */
2607         IWM_WRITE(sc, IWM_CSR_RESET, 0);
2608
2609         return 0;
2610 }
2611
2612 int
2613 iwm_pcie_load_given_ucode_8000(struct iwm_softc *sc,
2614         const struct iwm_fw_sects *image)
2615 {
2616         int ret = 0;
2617         int first_ucode_section;
2618
2619         IWM_DPRINTF(sc, IWM_DEBUG_RESET, "working with %s CPU\n",
2620                     image->is_dual_cpus ? "Dual" : "Single");
2621
2622         /* configure the ucode to be ready to get the secured image */
2623         /* release CPU reset */
2624         iwm_write_prph(sc, IWM_RELEASE_CPU_RESET, IWM_RELEASE_CPU_RESET_BIT);
2625
2626         /* load to FW the binary Secured sections of CPU1 */
2627         ret = iwm_pcie_load_cpu_sections_8000(sc, image, 1,
2628             &first_ucode_section);
2629         if (ret)
2630                 return ret;
2631
2632         /* load to FW the binary sections of CPU2 */
2633         return iwm_pcie_load_cpu_sections_8000(sc, image, 2,
2634             &first_ucode_section);
2635 }
2636
2637 /* XXX Get rid of this definition */
2638 static inline void
2639 iwm_enable_fw_load_int(struct iwm_softc *sc)
2640 {
2641         IWM_DPRINTF(sc, IWM_DEBUG_INTR, "Enabling FW load interrupt\n");
2642         sc->sc_intmask = IWM_CSR_INT_BIT_FH_TX;
2643         IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask);
2644 }
2645
2646 /* XXX Add proper rfkill support code */
2647 static int
2648 iwm_start_fw(struct iwm_softc *sc,
2649         const struct iwm_fw_sects *fw)
2650 {
2651         int ret;
2652
2653         /* This may fail if AMT took ownership of the device */
2654         if (iwm_prepare_card_hw(sc)) {
2655                 device_printf(sc->sc_dev,
2656                     "%s: Exit HW not ready\n", __func__);
2657                 ret = EIO;
2658                 goto out;
2659         }
2660
2661         IWM_WRITE(sc, IWM_CSR_INT, 0xFFFFFFFF);
2662
2663         iwm_disable_interrupts(sc);
2664
2665         /* make sure rfkill handshake bits are cleared */
2666         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL);
2667         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR,
2668             IWM_CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED);
2669
2670         /* clear (again), then enable host interrupts */
2671         IWM_WRITE(sc, IWM_CSR_INT, 0xFFFFFFFF);
2672
2673         ret = iwm_nic_init(sc);
2674         if (ret) {
2675                 device_printf(sc->sc_dev, "%s: Unable to init nic\n", __func__);
2676                 goto out;
2677         }
2678
2679         /*
2680          * Now, we load the firmware and don't want to be interrupted, even
2681          * by the RF-Kill interrupt (hence mask all the interrupt besides the
2682          * FH_TX interrupt which is needed to load the firmware). If the
2683          * RF-Kill switch is toggled, we will find out after having loaded
2684          * the firmware and return the proper value to the caller.
2685          */
2686         iwm_enable_fw_load_int(sc);
2687
2688         /* really make sure rfkill handshake bits are cleared */
2689         /* maybe we should write a few times more?  just to make sure */
2690         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL);
2691         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL);
2692
2693         /* Load the given image to the HW */
2694         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000)
2695                 ret = iwm_pcie_load_given_ucode_8000(sc, fw);
2696         else
2697                 ret = iwm_pcie_load_given_ucode(sc, fw);
2698
2699         /* XXX re-check RF-Kill state */
2700
2701 out:
2702         return ret;
2703 }
2704
2705 static int
2706 iwm_send_tx_ant_cfg(struct iwm_softc *sc, uint8_t valid_tx_ant)
2707 {
2708         struct iwm_tx_ant_cfg_cmd tx_ant_cmd = {
2709                 .valid = htole32(valid_tx_ant),
2710         };
2711
2712         return iwm_mvm_send_cmd_pdu(sc, IWM_TX_ANT_CONFIGURATION_CMD,
2713             IWM_CMD_SYNC, sizeof(tx_ant_cmd), &tx_ant_cmd);
2714 }
2715
2716 /* iwlwifi: mvm/fw.c */
2717 static int
2718 iwm_send_phy_cfg_cmd(struct iwm_softc *sc)
2719 {
2720         struct iwm_phy_cfg_cmd phy_cfg_cmd;
2721         enum iwm_ucode_type ucode_type = sc->cur_ucode;
2722
2723         /* Set parameters */
2724         phy_cfg_cmd.phy_cfg = htole32(iwm_mvm_get_phy_config(sc));
2725         phy_cfg_cmd.calib_control.event_trigger =
2726             sc->sc_default_calib[ucode_type].event_trigger;
2727         phy_cfg_cmd.calib_control.flow_trigger =
2728             sc->sc_default_calib[ucode_type].flow_trigger;
2729
2730         IWM_DPRINTF(sc, IWM_DEBUG_CMD | IWM_DEBUG_RESET,
2731             "Sending Phy CFG command: 0x%x\n", phy_cfg_cmd.phy_cfg);
2732         return iwm_mvm_send_cmd_pdu(sc, IWM_PHY_CONFIGURATION_CMD, IWM_CMD_SYNC,
2733             sizeof(phy_cfg_cmd), &phy_cfg_cmd);
2734 }
2735
2736 static int
2737 iwm_alive_fn(struct iwm_softc *sc, struct iwm_rx_packet *pkt, void *data)
2738 {
2739         struct iwm_mvm_alive_data *alive_data = data;
2740         struct iwm_mvm_alive_resp_ver1 *palive1;
2741         struct iwm_mvm_alive_resp_ver2 *palive2;
2742         struct iwm_mvm_alive_resp *palive;
2743
2744         if (iwm_rx_packet_payload_len(pkt) == sizeof(*palive1)) {
2745                 palive1 = (void *)pkt->data;
2746
2747                 sc->support_umac_log = FALSE;
2748                 sc->error_event_table =
2749                         le32toh(palive1->error_event_table_ptr);
2750                 sc->log_event_table =
2751                         le32toh(palive1->log_event_table_ptr);
2752                 alive_data->scd_base_addr = le32toh(palive1->scd_base_ptr);
2753
2754                 alive_data->valid = le16toh(palive1->status) ==
2755                                     IWM_ALIVE_STATUS_OK;
2756                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2757                             "Alive VER1 ucode status 0x%04x revision 0x%01X 0x%01X flags 0x%01X\n",
2758                              le16toh(palive1->status), palive1->ver_type,
2759                              palive1->ver_subtype, palive1->flags);
2760         } else if (iwm_rx_packet_payload_len(pkt) == sizeof(*palive2)) {
2761                 palive2 = (void *)pkt->data;
2762                 sc->error_event_table =
2763                         le32toh(palive2->error_event_table_ptr);
2764                 sc->log_event_table =
2765                         le32toh(palive2->log_event_table_ptr);
2766                 alive_data->scd_base_addr = le32toh(palive2->scd_base_ptr);
2767                 sc->umac_error_event_table =
2768                         le32toh(palive2->error_info_addr);
2769
2770                 alive_data->valid = le16toh(palive2->status) ==
2771                                     IWM_ALIVE_STATUS_OK;
2772                 if (sc->umac_error_event_table)
2773                         sc->support_umac_log = TRUE;
2774
2775                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2776                             "Alive VER2 ucode status 0x%04x revision 0x%01X 0x%01X flags 0x%01X\n",
2777                             le16toh(palive2->status), palive2->ver_type,
2778                             palive2->ver_subtype, palive2->flags);
2779
2780                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2781                             "UMAC version: Major - 0x%x, Minor - 0x%x\n",
2782                             palive2->umac_major, palive2->umac_minor);
2783         } else if (iwm_rx_packet_payload_len(pkt) == sizeof(*palive)) {
2784                 palive = (void *)pkt->data;
2785
2786                 sc->error_event_table =
2787                         le32toh(palive->error_event_table_ptr);
2788                 sc->log_event_table =
2789                         le32toh(palive->log_event_table_ptr);
2790                 alive_data->scd_base_addr = le32toh(palive->scd_base_ptr);
2791                 sc->umac_error_event_table =
2792                         le32toh(palive->error_info_addr);
2793
2794                 alive_data->valid = le16toh(palive->status) ==
2795                                     IWM_ALIVE_STATUS_OK;
2796                 if (sc->umac_error_event_table)
2797                         sc->support_umac_log = TRUE;
2798
2799                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2800                             "Alive VER3 ucode status 0x%04x revision 0x%01X 0x%01X flags 0x%01X\n",
2801                             le16toh(palive->status), palive->ver_type,
2802                             palive->ver_subtype, palive->flags);
2803
2804                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2805                             "UMAC version: Major - 0x%x, Minor - 0x%x\n",
2806                             le32toh(palive->umac_major),
2807                             le32toh(palive->umac_minor));
2808         }
2809
2810         return TRUE;
2811 }
2812
2813 static int
2814 iwm_wait_phy_db_entry(struct iwm_softc *sc,
2815         struct iwm_rx_packet *pkt, void *data)
2816 {
2817         struct iwm_phy_db *phy_db = data;
2818
2819         if (pkt->hdr.code != IWM_CALIB_RES_NOTIF_PHY_DB) {
2820                 if(pkt->hdr.code != IWM_INIT_COMPLETE_NOTIF) {
2821                         device_printf(sc->sc_dev, "%s: Unexpected cmd: %d\n",
2822                             __func__, pkt->hdr.code);
2823                 }
2824                 return TRUE;
2825         }
2826
2827         if (iwm_phy_db_set_section(phy_db, pkt)) {
2828                 device_printf(sc->sc_dev,
2829                     "%s: iwm_phy_db_set_section failed\n", __func__);
2830         }
2831
2832         return FALSE;
2833 }
2834
2835 static int
2836 iwm_mvm_load_ucode_wait_alive(struct iwm_softc *sc,
2837         enum iwm_ucode_type ucode_type)
2838 {
2839         struct iwm_notification_wait alive_wait;
2840         struct iwm_mvm_alive_data alive_data;
2841         const struct iwm_fw_sects *fw;
2842         enum iwm_ucode_type old_type = sc->cur_ucode;
2843         int error;
2844         static const uint16_t alive_cmd[] = { IWM_MVM_ALIVE };
2845
2846         if ((error = iwm_read_firmware(sc, ucode_type)) != 0) {
2847                 device_printf(sc->sc_dev, "iwm_read_firmware: failed %d\n",
2848                         error);
2849                 return error;
2850         }
2851         fw = &sc->sc_fw.fw_sects[ucode_type];
2852         sc->cur_ucode = ucode_type;
2853         sc->ucode_loaded = FALSE;
2854
2855         memset(&alive_data, 0, sizeof(alive_data));
2856         iwm_init_notification_wait(sc->sc_notif_wait, &alive_wait,
2857                                    alive_cmd, nitems(alive_cmd),
2858                                    iwm_alive_fn, &alive_data);
2859
2860         error = iwm_start_fw(sc, fw);
2861         if (error) {
2862                 device_printf(sc->sc_dev, "iwm_start_fw: failed %d\n", error);
2863                 sc->cur_ucode = old_type;
2864                 iwm_remove_notification(sc->sc_notif_wait, &alive_wait);
2865                 return error;
2866         }
2867
2868         /*
2869          * Some things may run in the background now, but we
2870          * just wait for the ALIVE notification here.
2871          */
2872         IWM_UNLOCK(sc);
2873         error = iwm_wait_notification(sc->sc_notif_wait, &alive_wait,
2874                                       IWM_MVM_UCODE_ALIVE_TIMEOUT);
2875         IWM_LOCK(sc);
2876         if (error) {
2877                 if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) {
2878                         device_printf(sc->sc_dev,
2879                             "SecBoot CPU1 Status: 0x%x, CPU2 Status: 0x%x\n",
2880                             iwm_read_prph(sc, IWM_SB_CPU_1_STATUS),
2881                             iwm_read_prph(sc, IWM_SB_CPU_2_STATUS));
2882                 }
2883                 sc->cur_ucode = old_type;
2884                 return error;
2885         }
2886
2887         if (!alive_data.valid) {
2888                 device_printf(sc->sc_dev, "%s: Loaded ucode is not valid\n",
2889                     __func__);
2890                 sc->cur_ucode = old_type;
2891                 return EIO;
2892         }
2893
2894         iwm_trans_pcie_fw_alive(sc, alive_data.scd_base_addr);
2895
2896         /*
2897          * configure and operate fw paging mechanism.
2898          * driver configures the paging flow only once, CPU2 paging image
2899          * included in the IWM_UCODE_INIT image.
2900          */
2901         if (fw->paging_mem_size) {
2902                 error = iwm_save_fw_paging(sc, fw);
2903                 if (error) {
2904                         device_printf(sc->sc_dev,
2905                             "%s: failed to save the FW paging image\n",
2906                             __func__);
2907                         return error;
2908                 }
2909
2910                 error = iwm_send_paging_cmd(sc, fw);
2911                 if (error) {
2912                         device_printf(sc->sc_dev,
2913                             "%s: failed to send the paging cmd\n", __func__);
2914                         iwm_free_fw_paging(sc);
2915                         return error;
2916                 }
2917         }
2918
2919         if (!error)
2920                 sc->ucode_loaded = TRUE;
2921         return error;
2922 }
2923
2924 /*
2925  * mvm misc bits
2926  */
2927
2928 /*
2929  * follows iwlwifi/fw.c
2930  */
2931 static int
2932 iwm_run_init_mvm_ucode(struct iwm_softc *sc, int justnvm)
2933 {
2934         struct iwm_notification_wait calib_wait;
2935         static const uint16_t init_complete[] = {
2936                 IWM_INIT_COMPLETE_NOTIF,
2937                 IWM_CALIB_RES_NOTIF_PHY_DB
2938         };
2939         int ret;
2940
2941         /* do not operate with rfkill switch turned on */
2942         if ((sc->sc_flags & IWM_FLAG_RFKILL) && !justnvm) {
2943                 device_printf(sc->sc_dev,
2944                     "radio is disabled by hardware switch\n");
2945                 return EPERM;
2946         }
2947
2948         iwm_init_notification_wait(sc->sc_notif_wait,
2949                                    &calib_wait,
2950                                    init_complete,
2951                                    nitems(init_complete),
2952                                    iwm_wait_phy_db_entry,
2953                                    sc->sc_phy_db);
2954
2955         /* Will also start the device */
2956         ret = iwm_mvm_load_ucode_wait_alive(sc, IWM_UCODE_INIT);
2957         if (ret) {
2958                 device_printf(sc->sc_dev, "Failed to start INIT ucode: %d\n",
2959                     ret);
2960                 goto error;
2961         }
2962
2963         if (justnvm) {
2964                 /* Read nvm */
2965                 ret = iwm_nvm_init(sc);
2966                 if (ret) {
2967                         device_printf(sc->sc_dev, "failed to read nvm\n");
2968                         goto error;
2969                 }
2970                 IEEE80211_ADDR_COPY(sc->sc_ic.ic_macaddr, sc->nvm_data->hw_addr);
2971                 goto error;
2972         }
2973
2974         ret = iwm_send_bt_init_conf(sc);
2975         if (ret) {
2976                 device_printf(sc->sc_dev,
2977                     "failed to send bt coex configuration: %d\n", ret);
2978                 goto error;
2979         }
2980
2981         /* Init Smart FIFO. */
2982         ret = iwm_mvm_sf_config(sc, IWM_SF_INIT_OFF);
2983         if (ret)
2984                 goto error;
2985
2986         /* Send TX valid antennas before triggering calibrations */
2987         ret = iwm_send_tx_ant_cfg(sc, iwm_mvm_get_valid_tx_ant(sc));
2988         if (ret) {
2989                 device_printf(sc->sc_dev,
2990                     "failed to send antennas before calibration: %d\n", ret);
2991                 goto error;
2992         }
2993
2994         /*
2995          * Send phy configurations command to init uCode
2996          * to start the 16.0 uCode init image internal calibrations.
2997          */
2998         ret = iwm_send_phy_cfg_cmd(sc);
2999         if (ret) {
3000                 device_printf(sc->sc_dev,
3001                     "%s: Failed to run INIT calibrations: %d\n",
3002                     __func__, ret);
3003                 goto error;
3004         }
3005
3006         /*
3007          * Nothing to do but wait for the init complete notification
3008          * from the firmware.
3009          */
3010         IWM_UNLOCK(sc);
3011         ret = iwm_wait_notification(sc->sc_notif_wait, &calib_wait,
3012             IWM_MVM_UCODE_CALIB_TIMEOUT);
3013         IWM_LOCK(sc);
3014
3015
3016         goto out;
3017
3018 error:
3019         iwm_remove_notification(sc->sc_notif_wait, &calib_wait);
3020 out:
3021         return ret;
3022 }
3023
3024 /*
3025  * receive side
3026  */
3027
3028 /* (re)stock rx ring, called at init-time and at runtime */
3029 static int
3030 iwm_rx_addbuf(struct iwm_softc *sc, int size, int idx)
3031 {
3032         struct iwm_rx_ring *ring = &sc->rxq;
3033         struct iwm_rx_data *data = &ring->data[idx];
3034         struct mbuf *m;
3035         bus_dmamap_t dmamap;
3036         bus_dma_segment_t seg;
3037         int nsegs, error;
3038
3039         m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, IWM_RBUF_SIZE);
3040         if (m == NULL)
3041                 return ENOBUFS;
3042
3043         m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
3044         error = bus_dmamap_load_mbuf_sg(ring->data_dmat, ring->spare_map, m,
3045             &seg, &nsegs, BUS_DMA_NOWAIT);
3046         if (error != 0) {
3047                 device_printf(sc->sc_dev,
3048                     "%s: can't map mbuf, error %d\n", __func__, error);
3049                 m_freem(m);
3050                 return error;
3051         }
3052
3053         if (data->m != NULL)
3054                 bus_dmamap_unload(ring->data_dmat, data->map);
3055
3056         /* Swap ring->spare_map with data->map */
3057         dmamap = data->map;
3058         data->map = ring->spare_map;
3059         ring->spare_map = dmamap;
3060
3061         bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_PREREAD);
3062         data->m = m;
3063
3064         /* Update RX descriptor. */
3065         KASSERT((seg.ds_addr & 255) == 0, ("seg.ds_addr not aligned"));
3066         ring->desc[idx] = htole32(seg.ds_addr >> 8);
3067         bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map,
3068             BUS_DMASYNC_PREWRITE);
3069
3070         return 0;
3071 }
3072
3073 /* iwlwifi: mvm/rx.c */
3074 /*
3075  * iwm_mvm_get_signal_strength - use new rx PHY INFO API
3076  * values are reported by the fw as positive values - need to negate
3077  * to obtain their dBM.  Account for missing antennas by replacing 0
3078  * values by -256dBm: practically 0 power and a non-feasible 8 bit value.
3079  */
3080 static int
3081 iwm_mvm_get_signal_strength(struct iwm_softc *sc, struct iwm_rx_phy_info *phy_info)
3082 {
3083         int energy_a, energy_b, energy_c, max_energy;
3084         uint32_t val;
3085
3086         val = le32toh(phy_info->non_cfg_phy[IWM_RX_INFO_ENERGY_ANT_ABC_IDX]);
3087         energy_a = (val & IWM_RX_INFO_ENERGY_ANT_A_MSK) >>
3088             IWM_RX_INFO_ENERGY_ANT_A_POS;
3089         energy_a = energy_a ? -energy_a : -256;
3090         energy_b = (val & IWM_RX_INFO_ENERGY_ANT_B_MSK) >>
3091             IWM_RX_INFO_ENERGY_ANT_B_POS;
3092         energy_b = energy_b ? -energy_b : -256;
3093         energy_c = (val & IWM_RX_INFO_ENERGY_ANT_C_MSK) >>
3094             IWM_RX_INFO_ENERGY_ANT_C_POS;
3095         energy_c = energy_c ? -energy_c : -256;
3096         max_energy = MAX(energy_a, energy_b);
3097         max_energy = MAX(max_energy, energy_c);
3098
3099         IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3100             "energy In A %d B %d C %d , and max %d\n",
3101             energy_a, energy_b, energy_c, max_energy);
3102
3103         return max_energy;
3104 }
3105
3106 static void
3107 iwm_mvm_rx_rx_phy_cmd(struct iwm_softc *sc,
3108         struct iwm_rx_packet *pkt, struct iwm_rx_data *data)
3109 {
3110         struct iwm_rx_phy_info *phy_info = (void *)pkt->data;
3111
3112         IWM_DPRINTF(sc, IWM_DEBUG_RECV, "received PHY stats\n");
3113         bus_dmamap_sync(sc->rxq.data_dmat, data->map, BUS_DMASYNC_POSTREAD);
3114
3115         memcpy(&sc->sc_last_phy_info, phy_info, sizeof(sc->sc_last_phy_info));
3116 }
3117
3118 /*
3119  * Retrieve the average noise (in dBm) among receivers.
3120  */
3121 static int
3122 iwm_get_noise(struct iwm_softc *sc,
3123     const struct iwm_mvm_statistics_rx_non_phy *stats)
3124 {
3125         int i, total, nbant, noise;
3126
3127         total = nbant = noise = 0;
3128         for (i = 0; i < 3; i++) {
3129                 noise = le32toh(stats->beacon_silence_rssi[i]) & 0xff;
3130                 IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: i=%d, noise=%d\n",
3131                     __func__,
3132                     i,
3133                     noise);
3134
3135                 if (noise) {
3136                         total += noise;
3137                         nbant++;
3138                 }
3139         }
3140
3141         IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: nbant=%d, total=%d\n",
3142             __func__, nbant, total);
3143 #if 0
3144         /* There should be at least one antenna but check anyway. */
3145         return (nbant == 0) ? -127 : (total / nbant) - 107;
3146 #else
3147         /* For now, just hard-code it to -96 to be safe */
3148         return (-96);
3149 #endif
3150 }
3151
3152 /*
3153  * iwm_mvm_rx_rx_mpdu - IWM_REPLY_RX_MPDU_CMD handler
3154  *
3155  * Handles the actual data of the Rx packet from the fw
3156  */
3157 static void
3158 iwm_mvm_rx_rx_mpdu(struct iwm_softc *sc, struct mbuf *m)
3159 {
3160         struct ieee80211com *ic = &sc->sc_ic;
3161         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
3162         struct ieee80211_frame *wh;
3163         struct ieee80211_node *ni;
3164         struct ieee80211_rx_stats rxs;
3165         struct iwm_rx_phy_info *phy_info;
3166         struct iwm_rx_mpdu_res_start *rx_res;
3167         struct iwm_rx_packet *pkt = mtod(m, struct iwm_rx_packet *);
3168         uint32_t len;
3169         uint32_t rx_pkt_status;
3170         int rssi;
3171
3172         phy_info = &sc->sc_last_phy_info;
3173         rx_res = (struct iwm_rx_mpdu_res_start *)pkt->data;
3174         wh = (struct ieee80211_frame *)(pkt->data + sizeof(*rx_res));
3175         len = le16toh(rx_res->byte_count);
3176         rx_pkt_status = le32toh(*(uint32_t *)(pkt->data + sizeof(*rx_res) + len));
3177
3178         if (__predict_false(phy_info->cfg_phy_cnt > 20)) {
3179                 device_printf(sc->sc_dev,
3180                     "dsp size out of range [0,20]: %d\n",
3181                     phy_info->cfg_phy_cnt);
3182                 goto fail;
3183         }
3184
3185         if (!(rx_pkt_status & IWM_RX_MPDU_RES_STATUS_CRC_OK) ||
3186             !(rx_pkt_status & IWM_RX_MPDU_RES_STATUS_OVERRUN_OK)) {
3187                 IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3188                     "Bad CRC or FIFO: 0x%08X.\n", rx_pkt_status);
3189                 goto fail;
3190         }
3191
3192         rssi = iwm_mvm_get_signal_strength(sc, phy_info);
3193
3194         /* Map it to relative value */
3195         rssi = rssi - sc->sc_noise;
3196
3197         /* replenish ring for the buffer we're going to feed to the sharks */
3198         if (iwm_rx_addbuf(sc, IWM_RBUF_SIZE, sc->rxq.cur) != 0) {
3199                 device_printf(sc->sc_dev, "%s: unable to add more buffers\n",
3200                     __func__);
3201                 goto fail;
3202         }
3203
3204         m->m_data = pkt->data + sizeof(*rx_res);
3205         m->m_pkthdr.len = m->m_len = len;
3206
3207         IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3208             "%s: rssi=%d, noise=%d\n", __func__, rssi, sc->sc_noise);
3209
3210         ni = ieee80211_find_rxnode(ic, (struct ieee80211_frame_min *)wh);
3211
3212         IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3213             "%s: phy_info: channel=%d, flags=0x%08x\n",
3214             __func__,
3215             le16toh(phy_info->channel),
3216             le16toh(phy_info->phy_flags));
3217
3218         /*
3219          * Populate an RX state struct with the provided information.
3220          */
3221         bzero(&rxs, sizeof(rxs));
3222         rxs.r_flags |= IEEE80211_R_IEEE | IEEE80211_R_FREQ;
3223         rxs.r_flags |= IEEE80211_R_NF | IEEE80211_R_RSSI;
3224         rxs.c_ieee = le16toh(phy_info->channel);
3225         if (le16toh(phy_info->phy_flags & IWM_RX_RES_PHY_FLAGS_BAND_24)) {
3226                 rxs.c_freq = ieee80211_ieee2mhz(rxs.c_ieee, IEEE80211_CHAN_2GHZ);
3227         } else {
3228                 rxs.c_freq = ieee80211_ieee2mhz(rxs.c_ieee, IEEE80211_CHAN_5GHZ);
3229         }
3230
3231         /* rssi is in 1/2db units */
3232         rxs.c_rssi = rssi * 2;
3233         rxs.c_nf = sc->sc_noise;
3234         if (ieee80211_add_rx_params(m, &rxs) == 0) {
3235                 if (ni)
3236                         ieee80211_free_node(ni);
3237                 goto fail;
3238         }
3239
3240         if (ieee80211_radiotap_active_vap(vap)) {
3241                 struct iwm_rx_radiotap_header *tap = &sc->sc_rxtap;
3242
3243                 tap->wr_flags = 0;
3244                 if (phy_info->phy_flags & htole16(IWM_PHY_INFO_FLAG_SHPREAMBLE))
3245                         tap->wr_flags |= IEEE80211_RADIOTAP_F_SHORTPRE;
3246                 tap->wr_chan_freq = htole16(rxs.c_freq);
3247                 /* XXX only if ic->ic_curchan->ic_ieee == rxs.c_ieee */
3248                 tap->wr_chan_flags = htole16(ic->ic_curchan->ic_flags);
3249                 tap->wr_dbm_antsignal = (int8_t)rssi;
3250                 tap->wr_dbm_antnoise = (int8_t)sc->sc_noise;
3251                 tap->wr_tsft = phy_info->system_timestamp;
3252                 switch (phy_info->rate) {
3253                 /* CCK rates. */
3254                 case  10: tap->wr_rate =   2; break;
3255                 case  20: tap->wr_rate =   4; break;
3256                 case  55: tap->wr_rate =  11; break;
3257                 case 110: tap->wr_rate =  22; break;
3258                 /* OFDM rates. */
3259                 case 0xd: tap->wr_rate =  12; break;
3260                 case 0xf: tap->wr_rate =  18; break;
3261                 case 0x5: tap->wr_rate =  24; break;
3262                 case 0x7: tap->wr_rate =  36; break;
3263                 case 0x9: tap->wr_rate =  48; break;
3264                 case 0xb: tap->wr_rate =  72; break;
3265                 case 0x1: tap->wr_rate =  96; break;
3266                 case 0x3: tap->wr_rate = 108; break;
3267                 /* Unknown rate: should not happen. */
3268                 default:  tap->wr_rate =   0;
3269                 }
3270         }
3271
3272         IWM_UNLOCK(sc);
3273         if (ni != NULL) {
3274                 IWM_DPRINTF(sc, IWM_DEBUG_RECV, "input m %p\n", m);
3275                 ieee80211_input_mimo(ni, m);
3276                 ieee80211_free_node(ni);
3277         } else {
3278                 IWM_DPRINTF(sc, IWM_DEBUG_RECV, "inputall m %p\n", m);
3279                 ieee80211_input_mimo_all(ic, m);
3280         }
3281         IWM_LOCK(sc);
3282
3283         return;
3284
3285 fail:
3286         counter_u64_add(ic->ic_ierrors, 1);
3287 }
3288
3289 static int
3290 iwm_mvm_rx_tx_cmd_single(struct iwm_softc *sc, struct iwm_rx_packet *pkt,
3291         struct iwm_node *in)
3292 {
3293         struct iwm_mvm_tx_resp *tx_resp = (void *)pkt->data;
3294         struct ieee80211_ratectl_tx_status *txs = &sc->sc_txs;
3295         struct ieee80211_node *ni = &in->in_ni;
3296         int status = le16toh(tx_resp->status.status) & IWM_TX_STATUS_MSK;
3297
3298         KASSERT(tx_resp->frame_count == 1, ("too many frames"));
3299
3300         /* Update rate control statistics. */
3301         IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "%s: status=0x%04x, seq=%d, fc=%d, btc=%d, frts=%d, ff=%d, irate=%08x, wmt=%d\n",
3302             __func__,
3303             (int) le16toh(tx_resp->status.status),
3304             (int) le16toh(tx_resp->status.sequence),
3305             tx_resp->frame_count,
3306             tx_resp->bt_kill_count,
3307             tx_resp->failure_rts,
3308             tx_resp->failure_frame,
3309             le32toh(tx_resp->initial_rate),
3310             (int) le16toh(tx_resp->wireless_media_time));
3311
3312         txs->flags = IEEE80211_RATECTL_STATUS_SHORT_RETRY |
3313                      IEEE80211_RATECTL_STATUS_LONG_RETRY;
3314         txs->short_retries = tx_resp->failure_rts;
3315         txs->long_retries = tx_resp->failure_frame;
3316         if (status != IWM_TX_STATUS_SUCCESS &&
3317             status != IWM_TX_STATUS_DIRECT_DONE) {
3318                 switch (status) {
3319                 case IWM_TX_STATUS_FAIL_SHORT_LIMIT:
3320                         txs->status = IEEE80211_RATECTL_TX_FAIL_SHORT;
3321                         break;
3322                 case IWM_TX_STATUS_FAIL_LONG_LIMIT:
3323                         txs->status = IEEE80211_RATECTL_TX_FAIL_LONG;
3324                         break;
3325                 case IWM_TX_STATUS_FAIL_LIFE_EXPIRE:
3326                         txs->status = IEEE80211_RATECTL_TX_FAIL_EXPIRED;
3327                         break;
3328                 default:
3329                         txs->status = IEEE80211_RATECTL_TX_FAIL_UNSPECIFIED;
3330                         break;
3331                 }
3332         } else {
3333                 txs->status = IEEE80211_RATECTL_TX_SUCCESS;
3334         }
3335         ieee80211_ratectl_tx_complete(ni, txs);
3336
3337         return (txs->status != IEEE80211_RATECTL_TX_SUCCESS);
3338 }
3339
3340 static void
3341 iwm_mvm_rx_tx_cmd(struct iwm_softc *sc,
3342         struct iwm_rx_packet *pkt, struct iwm_rx_data *data)
3343 {
3344         struct iwm_cmd_header *cmd_hdr = &pkt->hdr;
3345         int idx = cmd_hdr->idx;
3346         int qid = cmd_hdr->qid;
3347         struct iwm_tx_ring *ring = &sc->txq[qid];
3348         struct iwm_tx_data *txd = &ring->data[idx];
3349         struct iwm_node *in = txd->in;
3350         struct mbuf *m = txd->m;
3351         int status;
3352
3353         KASSERT(txd->done == 0, ("txd not done"));
3354         KASSERT(txd->in != NULL, ("txd without node"));
3355         KASSERT(txd->m != NULL, ("txd without mbuf"));
3356
3357         bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTREAD);
3358
3359         sc->sc_tx_timer = 0;
3360
3361         status = iwm_mvm_rx_tx_cmd_single(sc, pkt, in);
3362
3363         /* Unmap and free mbuf. */
3364         bus_dmamap_sync(ring->data_dmat, txd->map, BUS_DMASYNC_POSTWRITE);
3365         bus_dmamap_unload(ring->data_dmat, txd->map);
3366
3367         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3368             "free txd %p, in %p\n", txd, txd->in);
3369         txd->done = 1;
3370         txd->m = NULL;
3371         txd->in = NULL;
3372
3373         ieee80211_tx_complete(&in->in_ni, m, status);
3374
3375         if (--ring->queued < IWM_TX_RING_LOMARK) {
3376                 sc->qfullmsk &= ~(1 << ring->qid);
3377                 if (sc->qfullmsk == 0) {
3378                         iwm_start(sc);
3379                 }
3380         }
3381 }
3382
3383 /*
3384  * transmit side
3385  */
3386
3387 /*
3388  * Process a "command done" firmware notification.  This is where we wakeup
3389  * processes waiting for a synchronous command completion.
3390  * from if_iwn
3391  */
3392 static void
3393 iwm_cmd_done(struct iwm_softc *sc, struct iwm_rx_packet *pkt)
3394 {
3395         struct iwm_tx_ring *ring = &sc->txq[IWM_MVM_CMD_QUEUE];
3396         struct iwm_tx_data *data;
3397
3398         if (pkt->hdr.qid != IWM_MVM_CMD_QUEUE) {
3399                 return; /* Not a command ack. */
3400         }
3401
3402         /* XXX wide commands? */
3403         IWM_DPRINTF(sc, IWM_DEBUG_CMD,
3404             "cmd notification type 0x%x qid %d idx %d\n",
3405             pkt->hdr.code, pkt->hdr.qid, pkt->hdr.idx);
3406
3407         data = &ring->data[pkt->hdr.idx];
3408
3409         /* If the command was mapped in an mbuf, free it. */
3410         if (data->m != NULL) {
3411                 bus_dmamap_sync(ring->data_dmat, data->map,
3412                     BUS_DMASYNC_POSTWRITE);
3413                 bus_dmamap_unload(ring->data_dmat, data->map);
3414                 m_freem(data->m);
3415                 data->m = NULL;
3416         }
3417         wakeup(&ring->desc[pkt->hdr.idx]);
3418
3419         if (((pkt->hdr.idx + ring->queued) % IWM_TX_RING_COUNT) != ring->cur) {
3420                 device_printf(sc->sc_dev,
3421                     "%s: Some HCMDs skipped?: idx=%d queued=%d cur=%d\n",
3422                     __func__, pkt->hdr.idx, ring->queued, ring->cur);
3423                 /* XXX call iwm_force_nmi() */
3424         }
3425
3426         KASSERT(ring->queued > 0, ("ring->queued is empty?"));
3427         ring->queued--;
3428         if (ring->queued == 0)
3429                 iwm_pcie_clear_cmd_in_flight(sc);
3430 }
3431
3432 #if 0
3433 /*
3434  * necessary only for block ack mode
3435  */
3436 void
3437 iwm_update_sched(struct iwm_softc *sc, int qid, int idx, uint8_t sta_id,
3438         uint16_t len)
3439 {
3440         struct iwm_agn_scd_bc_tbl *scd_bc_tbl;
3441         uint16_t w_val;
3442
3443         scd_bc_tbl = sc->sched_dma.vaddr;
3444
3445         len += 8; /* magic numbers came naturally from paris */
3446         if (sc->sc_capaflags & IWM_UCODE_TLV_FLAGS_DW_BC_TABLE)
3447                 len = roundup(len, 4) / 4;
3448
3449         w_val = htole16(sta_id << 12 | len);
3450
3451         /* Update TX scheduler. */
3452         scd_bc_tbl[qid].tfd_offset[idx] = w_val;
3453         bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map,
3454             BUS_DMASYNC_PREWRITE);
3455
3456         /* I really wonder what this is ?!? */
3457         if (idx < IWM_TFD_QUEUE_SIZE_BC_DUP) {
3458                 scd_bc_tbl[qid].tfd_offset[IWM_TFD_QUEUE_SIZE_MAX + idx] = w_val;
3459                 bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map,
3460                     BUS_DMASYNC_PREWRITE);
3461         }
3462 }
3463 #endif
3464
3465 /*
3466  * Take an 802.11 (non-n) rate, find the relevant rate
3467  * table entry.  return the index into in_ridx[].
3468  *
3469  * The caller then uses that index back into in_ridx
3470  * to figure out the rate index programmed /into/
3471  * the firmware for this given node.
3472  */
3473 static int
3474 iwm_tx_rateidx_lookup(struct iwm_softc *sc, struct iwm_node *in,
3475     uint8_t rate)
3476 {
3477         int i;
3478         uint8_t r;
3479
3480         for (i = 0; i < nitems(in->in_ridx); i++) {
3481                 r = iwm_rates[in->in_ridx[i]].rate;
3482                 if (rate == r)
3483                         return (i);
3484         }
3485
3486         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE,
3487             "%s: couldn't find an entry for rate=%d\n",
3488             __func__,
3489             rate);
3490
3491         /* XXX Return the first */
3492         /* XXX TODO: have it return the /lowest/ */
3493         return (0);
3494 }
3495
3496 static int
3497 iwm_tx_rateidx_global_lookup(struct iwm_softc *sc, uint8_t rate)
3498 {
3499         int i;
3500
3501         for (i = 0; i < nitems(iwm_rates); i++) {
3502                 if (iwm_rates[i].rate == rate)
3503                         return (i);
3504         }
3505         /* XXX error? */
3506         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE,
3507             "%s: couldn't find an entry for rate=%d\n",
3508             __func__,
3509             rate);
3510         return (0);
3511 }
3512
3513 /*
3514  * Fill in the rate related information for a transmit command.
3515  */
3516 static const struct iwm_rate *
3517 iwm_tx_fill_cmd(struct iwm_softc *sc, struct iwm_node *in,
3518         struct mbuf *m, struct iwm_tx_cmd *tx)
3519 {
3520         struct ieee80211_node *ni = &in->in_ni;
3521         struct ieee80211_frame *wh;
3522         const struct ieee80211_txparam *tp = ni->ni_txparms;
3523         const struct iwm_rate *rinfo;
3524         int type;
3525         int ridx, rate_flags;
3526
3527         wh = mtod(m, struct ieee80211_frame *);
3528         type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK;
3529
3530         tx->rts_retry_limit = IWM_RTS_DFAULT_RETRY_LIMIT;
3531         tx->data_retry_limit = IWM_DEFAULT_TX_RETRY;
3532
3533         if (type == IEEE80211_FC0_TYPE_MGT ||
3534             type == IEEE80211_FC0_TYPE_CTL ||
3535             (m->m_flags & M_EAPOL) != 0) {
3536                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->mgmtrate);
3537                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3538                     "%s: MGT (%d)\n", __func__, tp->mgmtrate);
3539         } else if (IEEE80211_IS_MULTICAST(wh->i_addr1)) {
3540                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->mcastrate);
3541                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3542                     "%s: MCAST (%d)\n", __func__, tp->mcastrate);
3543         } else if (tp->ucastrate != IEEE80211_FIXED_RATE_NONE) {
3544                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->ucastrate);
3545                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3546                     "%s: FIXED_RATE (%d)\n", __func__, tp->ucastrate);
3547         } else {
3548                 int i;
3549
3550                 /* for data frames, use RS table */
3551                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: DATA\n", __func__);
3552                 /* XXX pass pktlen */
3553                 (void) ieee80211_ratectl_rate(ni, NULL, 0);
3554                 i = iwm_tx_rateidx_lookup(sc, in, ni->ni_txrate);
3555                 ridx = in->in_ridx[i];
3556
3557                 /* This is the index into the programmed table */
3558                 tx->initial_rate_index = i;
3559                 tx->tx_flags |= htole32(IWM_TX_CMD_FLG_STA_RATE);
3560
3561                 IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE,
3562                     "%s: start with i=%d, txrate %d\n",
3563                     __func__, i, iwm_rates[ridx].rate);
3564         }
3565
3566         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE,
3567             "%s: frame type=%d txrate %d\n",
3568                 __func__, type, iwm_rates[ridx].rate);
3569
3570         rinfo = &iwm_rates[ridx];
3571
3572         IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: ridx=%d; rate=%d, CCK=%d\n",
3573             __func__, ridx,
3574             rinfo->rate,
3575             !! (IWM_RIDX_IS_CCK(ridx))
3576             );
3577
3578         /* XXX TODO: hard-coded TX antenna? */
3579         rate_flags = 1 << IWM_RATE_MCS_ANT_POS;
3580         if (IWM_RIDX_IS_CCK(ridx))
3581                 rate_flags |= IWM_RATE_MCS_CCK_MSK;
3582         tx->rate_n_flags = htole32(rate_flags | rinfo->plcp);
3583
3584         return rinfo;
3585 }
3586
3587 #define TB0_SIZE 16
3588 static int
3589 iwm_tx(struct iwm_softc *sc, struct mbuf *m, struct ieee80211_node *ni, int ac)
3590 {
3591         struct ieee80211com *ic = &sc->sc_ic;
3592         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
3593         struct iwm_node *in = IWM_NODE(ni);
3594         struct iwm_tx_ring *ring;
3595         struct iwm_tx_data *data;
3596         struct iwm_tfd *desc;
3597         struct iwm_device_cmd *cmd;
3598         struct iwm_tx_cmd *tx;
3599         struct ieee80211_frame *wh;
3600         struct ieee80211_key *k = NULL;
3601         struct mbuf *m1;
3602         const struct iwm_rate *rinfo;
3603         uint32_t flags;
3604         u_int hdrlen;
3605         bus_dma_segment_t *seg, segs[IWM_MAX_SCATTER];
3606         int nsegs;
3607         uint8_t tid, type;
3608         int i, totlen, error, pad;
3609
3610         wh = mtod(m, struct ieee80211_frame *);
3611         hdrlen = ieee80211_anyhdrsize(wh);
3612         type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK;
3613         tid = 0;
3614         ring = &sc->txq[ac];
3615         desc = &ring->desc[ring->cur];
3616         memset(desc, 0, sizeof(*desc));
3617         data = &ring->data[ring->cur];
3618
3619         /* Fill out iwm_tx_cmd to send to the firmware */
3620         cmd = &ring->cmd[ring->cur];
3621         cmd->hdr.code = IWM_TX_CMD;
3622         cmd->hdr.flags = 0;
3623         cmd->hdr.qid = ring->qid;
3624         cmd->hdr.idx = ring->cur;
3625
3626         tx = (void *)cmd->data;
3627         memset(tx, 0, sizeof(*tx));
3628
3629         rinfo = iwm_tx_fill_cmd(sc, in, m, tx);
3630
3631         /* Encrypt the frame if need be. */
3632         if (wh->i_fc[1] & IEEE80211_FC1_PROTECTED) {
3633                 /* Retrieve key for TX && do software encryption. */
3634                 k = ieee80211_crypto_encap(ni, m);
3635                 if (k == NULL) {
3636                         m_freem(m);
3637                         return (ENOBUFS);
3638                 }
3639                 /* 802.11 header may have moved. */
3640                 wh = mtod(m, struct ieee80211_frame *);
3641         }
3642
3643         if (ieee80211_radiotap_active_vap(vap)) {
3644                 struct iwm_tx_radiotap_header *tap = &sc->sc_txtap;
3645
3646                 tap->wt_flags = 0;
3647                 tap->wt_chan_freq = htole16(ni->ni_chan->ic_freq);
3648                 tap->wt_chan_flags = htole16(ni->ni_chan->ic_flags);
3649                 tap->wt_rate = rinfo->rate;
3650                 if (k != NULL)
3651                         tap->wt_flags |= IEEE80211_RADIOTAP_F_WEP;
3652                 ieee80211_radiotap_tx(vap, m);
3653         }
3654
3655
3656         totlen = m->m_pkthdr.len;
3657
3658         flags = 0;
3659         if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) {
3660                 flags |= IWM_TX_CMD_FLG_ACK;
3661         }
3662
3663         if (type == IEEE80211_FC0_TYPE_DATA
3664             && (totlen + IEEE80211_CRC_LEN > vap->iv_rtsthreshold)
3665             && !IEEE80211_IS_MULTICAST(wh->i_addr1)) {
3666                 flags |= IWM_TX_CMD_FLG_PROT_REQUIRE;
3667         }
3668
3669         if (IEEE80211_IS_MULTICAST(wh->i_addr1) ||
3670             type != IEEE80211_FC0_TYPE_DATA)
3671                 tx->sta_id = sc->sc_aux_sta.sta_id;
3672         else
3673                 tx->sta_id = IWM_STATION_ID;
3674
3675         if (type == IEEE80211_FC0_TYPE_MGT) {
3676                 uint8_t subtype = wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK;
3677
3678                 if (subtype == IEEE80211_FC0_SUBTYPE_ASSOC_REQ ||
3679                     subtype == IEEE80211_FC0_SUBTYPE_REASSOC_REQ) {
3680                         tx->pm_frame_timeout = htole16(IWM_PM_FRAME_ASSOC);
3681                 } else if (subtype == IEEE80211_FC0_SUBTYPE_ACTION) {
3682                         tx->pm_frame_timeout = htole16(IWM_PM_FRAME_NONE);
3683                 } else {
3684                         tx->pm_frame_timeout = htole16(IWM_PM_FRAME_MGMT);
3685                 }
3686         } else {
3687                 tx->pm_frame_timeout = htole16(IWM_PM_FRAME_NONE);
3688         }
3689
3690         if (hdrlen & 3) {
3691                 /* First segment length must be a multiple of 4. */
3692                 flags |= IWM_TX_CMD_FLG_MH_PAD;
3693                 pad = 4 - (hdrlen & 3);
3694         } else
3695                 pad = 0;
3696
3697         tx->driver_txop = 0;
3698         tx->next_frame_len = 0;
3699
3700         tx->len = htole16(totlen);
3701         tx->tid_tspec = tid;
3702         tx->life_time = htole32(IWM_TX_CMD_LIFE_TIME_INFINITE);
3703
3704         /* Set physical address of "scratch area". */
3705         tx->dram_lsb_ptr = htole32(data->scratch_paddr);
3706         tx->dram_msb_ptr = iwm_get_dma_hi_addr(data->scratch_paddr);
3707
3708         /* Copy 802.11 header in TX command. */
3709         memcpy(((uint8_t *)tx) + sizeof(*tx), wh, hdrlen);
3710
3711         flags |= IWM_TX_CMD_FLG_BT_DIS | IWM_TX_CMD_FLG_SEQ_CTL;
3712
3713         tx->sec_ctl = 0;
3714         tx->tx_flags |= htole32(flags);
3715
3716         /* Trim 802.11 header. */
3717         m_adj(m, hdrlen);
3718         error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m,
3719             segs, &nsegs, BUS_DMA_NOWAIT);
3720         if (error != 0) {
3721                 if (error != EFBIG) {
3722                         device_printf(sc->sc_dev, "can't map mbuf (error %d)\n",
3723                             error);
3724                         m_freem(m);
3725                         return error;
3726                 }
3727                 /* Too many DMA segments, linearize mbuf. */
3728                 m1 = m_collapse(m, M_NOWAIT, IWM_MAX_SCATTER - 2);
3729                 if (m1 == NULL) {
3730                         device_printf(sc->sc_dev,
3731                             "%s: could not defrag mbuf\n", __func__);
3732                         m_freem(m);
3733                         return (ENOBUFS);
3734                 }
3735                 m = m1;
3736
3737                 error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m,
3738                     segs, &nsegs, BUS_DMA_NOWAIT);
3739                 if (error != 0) {
3740                         device_printf(sc->sc_dev, "can't map mbuf (error %d)\n",
3741                             error);
3742                         m_freem(m);
3743                         return error;
3744                 }
3745         }
3746         data->m = m;
3747         data->in = in;
3748         data->done = 0;
3749
3750         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3751             "sending txd %p, in %p\n", data, data->in);
3752         KASSERT(data->in != NULL, ("node is NULL"));
3753
3754         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3755             "sending data: qid=%d idx=%d len=%d nsegs=%d txflags=0x%08x rate_n_flags=0x%08x rateidx=%u\n",
3756             ring->qid, ring->cur, totlen, nsegs,
3757             le32toh(tx->tx_flags),
3758             le32toh(tx->rate_n_flags),
3759             tx->initial_rate_index
3760             );
3761
3762         /* Fill TX descriptor. */
3763         desc->num_tbs = 2 + nsegs;
3764
3765         desc->tbs[0].lo = htole32(data->cmd_paddr);
3766         desc->tbs[0].hi_n_len = htole16(iwm_get_dma_hi_addr(data->cmd_paddr)) |
3767             (TB0_SIZE << 4);
3768         desc->tbs[1].lo = htole32(data->cmd_paddr + TB0_SIZE);
3769         desc->tbs[1].hi_n_len = htole16(iwm_get_dma_hi_addr(data->cmd_paddr)) |
3770             ((sizeof(struct iwm_cmd_header) + sizeof(*tx)
3771               + hdrlen + pad - TB0_SIZE) << 4);
3772
3773         /* Other DMA segments are for data payload. */
3774         for (i = 0; i < nsegs; i++) {
3775                 seg = &segs[i];
3776                 desc->tbs[i+2].lo = htole32(seg->ds_addr);
3777                 desc->tbs[i+2].hi_n_len = \
3778                     htole16(iwm_get_dma_hi_addr(seg->ds_addr))
3779                     | ((seg->ds_len) << 4);
3780         }
3781
3782         bus_dmamap_sync(ring->data_dmat, data->map,
3783             BUS_DMASYNC_PREWRITE);
3784         bus_dmamap_sync(ring->cmd_dma.tag, ring->cmd_dma.map,
3785             BUS_DMASYNC_PREWRITE);
3786         bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map,
3787             BUS_DMASYNC_PREWRITE);
3788
3789 #if 0
3790         iwm_update_sched(sc, ring->qid, ring->cur, tx->sta_id, le16toh(tx->len));
3791 #endif
3792
3793         /* Kick TX ring. */
3794         ring->cur = (ring->cur + 1) % IWM_TX_RING_COUNT;
3795         IWM_WRITE(sc, IWM_HBUS_TARG_WRPTR, ring->qid << 8 | ring->cur);
3796
3797         /* Mark TX ring as full if we reach a certain threshold. */
3798         if (++ring->queued > IWM_TX_RING_HIMARK) {
3799                 sc->qfullmsk |= 1 << ring->qid;
3800         }
3801
3802         return 0;
3803 }
3804
3805 static int
3806 iwm_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
3807     const struct ieee80211_bpf_params *params)
3808 {
3809         struct ieee80211com *ic = ni->ni_ic;
3810         struct iwm_softc *sc = ic->ic_softc;
3811         int error = 0;
3812
3813         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3814             "->%s begin\n", __func__);
3815
3816         if ((sc->sc_flags & IWM_FLAG_HW_INITED) == 0) {
3817                 m_freem(m);
3818                 IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3819                     "<-%s not RUNNING\n", __func__);
3820                 return (ENETDOWN);
3821         }
3822
3823         IWM_LOCK(sc);
3824         /* XXX fix this */
3825         if (params == NULL) {
3826                 error = iwm_tx(sc, m, ni, 0);
3827         } else {
3828                 error = iwm_tx(sc, m, ni, 0);
3829         }
3830         sc->sc_tx_timer = 5;
3831         IWM_UNLOCK(sc);
3832
3833         return (error);
3834 }
3835
3836 /*
3837  * mvm/tx.c
3838  */
3839
3840 /*
3841  * Note that there are transports that buffer frames before they reach
3842  * the firmware. This means that after flush_tx_path is called, the
3843  * queue might not be empty. The race-free way to handle this is to:
3844  * 1) set the station as draining
3845  * 2) flush the Tx path
3846  * 3) wait for the transport queues to be empty
3847  */
3848 int
3849 iwm_mvm_flush_tx_path(struct iwm_softc *sc, uint32_t tfd_msk, uint32_t flags)
3850 {
3851         int ret;
3852         struct iwm_tx_path_flush_cmd flush_cmd = {
3853                 .queues_ctl = htole32(tfd_msk),
3854                 .flush_ctl = htole16(IWM_DUMP_TX_FIFO_FLUSH),
3855         };
3856
3857         ret = iwm_mvm_send_cmd_pdu(sc, IWM_TXPATH_FLUSH, flags,
3858             sizeof(flush_cmd), &flush_cmd);
3859         if (ret)
3860                 device_printf(sc->sc_dev,
3861                     "Flushing tx queue failed: %d\n", ret);
3862         return ret;
3863 }
3864
3865 /*
3866  * BEGIN mvm/sta.c
3867  */
3868
3869 static int
3870 iwm_mvm_send_add_sta_cmd_status(struct iwm_softc *sc,
3871         struct iwm_mvm_add_sta_cmd *cmd, int *status)
3872 {
3873         return iwm_mvm_send_cmd_pdu_status(sc, IWM_ADD_STA, sizeof(*cmd),
3874             cmd, status);
3875 }
3876
3877 /* send station add/update command to firmware */
3878 static int
3879 iwm_mvm_sta_send_to_fw(struct iwm_softc *sc, struct iwm_node *in, int update)
3880 {
3881         struct iwm_mvm_add_sta_cmd add_sta_cmd;
3882         int ret;
3883         uint32_t status;
3884
3885         memset(&add_sta_cmd, 0, sizeof(add_sta_cmd));
3886
3887         add_sta_cmd.sta_id = IWM_STATION_ID;
3888         add_sta_cmd.mac_id_n_color
3889             = htole32(IWM_FW_CMD_ID_AND_COLOR(IWM_DEFAULT_MACID,
3890                 IWM_DEFAULT_COLOR));
3891         if (!update) {
3892                 int ac;
3893                 for (ac = 0; ac < WME_NUM_AC; ac++) {
3894                         add_sta_cmd.tfd_queue_msk |=
3895                             htole32(1 << iwm_mvm_ac_to_tx_fifo[ac]);
3896                 }
3897                 IEEE80211_ADDR_COPY(&add_sta_cmd.addr, in->in_ni.ni_bssid);
3898         }
3899         add_sta_cmd.add_modify = update ? 1 : 0;
3900         add_sta_cmd.station_flags_msk
3901             |= htole32(IWM_STA_FLG_FAT_EN_MSK | IWM_STA_FLG_MIMO_EN_MSK);
3902         add_sta_cmd.tid_disable_tx = htole16(0xffff);
3903         if (update)
3904                 add_sta_cmd.modify_mask |= (IWM_STA_MODIFY_TID_DISABLE_TX);
3905
3906         status = IWM_ADD_STA_SUCCESS;
3907         ret = iwm_mvm_send_add_sta_cmd_status(sc, &add_sta_cmd, &status);
3908         if (ret)
3909                 return ret;
3910
3911         switch (status) {
3912         case IWM_ADD_STA_SUCCESS:
3913                 break;
3914         default:
3915                 ret = EIO;
3916                 device_printf(sc->sc_dev, "IWM_ADD_STA failed\n");
3917                 break;
3918         }
3919
3920         return ret;
3921 }
3922
3923 static int
3924 iwm_mvm_add_sta(struct iwm_softc *sc, struct iwm_node *in)
3925 {
3926         return iwm_mvm_sta_send_to_fw(sc, in, 0);
3927 }
3928
3929 static int
3930 iwm_mvm_update_sta(struct iwm_softc *sc, struct iwm_node *in)
3931 {
3932         return iwm_mvm_sta_send_to_fw(sc, in, 1);
3933 }
3934
3935 static int
3936 iwm_mvm_add_int_sta_common(struct iwm_softc *sc, struct iwm_int_sta *sta,
3937         const uint8_t *addr, uint16_t mac_id, uint16_t color)
3938 {
3939         struct iwm_mvm_add_sta_cmd cmd;
3940         int ret;
3941         uint32_t status;
3942
3943         memset(&cmd, 0, sizeof(cmd));
3944         cmd.sta_id = sta->sta_id;
3945         cmd.mac_id_n_color = htole32(IWM_FW_CMD_ID_AND_COLOR(mac_id, color));
3946
3947         cmd.tfd_queue_msk = htole32(sta->tfd_queue_msk);
3948         cmd.tid_disable_tx = htole16(0xffff);
3949
3950         if (addr)
3951                 IEEE80211_ADDR_COPY(cmd.addr, addr);
3952
3953         ret = iwm_mvm_send_add_sta_cmd_status(sc, &cmd, &status);
3954         if (ret)
3955                 return ret;
3956
3957         switch (status) {
3958         case IWM_ADD_STA_SUCCESS:
3959                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
3960                     "%s: Internal station added.\n", __func__);
3961                 return 0;
3962         default:
3963                 device_printf(sc->sc_dev,
3964                     "%s: Add internal station failed, status=0x%x\n",
3965                     __func__, status);
3966                 ret = EIO;
3967                 break;
3968         }
3969         return ret;
3970 }
3971
3972 static int
3973 iwm_mvm_add_aux_sta(struct iwm_softc *sc)
3974 {
3975         int ret;
3976
3977         sc->sc_aux_sta.sta_id = IWM_AUX_STA_ID;
3978         sc->sc_aux_sta.tfd_queue_msk = (1 << IWM_MVM_AUX_QUEUE);
3979
3980         ret = iwm_enable_txq(sc, 0, IWM_MVM_AUX_QUEUE, IWM_MVM_TX_FIFO_MCAST);
3981         if (ret)
3982                 return ret;
3983
3984         ret = iwm_mvm_add_int_sta_common(sc,
3985             &sc->sc_aux_sta, NULL, IWM_MAC_INDEX_AUX, 0);
3986
3987         if (ret)
3988                 memset(&sc->sc_aux_sta, 0, sizeof(sc->sc_aux_sta));
3989         return ret;
3990 }
3991
3992 /*
3993  * END mvm/sta.c
3994  */
3995
3996 /*
3997  * BEGIN mvm/quota.c
3998  */
3999
4000 static int
4001 iwm_mvm_update_quotas(struct iwm_softc *sc, struct iwm_node *in)
4002 {
4003         struct iwm_time_quota_cmd cmd;
4004         int i, idx, ret, num_active_macs, quota, quota_rem;
4005         int colors[IWM_MAX_BINDINGS] = { -1, -1, -1, -1, };
4006         int n_ifs[IWM_MAX_BINDINGS] = {0, };
4007         uint16_t id;
4008
4009         memset(&cmd, 0, sizeof(cmd));
4010
4011         /* currently, PHY ID == binding ID */
4012         if (in) {
4013                 id = in->in_phyctxt->id;
4014                 KASSERT(id < IWM_MAX_BINDINGS, ("invalid id"));
4015                 colors[id] = in->in_phyctxt->color;
4016
4017                 if (1)
4018                         n_ifs[id] = 1;
4019         }
4020
4021         /*
4022          * The FW's scheduling session consists of
4023          * IWM_MVM_MAX_QUOTA fragments. Divide these fragments
4024          * equally between all the bindings that require quota
4025          */
4026         num_active_macs = 0;
4027         for (i = 0; i < IWM_MAX_BINDINGS; i++) {
4028                 cmd.quotas[i].id_and_color = htole32(IWM_FW_CTXT_INVALID);
4029                 num_active_macs += n_ifs[i];
4030         }
4031
4032         quota = 0;
4033         quota_rem = 0;
4034         if (num_active_macs) {
4035                 quota = IWM_MVM_MAX_QUOTA / num_active_macs;
4036                 quota_rem = IWM_MVM_MAX_QUOTA % num_active_macs;
4037         }
4038
4039         for (idx = 0, i = 0; i < IWM_MAX_BINDINGS; i++) {
4040                 if (colors[i] < 0)
4041                         continue;
4042
4043                 cmd.quotas[idx].id_and_color =
4044                         htole32(IWM_FW_CMD_ID_AND_COLOR(i, colors[i]));
4045
4046                 if (n_ifs[i] <= 0) {
4047                         cmd.quotas[idx].quota = htole32(0);
4048                         cmd.quotas[idx].max_duration = htole32(0);
4049                 } else {
4050                         cmd.quotas[idx].quota = htole32(quota * n_ifs[i]);
4051                         cmd.quotas[idx].max_duration = htole32(0);
4052                 }
4053                 idx++;
4054         }
4055
4056         /* Give the remainder of the session to the first binding */
4057         cmd.quotas[0].quota = htole32(le32toh(cmd.quotas[0].quota) + quota_rem);
4058
4059         ret = iwm_mvm_send_cmd_pdu(sc, IWM_TIME_QUOTA_CMD, IWM_CMD_SYNC,
4060             sizeof(cmd), &cmd);
4061         if (ret)
4062                 device_printf(sc->sc_dev,
4063                     "%s: Failed to send quota: %d\n", __func__, ret);
4064         return ret;
4065 }
4066
4067 /*
4068  * END mvm/quota.c
4069  */
4070
4071 /*
4072  * ieee80211 routines
4073  */
4074
4075 /*
4076  * Change to AUTH state in 80211 state machine.  Roughly matches what
4077  * Linux does in bss_info_changed().
4078  */
4079 static int
4080 iwm_auth(struct ieee80211vap *vap, struct iwm_softc *sc)
4081 {
4082         struct ieee80211_node *ni;
4083         struct iwm_node *in;
4084         struct iwm_vap *iv = IWM_VAP(vap);
4085         uint32_t duration;
4086         int error;
4087
4088         /*
4089          * XXX i have a feeling that the vap node is being
4090          * freed from underneath us. Grr.
4091          */
4092         ni = ieee80211_ref_node(vap->iv_bss);
4093         in = IWM_NODE(ni);
4094         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_STATE,
4095             "%s: called; vap=%p, bss ni=%p\n",
4096             __func__,
4097             vap,
4098             ni);
4099
4100         in->in_assoc = 0;
4101
4102         error = iwm_mvm_sf_config(sc, IWM_SF_FULL_ON);
4103         if (error != 0)
4104                 return error;
4105
4106         error = iwm_allow_mcast(vap, sc);
4107         if (error) {
4108                 device_printf(sc->sc_dev,
4109                     "%s: failed to set multicast\n", __func__);
4110                 goto out;
4111         }
4112
4113         /*
4114          * This is where it deviates from what Linux does.
4115          *
4116          * Linux iwlwifi doesn't reset the nic each time, nor does it
4117          * call ctxt_add() here.  Instead, it adds it during vap creation,
4118          * and always does a mac_ctx_changed().
4119          *
4120          * The openbsd port doesn't attempt to do that - it reset things
4121          * at odd states and does the add here.
4122          *
4123          * So, until the state handling is fixed (ie, we never reset
4124          * the NIC except for a firmware failure, which should drag
4125          * the NIC back to IDLE, re-setup and re-add all the mac/phy
4126          * contexts that are required), let's do a dirty hack here.
4127          */
4128         if (iv->is_uploaded) {
4129                 if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) {
4130                         device_printf(sc->sc_dev,
4131                             "%s: failed to update MAC\n", __func__);
4132                         goto out;
4133                 }
4134                 if ((error = iwm_mvm_phy_ctxt_changed(sc, &sc->sc_phyctxt[0],
4135                     in->in_ni.ni_chan, 1, 1)) != 0) {
4136                         device_printf(sc->sc_dev,
4137                             "%s: failed update phy ctxt\n", __func__);
4138                         goto out;
4139                 }
4140                 in->in_phyctxt = &sc->sc_phyctxt[0];
4141
4142                 if ((error = iwm_mvm_binding_update(sc, in)) != 0) {
4143                         device_printf(sc->sc_dev,
4144                             "%s: binding update cmd\n", __func__);
4145                         goto out;
4146                 }
4147                 if ((error = iwm_mvm_update_sta(sc, in)) != 0) {
4148                         device_printf(sc->sc_dev,
4149                             "%s: failed to update sta\n", __func__);
4150                         goto out;
4151                 }
4152         } else {
4153                 if ((error = iwm_mvm_mac_ctxt_add(sc, vap)) != 0) {
4154                         device_printf(sc->sc_dev,
4155                             "%s: failed to add MAC\n", __func__);
4156                         goto out;
4157                 }
4158                 if ((error = iwm_mvm_phy_ctxt_changed(sc, &sc->sc_phyctxt[0],
4159                     in->in_ni.ni_chan, 1, 1)) != 0) {
4160                         device_printf(sc->sc_dev,
4161                             "%s: failed add phy ctxt!\n", __func__);
4162                         error = ETIMEDOUT;
4163                         goto out;
4164                 }
4165                 in->in_phyctxt = &sc->sc_phyctxt[0];
4166
4167                 if ((error = iwm_mvm_binding_add_vif(sc, in)) != 0) {
4168                         device_printf(sc->sc_dev,
4169                             "%s: binding add cmd\n", __func__);
4170                         goto out;
4171                 }
4172                 if ((error = iwm_mvm_add_sta(sc, in)) != 0) {
4173                         device_printf(sc->sc_dev,
4174                             "%s: failed to add sta\n", __func__);
4175                         goto out;
4176                 }
4177         }
4178
4179         /*
4180          * Prevent the FW from wandering off channel during association
4181          * by "protecting" the session with a time event.
4182          */
4183         /* XXX duration is in units of TU, not MS */
4184         duration = IWM_MVM_TE_SESSION_PROTECTION_MAX_TIME_MS;
4185         iwm_mvm_protect_session(sc, in, duration, 500 /* XXX magic number */);
4186         DELAY(100);
4187
4188         error = 0;
4189 out:
4190         ieee80211_free_node(ni);
4191         return (error);
4192 }
4193
4194 static int
4195 iwm_assoc(struct ieee80211vap *vap, struct iwm_softc *sc)
4196 {
4197         struct iwm_node *in = IWM_NODE(vap->iv_bss);
4198         int error;
4199
4200         if ((error = iwm_mvm_update_sta(sc, in)) != 0) {
4201                 device_printf(sc->sc_dev,
4202                     "%s: failed to update STA\n", __func__);
4203                 return error;
4204         }
4205
4206         in->in_assoc = 1;
4207         if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) {
4208                 device_printf(sc->sc_dev,
4209                     "%s: failed to update MAC\n", __func__);
4210                 return error;
4211         }
4212
4213         return 0;
4214 }
4215
4216 static int
4217 iwm_release(struct iwm_softc *sc, struct iwm_node *in)
4218 {
4219         uint32_t tfd_msk;
4220
4221         /*
4222          * Ok, so *technically* the proper set of calls for going
4223          * from RUN back to SCAN is:
4224          *
4225          * iwm_mvm_power_mac_disable(sc, in);
4226          * iwm_mvm_mac_ctxt_changed(sc, in);
4227          * iwm_mvm_rm_sta(sc, in);
4228          * iwm_mvm_update_quotas(sc, NULL);
4229          * iwm_mvm_mac_ctxt_changed(sc, in);
4230          * iwm_mvm_binding_remove_vif(sc, in);
4231          * iwm_mvm_mac_ctxt_remove(sc, in);
4232          *
4233          * However, that freezes the device not matter which permutations
4234          * and modifications are attempted.  Obviously, this driver is missing
4235          * something since it works in the Linux driver, but figuring out what
4236          * is missing is a little more complicated.  Now, since we're going
4237          * back to nothing anyway, we'll just do a complete device reset.
4238          * Up your's, device!
4239          */
4240         /*
4241          * Just using 0xf for the queues mask is fine as long as we only
4242          * get here from RUN state.
4243          */
4244         tfd_msk = 0xf;
4245         mbufq_drain(&sc->sc_snd);
4246         iwm_mvm_flush_tx_path(sc, tfd_msk, IWM_CMD_SYNC);
4247         /*
4248          * We seem to get away with just synchronously sending the
4249          * IWM_TXPATH_FLUSH command.
4250          */
4251 //      iwm_trans_wait_tx_queue_empty(sc, tfd_msk);
4252         iwm_stop_device(sc);
4253         iwm_init_hw(sc);
4254         if (in)
4255                 in->in_assoc = 0;
4256         return 0;
4257
4258 #if 0
4259         int error;
4260
4261         iwm_mvm_power_mac_disable(sc, in);
4262
4263         if ((error = iwm_mvm_mac_ctxt_changed(sc, in)) != 0) {
4264                 device_printf(sc->sc_dev, "mac ctxt change fail 1 %d\n", error);
4265                 return error;
4266         }
4267
4268         if ((error = iwm_mvm_rm_sta(sc, in)) != 0) {
4269                 device_printf(sc->sc_dev, "sta remove fail %d\n", error);
4270                 return error;
4271         }
4272         error = iwm_mvm_rm_sta(sc, in);
4273         in->in_assoc = 0;
4274         iwm_mvm_update_quotas(sc, NULL);
4275         if ((error = iwm_mvm_mac_ctxt_changed(sc, in)) != 0) {
4276                 device_printf(sc->sc_dev, "mac ctxt change fail 2 %d\n", error);
4277                 return error;
4278         }
4279         iwm_mvm_binding_remove_vif(sc, in);
4280
4281         iwm_mvm_mac_ctxt_remove(sc, in);
4282
4283         return error;
4284 #endif
4285 }
4286
4287 static struct ieee80211_node *
4288 iwm_node_alloc(struct ieee80211vap *vap, const uint8_t mac[IEEE80211_ADDR_LEN])
4289 {
4290         return malloc(sizeof (struct iwm_node), M_80211_NODE,
4291             M_NOWAIT | M_ZERO);
4292 }
4293
4294 uint8_t
4295 iwm_ridx2rate(struct ieee80211_rateset *rs, int ridx)
4296 {
4297         int i;
4298         uint8_t rval;
4299
4300         for (i = 0; i < rs->rs_nrates; i++) {
4301                 rval = (rs->rs_rates[i] & IEEE80211_RATE_VAL);
4302                 if (rval == iwm_rates[ridx].rate)
4303                         return rs->rs_rates[i];
4304         }
4305
4306         return 0;
4307 }
4308
4309 static void
4310 iwm_setrates(struct iwm_softc *sc, struct iwm_node *in)
4311 {
4312         struct ieee80211_node *ni = &in->in_ni;
4313         struct iwm_lq_cmd *lq = &in->in_lq;
4314         int nrates = ni->ni_rates.rs_nrates;
4315         int i, ridx, tab = 0;
4316 //      int txant = 0;
4317
4318         if (nrates > nitems(lq->rs_table)) {
4319                 device_printf(sc->sc_dev,
4320                     "%s: node supports %d rates, driver handles "
4321                     "only %zu\n", __func__, nrates, nitems(lq->rs_table));
4322                 return;
4323         }
4324         if (nrates == 0) {
4325                 device_printf(sc->sc_dev,
4326                     "%s: node supports 0 rates, odd!\n", __func__);
4327                 return;
4328         }
4329
4330         /*
4331          * XXX .. and most of iwm_node is not initialised explicitly;
4332          * it's all just 0x0 passed to the firmware.
4333          */
4334
4335         /* first figure out which rates we should support */
4336         /* XXX TODO: this isn't 11n aware /at all/ */
4337         memset(&in->in_ridx, -1, sizeof(in->in_ridx));
4338         IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
4339             "%s: nrates=%d\n", __func__, nrates);
4340
4341         /*
4342          * Loop over nrates and populate in_ridx from the highest
4343          * rate to the lowest rate.  Remember, in_ridx[] has
4344          * IEEE80211_RATE_MAXSIZE entries!
4345          */
4346         for (i = 0; i < min(nrates, IEEE80211_RATE_MAXSIZE); i++) {
4347                 int rate = ni->ni_rates.rs_rates[(nrates - 1) - i] & IEEE80211_RATE_VAL;
4348
4349                 /* Map 802.11 rate to HW rate index. */
4350                 for (ridx = 0; ridx <= IWM_RIDX_MAX; ridx++)
4351                         if (iwm_rates[ridx].rate == rate)
4352                                 break;
4353                 if (ridx > IWM_RIDX_MAX) {
4354                         device_printf(sc->sc_dev,
4355                             "%s: WARNING: device rate for %d not found!\n",
4356                             __func__, rate);
4357                 } else {
4358                         IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
4359                             "%s: rate: i: %d, rate=%d, ridx=%d\n",
4360                             __func__,
4361                             i,
4362                             rate,
4363                             ridx);
4364                         in->in_ridx[i] = ridx;
4365                 }
4366         }
4367
4368         /* then construct a lq_cmd based on those */
4369         memset(lq, 0, sizeof(*lq));
4370         lq->sta_id = IWM_STATION_ID;
4371
4372         /* For HT, always enable RTS/CTS to avoid excessive retries. */
4373         if (ni->ni_flags & IEEE80211_NODE_HT)
4374                 lq->flags |= IWM_LQ_FLAG_USE_RTS_MSK;
4375
4376         /*
4377          * are these used? (we don't do SISO or MIMO)
4378          * need to set them to non-zero, though, or we get an error.
4379          */
4380         lq->single_stream_ant_msk = 1;
4381         lq->dual_stream_ant_msk = 1;
4382
4383         /*
4384          * Build the actual rate selection table.
4385          * The lowest bits are the rates.  Additionally,
4386          * CCK needs bit 9 to be set.  The rest of the bits
4387          * we add to the table select the tx antenna
4388          * Note that we add the rates in the highest rate first
4389          * (opposite of ni_rates).
4390          */
4391         /*
4392          * XXX TODO: this should be looping over the min of nrates
4393          * and LQ_MAX_RETRY_NUM.  Sigh.
4394          */
4395         for (i = 0; i < nrates; i++) {
4396                 int nextant;
4397
4398 #if 0
4399                 if (txant == 0)
4400                         txant = iwm_mvm_get_valid_tx_ant(sc);
4401                 nextant = 1<<(ffs(txant)-1);
4402                 txant &= ~nextant;
4403 #else
4404                 nextant = iwm_mvm_get_valid_tx_ant(sc);
4405 #endif
4406                 /*
4407                  * Map the rate id into a rate index into
4408                  * our hardware table containing the
4409                  * configuration to use for this rate.
4410                  */
4411                 ridx = in->in_ridx[i];
4412                 tab = iwm_rates[ridx].plcp;
4413                 tab |= nextant << IWM_RATE_MCS_ANT_POS;
4414                 if (IWM_RIDX_IS_CCK(ridx))
4415                         tab |= IWM_RATE_MCS_CCK_MSK;
4416                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
4417                     "station rate i=%d, rate=%d, hw=%x\n",
4418                     i, iwm_rates[ridx].rate, tab);
4419                 lq->rs_table[i] = htole32(tab);
4420         }
4421         /* then fill the rest with the lowest possible rate */
4422         for (i = nrates; i < nitems(lq->rs_table); i++) {
4423                 KASSERT(tab != 0, ("invalid tab"));
4424                 lq->rs_table[i] = htole32(tab);
4425         }
4426 }
4427
4428 static int
4429 iwm_media_change(struct ifnet *ifp)
4430 {
4431         struct ieee80211vap *vap = ifp->if_softc;
4432         struct ieee80211com *ic = vap->iv_ic;
4433         struct iwm_softc *sc = ic->ic_softc;
4434         int error;
4435
4436         error = ieee80211_media_change(ifp);
4437         if (error != ENETRESET)
4438                 return error;
4439
4440         IWM_LOCK(sc);
4441         if (ic->ic_nrunning > 0) {
4442                 iwm_stop(sc);
4443                 iwm_init(sc);
4444         }
4445         IWM_UNLOCK(sc);
4446         return error;
4447 }
4448
4449
4450 static int
4451 iwm_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
4452 {
4453         struct iwm_vap *ivp = IWM_VAP(vap);
4454         struct ieee80211com *ic = vap->iv_ic;
4455         struct iwm_softc *sc = ic->ic_softc;
4456         struct iwm_node *in;
4457         int error;
4458
4459         IWM_DPRINTF(sc, IWM_DEBUG_STATE,
4460             "switching state %s -> %s\n",
4461             ieee80211_state_name[vap->iv_state],
4462             ieee80211_state_name[nstate]);
4463         IEEE80211_UNLOCK(ic);
4464         IWM_LOCK(sc);
4465
4466         if (vap->iv_state == IEEE80211_S_SCAN && nstate != vap->iv_state)
4467                 iwm_led_blink_stop(sc);
4468
4469         /* disable beacon filtering if we're hopping out of RUN */
4470         if (vap->iv_state == IEEE80211_S_RUN && nstate != vap->iv_state) {
4471                 iwm_mvm_disable_beacon_filter(sc);
4472
4473                 if (((in = IWM_NODE(vap->iv_bss)) != NULL))
4474                         in->in_assoc = 0;
4475
4476                 if (nstate == IEEE80211_S_INIT) {
4477                         IWM_UNLOCK(sc);
4478                         IEEE80211_LOCK(ic);
4479                         error = ivp->iv_newstate(vap, nstate, arg);
4480                         IEEE80211_UNLOCK(ic);
4481                         IWM_LOCK(sc);
4482                         iwm_release(sc, NULL);
4483                         IWM_UNLOCK(sc);
4484                         IEEE80211_LOCK(ic);
4485                         return error;
4486                 }
4487
4488                 /*
4489                  * It's impossible to directly go RUN->SCAN. If we iwm_release()
4490                  * above then the card will be completely reinitialized,
4491                  * so the driver must do everything necessary to bring the card
4492                  * from INIT to SCAN.
4493                  *
4494                  * Additionally, upon receiving deauth frame from AP,
4495                  * OpenBSD 802.11 stack puts the driver in IEEE80211_S_AUTH
4496                  * state. This will also fail with this driver, so bring the FSM
4497                  * from IEEE80211_S_RUN to IEEE80211_S_SCAN in this case as well.
4498                  *
4499                  * XXX TODO: fix this for FreeBSD!
4500                  */
4501                 if (nstate == IEEE80211_S_SCAN ||
4502                     nstate == IEEE80211_S_AUTH ||
4503                     nstate == IEEE80211_S_ASSOC) {
4504                         IWM_DPRINTF(sc, IWM_DEBUG_STATE,
4505                             "Force transition to INIT; MGT=%d\n", arg);
4506                         IWM_UNLOCK(sc);
4507                         IEEE80211_LOCK(ic);
4508                         /* Always pass arg as -1 since we can't Tx right now. */
4509                         /*
4510                          * XXX arg is just ignored anyway when transitioning
4511                          *     to IEEE80211_S_INIT.
4512                          */
4513                         vap->iv_newstate(vap, IEEE80211_S_INIT, -1);
4514                         IWM_DPRINTF(sc, IWM_DEBUG_STATE,
4515                             "Going INIT->SCAN\n");
4516                         nstate = IEEE80211_S_SCAN;
4517                         IEEE80211_UNLOCK(ic);
4518                         IWM_LOCK(sc);
4519                 }
4520         }
4521
4522         switch (nstate) {
4523         case IEEE80211_S_INIT:
4524                 break;
4525
4526         case IEEE80211_S_AUTH:
4527                 if ((error = iwm_auth(vap, sc)) != 0) {
4528                         device_printf(sc->sc_dev,
4529                             "%s: could not move to auth state: %d\n",
4530                             __func__, error);
4531                         break;
4532                 }
4533                 break;
4534
4535         case IEEE80211_S_ASSOC:
4536                 /*
4537                  * EBS may be disabled due to previous failures reported by FW.
4538                  * Reset EBS status here assuming environment has been changed.
4539                  */
4540                 sc->last_ebs_successful = TRUE;
4541                 if ((error = iwm_assoc(vap, sc)) != 0) {
4542                         device_printf(sc->sc_dev,
4543                             "%s: failed to associate: %d\n", __func__,
4544                             error);
4545                         break;
4546                 }
4547                 break;
4548
4549         case IEEE80211_S_RUN:
4550         {
4551                 struct iwm_host_cmd cmd = {
4552                         .id = IWM_LQ_CMD,
4553                         .len = { sizeof(in->in_lq), },
4554                         .flags = IWM_CMD_SYNC,
4555                 };
4556
4557                 /* Update the association state, now we have it all */
4558                 /* (eg associd comes in at this point */
4559                 error = iwm_assoc(vap, sc);
4560                 if (error != 0) {
4561                         device_printf(sc->sc_dev,
4562                             "%s: failed to update association state: %d\n",
4563                             __func__,
4564                             error);
4565                         break;
4566                 }
4567
4568                 in = IWM_NODE(vap->iv_bss);
4569                 iwm_mvm_power_mac_update_mode(sc, in);
4570                 iwm_mvm_enable_beacon_filter(sc, in);
4571                 iwm_mvm_update_quotas(sc, in);
4572                 iwm_setrates(sc, in);
4573
4574                 cmd.data[0] = &in->in_lq;
4575                 if ((error = iwm_send_cmd(sc, &cmd)) != 0) {
4576                         device_printf(sc->sc_dev,
4577                             "%s: IWM_LQ_CMD failed\n", __func__);
4578                 }
4579
4580                 iwm_mvm_led_enable(sc);
4581                 break;
4582         }
4583
4584         default:
4585                 break;
4586         }
4587         IWM_UNLOCK(sc);
4588         IEEE80211_LOCK(ic);
4589
4590         return (ivp->iv_newstate(vap, nstate, arg));
4591 }
4592
4593 void
4594 iwm_endscan_cb(void *arg, int pending)
4595 {
4596         struct iwm_softc *sc = arg;
4597         struct ieee80211com *ic = &sc->sc_ic;
4598
4599         IWM_DPRINTF(sc, IWM_DEBUG_SCAN | IWM_DEBUG_TRACE,
4600             "%s: scan ended\n",
4601             __func__);
4602
4603         ieee80211_scan_done(TAILQ_FIRST(&ic->ic_vaps));
4604 }
4605
4606 /*
4607  * Aging and idle timeouts for the different possible scenarios
4608  * in default configuration
4609  */
4610 static const uint32_t
4611 iwm_sf_full_timeout_def[IWM_SF_NUM_SCENARIO][IWM_SF_NUM_TIMEOUT_TYPES] = {
4612         {
4613                 htole32(IWM_SF_SINGLE_UNICAST_AGING_TIMER_DEF),
4614                 htole32(IWM_SF_SINGLE_UNICAST_IDLE_TIMER_DEF)
4615         },
4616         {
4617                 htole32(IWM_SF_AGG_UNICAST_AGING_TIMER_DEF),
4618                 htole32(IWM_SF_AGG_UNICAST_IDLE_TIMER_DEF)
4619         },
4620         {
4621                 htole32(IWM_SF_MCAST_AGING_TIMER_DEF),
4622                 htole32(IWM_SF_MCAST_IDLE_TIMER_DEF)
4623         },
4624         {
4625                 htole32(IWM_SF_BA_AGING_TIMER_DEF),
4626                 htole32(IWM_SF_BA_IDLE_TIMER_DEF)
4627         },
4628         {
4629                 htole32(IWM_SF_TX_RE_AGING_TIMER_DEF),
4630                 htole32(IWM_SF_TX_RE_IDLE_TIMER_DEF)
4631         },
4632 };
4633
4634 /*
4635  * Aging and idle timeouts for the different possible scenarios
4636  * in single BSS MAC configuration.
4637  */
4638 static const uint32_t
4639 iwm_sf_full_timeout[IWM_SF_NUM_SCENARIO][IWM_SF_NUM_TIMEOUT_TYPES] = {
4640         {
4641                 htole32(IWM_SF_SINGLE_UNICAST_AGING_TIMER),
4642                 htole32(IWM_SF_SINGLE_UNICAST_IDLE_TIMER)
4643         },
4644         {
4645                 htole32(IWM_SF_AGG_UNICAST_AGING_TIMER),
4646                 htole32(IWM_SF_AGG_UNICAST_IDLE_TIMER)
4647         },
4648         {
4649                 htole32(IWM_SF_MCAST_AGING_TIMER),
4650                 htole32(IWM_SF_MCAST_IDLE_TIMER)
4651         },
4652         {
4653                 htole32(IWM_SF_BA_AGING_TIMER),
4654                 htole32(IWM_SF_BA_IDLE_TIMER)
4655         },
4656         {
4657                 htole32(IWM_SF_TX_RE_AGING_TIMER),
4658                 htole32(IWM_SF_TX_RE_IDLE_TIMER)
4659         },
4660 };
4661
4662 static void
4663 iwm_mvm_fill_sf_command(struct iwm_softc *sc, struct iwm_sf_cfg_cmd *sf_cmd,
4664     struct ieee80211_node *ni)
4665 {
4666         int i, j, watermark;
4667
4668         sf_cmd->watermark[IWM_SF_LONG_DELAY_ON] = htole32(IWM_SF_W_MARK_SCAN);
4669
4670         /*
4671          * If we are in association flow - check antenna configuration
4672          * capabilities of the AP station, and choose the watermark accordingly.
4673          */
4674         if (ni) {
4675                 if (ni->ni_flags & IEEE80211_NODE_HT) {
4676 #ifdef notyet
4677                         if (ni->ni_rxmcs[2] != 0)
4678                                 watermark = IWM_SF_W_MARK_MIMO3;
4679                         else if (ni->ni_rxmcs[1] != 0)
4680                                 watermark = IWM_SF_W_MARK_MIMO2;
4681                         else
4682 #endif
4683                                 watermark = IWM_SF_W_MARK_SISO;
4684                 } else {
4685                         watermark = IWM_SF_W_MARK_LEGACY;
4686                 }
4687         /* default watermark value for unassociated mode. */
4688         } else {
4689                 watermark = IWM_SF_W_MARK_MIMO2;
4690         }
4691         sf_cmd->watermark[IWM_SF_FULL_ON] = htole32(watermark);
4692
4693         for (i = 0; i < IWM_SF_NUM_SCENARIO; i++) {
4694                 for (j = 0; j < IWM_SF_NUM_TIMEOUT_TYPES; j++) {
4695                         sf_cmd->long_delay_timeouts[i][j] =
4696                                         htole32(IWM_SF_LONG_DELAY_AGING_TIMER);
4697                 }
4698         }
4699
4700         if (ni) {
4701                 memcpy(sf_cmd->full_on_timeouts, iwm_sf_full_timeout,
4702                        sizeof(iwm_sf_full_timeout));
4703         } else {
4704                 memcpy(sf_cmd->full_on_timeouts, iwm_sf_full_timeout_def,
4705                        sizeof(iwm_sf_full_timeout_def));
4706         }
4707 }
4708
4709 static int
4710 iwm_mvm_sf_config(struct iwm_softc *sc, enum iwm_sf_state new_state)
4711 {
4712         struct ieee80211com *ic = &sc->sc_ic;
4713         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
4714         struct iwm_sf_cfg_cmd sf_cmd = {
4715                 .state = htole32(IWM_SF_FULL_ON),
4716         };
4717         int ret = 0;
4718
4719         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000)
4720                 sf_cmd.state |= htole32(IWM_SF_CFG_DUMMY_NOTIF_OFF);
4721
4722         switch (new_state) {
4723         case IWM_SF_UNINIT:
4724         case IWM_SF_INIT_OFF:
4725                 iwm_mvm_fill_sf_command(sc, &sf_cmd, NULL);
4726                 break;
4727         case IWM_SF_FULL_ON:
4728                 iwm_mvm_fill_sf_command(sc, &sf_cmd, vap->iv_bss);
4729                 break;
4730         default:
4731                 IWM_DPRINTF(sc, IWM_DEBUG_PWRSAVE,
4732                     "Invalid state: %d. not sending Smart Fifo cmd\n",
4733                           new_state);
4734                 return EINVAL;
4735         }
4736
4737         ret = iwm_mvm_send_cmd_pdu(sc, IWM_REPLY_SF_CFG_CMD, IWM_CMD_ASYNC,
4738                                    sizeof(sf_cmd), &sf_cmd);
4739         return ret;
4740 }
4741
4742 static int
4743 iwm_send_bt_init_conf(struct iwm_softc *sc)
4744 {
4745         struct iwm_bt_coex_cmd bt_cmd;
4746
4747         bt_cmd.mode = htole32(IWM_BT_COEX_WIFI);
4748         bt_cmd.enabled_modules = htole32(IWM_BT_COEX_HIGH_BAND_RET);
4749
4750         return iwm_mvm_send_cmd_pdu(sc, IWM_BT_CONFIG, 0, sizeof(bt_cmd),
4751             &bt_cmd);
4752 }
4753
4754 static int
4755 iwm_send_update_mcc_cmd(struct iwm_softc *sc, const char *alpha2)
4756 {
4757         struct iwm_mcc_update_cmd mcc_cmd;
4758         struct iwm_host_cmd hcmd = {
4759                 .id = IWM_MCC_UPDATE_CMD,
4760                 .flags = (IWM_CMD_SYNC | IWM_CMD_WANT_SKB),
4761                 .data = { &mcc_cmd },
4762         };
4763         int ret;
4764 #ifdef IWM_DEBUG
4765         struct iwm_rx_packet *pkt;
4766         struct iwm_mcc_update_resp_v1 *mcc_resp_v1 = NULL;
4767         struct iwm_mcc_update_resp *mcc_resp;
4768         int n_channels;
4769         uint16_t mcc;
4770 #endif
4771         int resp_v2 = fw_has_capa(&sc->ucode_capa,
4772             IWM_UCODE_TLV_CAPA_LAR_SUPPORT_V2);
4773
4774         memset(&mcc_cmd, 0, sizeof(mcc_cmd));
4775         mcc_cmd.mcc = htole16(alpha2[0] << 8 | alpha2[1]);
4776         if (fw_has_api(&sc->ucode_capa, IWM_UCODE_TLV_API_WIFI_MCC_UPDATE) ||
4777             fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_LAR_MULTI_MCC))
4778                 mcc_cmd.source_id = IWM_MCC_SOURCE_GET_CURRENT;
4779         else
4780                 mcc_cmd.source_id = IWM_MCC_SOURCE_OLD_FW;
4781
4782         if (resp_v2)
4783                 hcmd.len[0] = sizeof(struct iwm_mcc_update_cmd);
4784         else
4785                 hcmd.len[0] = sizeof(struct iwm_mcc_update_cmd_v1);
4786
4787         IWM_DPRINTF(sc, IWM_DEBUG_NODE,
4788             "send MCC update to FW with '%c%c' src = %d\n",
4789             alpha2[0], alpha2[1], mcc_cmd.source_id);
4790
4791         ret = iwm_send_cmd(sc, &hcmd);
4792         if (ret)
4793                 return ret;
4794
4795 #ifdef IWM_DEBUG
4796         pkt = hcmd.resp_pkt;
4797
4798         /* Extract MCC response */
4799         if (resp_v2) {
4800                 mcc_resp = (void *)pkt->data;
4801                 mcc = mcc_resp->mcc;
4802                 n_channels =  le32toh(mcc_resp->n_channels);
4803         } else {
4804                 mcc_resp_v1 = (void *)pkt->data;
4805                 mcc = mcc_resp_v1->mcc;
4806                 n_channels =  le32toh(mcc_resp_v1->n_channels);
4807         }
4808
4809         /* W/A for a FW/NVM issue - returns 0x00 for the world domain */
4810         if (mcc == 0)
4811                 mcc = 0x3030;  /* "00" - world */
4812
4813         IWM_DPRINTF(sc, IWM_DEBUG_NODE,
4814             "regulatory domain '%c%c' (%d channels available)\n",
4815             mcc >> 8, mcc & 0xff, n_channels);
4816 #endif
4817         iwm_free_resp(sc, &hcmd);
4818
4819         return 0;
4820 }
4821
4822 static void
4823 iwm_mvm_tt_tx_backoff(struct iwm_softc *sc, uint32_t backoff)
4824 {
4825         struct iwm_host_cmd cmd = {
4826                 .id = IWM_REPLY_THERMAL_MNG_BACKOFF,
4827                 .len = { sizeof(uint32_t), },
4828                 .data = { &backoff, },
4829         };
4830
4831         if (iwm_send_cmd(sc, &cmd) != 0) {
4832                 device_printf(sc->sc_dev,
4833                     "failed to change thermal tx backoff\n");
4834         }
4835 }
4836
4837 static int
4838 iwm_init_hw(struct iwm_softc *sc)
4839 {
4840         struct ieee80211com *ic = &sc->sc_ic;
4841         int error, i, ac;
4842
4843         if ((error = iwm_start_hw(sc)) != 0) {
4844                 printf("iwm_start_hw: failed %d\n", error);
4845                 return error;
4846         }
4847
4848         if ((error = iwm_run_init_mvm_ucode(sc, 0)) != 0) {
4849                 printf("iwm_run_init_mvm_ucode: failed %d\n", error);
4850                 return error;
4851         }
4852
4853         /*
4854          * should stop and start HW since that INIT
4855          * image just loaded
4856          */
4857         iwm_stop_device(sc);
4858         if ((error = iwm_start_hw(sc)) != 0) {
4859                 device_printf(sc->sc_dev, "could not initialize hardware\n");
4860                 return error;
4861         }
4862
4863         /* omstart, this time with the regular firmware */
4864         error = iwm_mvm_load_ucode_wait_alive(sc, IWM_UCODE_REGULAR);
4865         if (error) {
4866                 device_printf(sc->sc_dev, "could not load firmware\n");
4867                 goto error;
4868         }
4869
4870         if ((error = iwm_send_bt_init_conf(sc)) != 0) {
4871                 device_printf(sc->sc_dev, "bt init conf failed\n");
4872                 goto error;
4873         }
4874
4875         error = iwm_send_tx_ant_cfg(sc, iwm_mvm_get_valid_tx_ant(sc));
4876         if (error != 0) {
4877                 device_printf(sc->sc_dev, "antenna config failed\n");
4878                 goto error;
4879         }
4880
4881         /* Send phy db control command and then phy db calibration */
4882         if ((error = iwm_send_phy_db_data(sc->sc_phy_db)) != 0)
4883                 goto error;
4884
4885         if ((error = iwm_send_phy_cfg_cmd(sc)) != 0) {
4886                 device_printf(sc->sc_dev, "phy_cfg_cmd failed\n");
4887                 goto error;
4888         }
4889
4890         /* Add auxiliary station for scanning */
4891         if ((error = iwm_mvm_add_aux_sta(sc)) != 0) {
4892                 device_printf(sc->sc_dev, "add_aux_sta failed\n");
4893                 goto error;
4894         }
4895
4896         for (i = 0; i < IWM_NUM_PHY_CTX; i++) {
4897                 /*
4898                  * The channel used here isn't relevant as it's
4899                  * going to be overwritten in the other flows.
4900                  * For now use the first channel we have.
4901                  */
4902                 if ((error = iwm_mvm_phy_ctxt_add(sc,
4903                     &sc->sc_phyctxt[i], &ic->ic_channels[1], 1, 1)) != 0)
4904                         goto error;
4905         }
4906
4907         /* Initialize tx backoffs to the minimum. */
4908         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
4909                 iwm_mvm_tt_tx_backoff(sc, 0);
4910
4911         error = iwm_mvm_power_update_device(sc);
4912         if (error)
4913                 goto error;
4914
4915         if (fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_LAR_SUPPORT)) {
4916                 if ((error = iwm_send_update_mcc_cmd(sc, "ZZ")) != 0)
4917                         goto error;
4918         }
4919
4920         if (fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_UMAC_SCAN)) {
4921                 if ((error = iwm_mvm_config_umac_scan(sc)) != 0)
4922                         goto error;
4923         }
4924
4925         /* Enable Tx queues. */
4926         for (ac = 0; ac < WME_NUM_AC; ac++) {
4927                 error = iwm_enable_txq(sc, IWM_STATION_ID, ac,
4928                     iwm_mvm_ac_to_tx_fifo[ac]);
4929                 if (error)
4930                         goto error;
4931         }
4932
4933         if ((error = iwm_mvm_disable_beacon_filter(sc)) != 0) {
4934                 device_printf(sc->sc_dev, "failed to disable beacon filter\n");
4935                 goto error;
4936         }
4937
4938         return 0;
4939
4940  error:
4941         iwm_stop_device(sc);
4942         return error;
4943 }
4944
4945 /* Allow multicast from our BSSID. */
4946 static int
4947 iwm_allow_mcast(struct ieee80211vap *vap, struct iwm_softc *sc)
4948 {
4949         struct ieee80211_node *ni = vap->iv_bss;
4950         struct iwm_mcast_filter_cmd *cmd;
4951         size_t size;
4952         int error;
4953
4954         size = roundup(sizeof(*cmd), 4);
4955         cmd = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
4956         if (cmd == NULL)
4957                 return ENOMEM;
4958         cmd->filter_own = 1;
4959         cmd->port_id = 0;
4960         cmd->count = 0;
4961         cmd->pass_all = 1;
4962         IEEE80211_ADDR_COPY(cmd->bssid, ni->ni_bssid);
4963
4964         error = iwm_mvm_send_cmd_pdu(sc, IWM_MCAST_FILTER_CMD,
4965             IWM_CMD_SYNC, size, cmd);
4966         free(cmd, M_DEVBUF);
4967
4968         return (error);
4969 }
4970
4971 /*
4972  * ifnet interfaces
4973  */
4974
4975 static void
4976 iwm_init(struct iwm_softc *sc)
4977 {
4978         int error;
4979
4980         if (sc->sc_flags & IWM_FLAG_HW_INITED) {
4981                 return;
4982         }
4983         sc->sc_generation++;
4984         sc->sc_flags &= ~IWM_FLAG_STOPPED;
4985
4986         if ((error = iwm_init_hw(sc)) != 0) {
4987                 printf("iwm_init_hw failed %d\n", error);
4988                 iwm_stop(sc);
4989                 return;
4990         }
4991
4992         /*
4993          * Ok, firmware loaded and we are jogging
4994          */
4995         sc->sc_flags |= IWM_FLAG_HW_INITED;
4996         callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog, sc);
4997 }
4998
4999 static int
5000 iwm_transmit(struct ieee80211com *ic, struct mbuf *m)
5001 {
5002         struct iwm_softc *sc;
5003         int error;
5004
5005         sc = ic->ic_softc;
5006
5007         IWM_LOCK(sc);
5008         if ((sc->sc_flags & IWM_FLAG_HW_INITED) == 0) {
5009                 IWM_UNLOCK(sc);
5010                 return (ENXIO);
5011         }
5012         error = mbufq_enqueue(&sc->sc_snd, m);
5013         if (error) {
5014                 IWM_UNLOCK(sc);
5015                 return (error);
5016         }
5017         iwm_start(sc);
5018         IWM_UNLOCK(sc);
5019         return (0);
5020 }
5021
5022 /*
5023  * Dequeue packets from sendq and call send.
5024  */
5025 static void
5026 iwm_start(struct iwm_softc *sc)
5027 {
5028         struct ieee80211_node *ni;
5029         struct mbuf *m;
5030         int ac = 0;
5031
5032         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TRACE, "->%s\n", __func__);
5033         while (sc->qfullmsk == 0 &&
5034                 (m = mbufq_dequeue(&sc->sc_snd)) != NULL) {
5035                 ni = (struct ieee80211_node *)m->m_pkthdr.rcvif;
5036                 if (iwm_tx(sc, m, ni, ac) != 0) {
5037                         if_inc_counter(ni->ni_vap->iv_ifp,
5038                             IFCOUNTER_OERRORS, 1);
5039                         ieee80211_free_node(ni);
5040                         continue;
5041                 }
5042                 sc->sc_tx_timer = 15;
5043         }
5044         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TRACE, "<-%s\n", __func__);
5045 }
5046
5047 static void
5048 iwm_stop(struct iwm_softc *sc)
5049 {
5050
5051         sc->sc_flags &= ~IWM_FLAG_HW_INITED;
5052         sc->sc_flags |= IWM_FLAG_STOPPED;
5053         sc->sc_generation++;
5054         iwm_led_blink_stop(sc);
5055         sc->sc_tx_timer = 0;
5056         iwm_stop_device(sc);
5057         sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
5058 }
5059
5060 static void
5061 iwm_watchdog(void *arg)
5062 {
5063         struct iwm_softc *sc = arg;
5064         struct ieee80211com *ic = &sc->sc_ic;
5065
5066         if (sc->sc_tx_timer > 0) {
5067                 if (--sc->sc_tx_timer == 0) {
5068                         device_printf(sc->sc_dev, "device timeout\n");
5069 #ifdef IWM_DEBUG
5070                         iwm_nic_error(sc);
5071 #endif
5072                         ieee80211_restart_all(ic);
5073                         counter_u64_add(sc->sc_ic.ic_oerrors, 1);
5074                         return;
5075                 }
5076         }
5077         callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog, sc);
5078 }
5079
5080 static void
5081 iwm_parent(struct ieee80211com *ic)
5082 {
5083         struct iwm_softc *sc = ic->ic_softc;
5084         int startall = 0;
5085
5086         IWM_LOCK(sc);
5087         if (ic->ic_nrunning > 0) {
5088                 if (!(sc->sc_flags & IWM_FLAG_HW_INITED)) {
5089                         iwm_init(sc);
5090                         startall = 1;
5091                 }
5092         } else if (sc->sc_flags & IWM_FLAG_HW_INITED)
5093                 iwm_stop(sc);
5094         IWM_UNLOCK(sc);
5095         if (startall)
5096                 ieee80211_start_all(ic);
5097 }
5098
5099 /*
5100  * The interrupt side of things
5101  */
5102
5103 /*
5104  * error dumping routines are from iwlwifi/mvm/utils.c
5105  */
5106
5107 /*
5108  * Note: This structure is read from the device with IO accesses,
5109  * and the reading already does the endian conversion. As it is
5110  * read with uint32_t-sized accesses, any members with a different size
5111  * need to be ordered correctly though!
5112  */
5113 struct iwm_error_event_table {
5114         uint32_t valid;         /* (nonzero) valid, (0) log is empty */
5115         uint32_t error_id;              /* type of error */
5116         uint32_t trm_hw_status0;        /* TRM HW status */
5117         uint32_t trm_hw_status1;        /* TRM HW status */
5118         uint32_t blink2;                /* branch link */
5119         uint32_t ilink1;                /* interrupt link */
5120         uint32_t ilink2;                /* interrupt link */
5121         uint32_t data1;         /* error-specific data */
5122         uint32_t data2;         /* error-specific data */
5123         uint32_t data3;         /* error-specific data */
5124         uint32_t bcon_time;             /* beacon timer */
5125         uint32_t tsf_low;               /* network timestamp function timer */
5126         uint32_t tsf_hi;                /* network timestamp function timer */
5127         uint32_t gp1;           /* GP1 timer register */
5128         uint32_t gp2;           /* GP2 timer register */
5129         uint32_t fw_rev_type;   /* firmware revision type */
5130         uint32_t major;         /* uCode version major */
5131         uint32_t minor;         /* uCode version minor */
5132         uint32_t hw_ver;                /* HW Silicon version */
5133         uint32_t brd_ver;               /* HW board version */
5134         uint32_t log_pc;                /* log program counter */
5135         uint32_t frame_ptr;             /* frame pointer */
5136         uint32_t stack_ptr;             /* stack pointer */
5137         uint32_t hcmd;          /* last host command header */
5138         uint32_t isr0;          /* isr status register LMPM_NIC_ISR0:
5139                                  * rxtx_flag */
5140         uint32_t isr1;          /* isr status register LMPM_NIC_ISR1:
5141                                  * host_flag */
5142         uint32_t isr2;          /* isr status register LMPM_NIC_ISR2:
5143                                  * enc_flag */
5144         uint32_t isr3;          /* isr status register LMPM_NIC_ISR3:
5145                                  * time_flag */
5146         uint32_t isr4;          /* isr status register LMPM_NIC_ISR4:
5147                                  * wico interrupt */
5148         uint32_t last_cmd_id;   /* last HCMD id handled by the firmware */
5149         uint32_t wait_event;            /* wait event() caller address */
5150         uint32_t l2p_control;   /* L2pControlField */
5151         uint32_t l2p_duration;  /* L2pDurationField */
5152         uint32_t l2p_mhvalid;   /* L2pMhValidBits */
5153         uint32_t l2p_addr_match;        /* L2pAddrMatchStat */
5154         uint32_t lmpm_pmg_sel;  /* indicate which clocks are turned on
5155                                  * (LMPM_PMG_SEL) */
5156         uint32_t u_timestamp;   /* indicate when the date and time of the
5157                                  * compilation */
5158         uint32_t flow_handler;  /* FH read/write pointers, RX credit */
5159 } __packed /* LOG_ERROR_TABLE_API_S_VER_3 */;
5160
5161 /*
5162  * UMAC error struct - relevant starting from family 8000 chip.
5163  * Note: This structure is read from the device with IO accesses,
5164  * and the reading already does the endian conversion. As it is
5165  * read with u32-sized accesses, any members with a different size
5166  * need to be ordered correctly though!
5167  */
5168 struct iwm_umac_error_event_table {
5169         uint32_t valid;         /* (nonzero) valid, (0) log is empty */
5170         uint32_t error_id;      /* type of error */
5171         uint32_t blink1;        /* branch link */
5172         uint32_t blink2;        /* branch link */
5173         uint32_t ilink1;        /* interrupt link */
5174         uint32_t ilink2;        /* interrupt link */
5175         uint32_t data1;         /* error-specific data */
5176         uint32_t data2;         /* error-specific data */
5177         uint32_t data3;         /* error-specific data */
5178         uint32_t umac_major;
5179         uint32_t umac_minor;
5180         uint32_t frame_pointer; /* core register 27*/
5181         uint32_t stack_pointer; /* core register 28 */
5182         uint32_t cmd_header;    /* latest host cmd sent to UMAC */
5183         uint32_t nic_isr_pref;  /* ISR status register */
5184 } __packed;
5185
5186 #define ERROR_START_OFFSET  (1 * sizeof(uint32_t))
5187 #define ERROR_ELEM_SIZE     (7 * sizeof(uint32_t))
5188
5189 #ifdef IWM_DEBUG
5190 struct {
5191         const char *name;
5192         uint8_t num;
5193 } advanced_lookup[] = {
5194         { "NMI_INTERRUPT_WDG", 0x34 },
5195         { "SYSASSERT", 0x35 },
5196         { "UCODE_VERSION_MISMATCH", 0x37 },
5197         { "BAD_COMMAND", 0x38 },
5198         { "NMI_INTERRUPT_DATA_ACTION_PT", 0x3C },
5199         { "FATAL_ERROR", 0x3D },
5200         { "NMI_TRM_HW_ERR", 0x46 },
5201         { "NMI_INTERRUPT_TRM", 0x4C },
5202         { "NMI_INTERRUPT_BREAK_POINT", 0x54 },
5203         { "NMI_INTERRUPT_WDG_RXF_FULL", 0x5C },
5204         { "NMI_INTERRUPT_WDG_NO_RBD_RXF_FULL", 0x64 },
5205         { "NMI_INTERRUPT_HOST", 0x66 },
5206         { "NMI_INTERRUPT_ACTION_PT", 0x7C },
5207         { "NMI_INTERRUPT_UNKNOWN", 0x84 },
5208         { "NMI_INTERRUPT_INST_ACTION_PT", 0x86 },
5209         { "ADVANCED_SYSASSERT", 0 },
5210 };
5211
5212 static const char *
5213 iwm_desc_lookup(uint32_t num)
5214 {
5215         int i;
5216
5217         for (i = 0; i < nitems(advanced_lookup) - 1; i++)
5218                 if (advanced_lookup[i].num == num)
5219                         return advanced_lookup[i].name;
5220
5221         /* No entry matches 'num', so it is the last: ADVANCED_SYSASSERT */
5222         return advanced_lookup[i].name;
5223 }
5224
5225 static void
5226 iwm_nic_umac_error(struct iwm_softc *sc)
5227 {
5228         struct iwm_umac_error_event_table table;
5229         uint32_t base;
5230
5231         base = sc->umac_error_event_table;
5232
5233         if (base < 0x800000) {
5234                 device_printf(sc->sc_dev, "Invalid error log pointer 0x%08x\n",
5235                     base);
5236                 return;
5237         }
5238
5239         if (iwm_read_mem(sc, base, &table, sizeof(table)/sizeof(uint32_t))) {
5240                 device_printf(sc->sc_dev, "reading errlog failed\n");
5241                 return;
5242         }
5243
5244         if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) {
5245                 device_printf(sc->sc_dev, "Start UMAC Error Log Dump:\n");
5246                 device_printf(sc->sc_dev, "Status: 0x%x, count: %d\n",
5247                     sc->sc_flags, table.valid);
5248         }
5249
5250         device_printf(sc->sc_dev, "0x%08X | %s\n", table.error_id,
5251                 iwm_desc_lookup(table.error_id));
5252         device_printf(sc->sc_dev, "0x%08X | umac branchlink1\n", table.blink1);
5253         device_printf(sc->sc_dev, "0x%08X | umac branchlink2\n", table.blink2);
5254         device_printf(sc->sc_dev, "0x%08X | umac interruptlink1\n",
5255             table.ilink1);
5256         device_printf(sc->sc_dev, "0x%08X | umac interruptlink2\n",
5257             table.ilink2);
5258         device_printf(sc->sc_dev, "0x%08X | umac data1\n", table.data1);
5259         device_printf(sc->sc_dev, "0x%08X | umac data2\n", table.data2);
5260         device_printf(sc->sc_dev, "0x%08X | umac data3\n", table.data3);
5261         device_printf(sc->sc_dev, "0x%08X | umac major\n", table.umac_major);
5262         device_printf(sc->sc_dev, "0x%08X | umac minor\n", table.umac_minor);
5263         device_printf(sc->sc_dev, "0x%08X | frame pointer\n",
5264             table.frame_pointer);
5265         device_printf(sc->sc_dev, "0x%08X | stack pointer\n",
5266             table.stack_pointer);
5267         device_printf(sc->sc_dev, "0x%08X | last host cmd\n", table.cmd_header);
5268         device_printf(sc->sc_dev, "0x%08X | isr status reg\n",
5269             table.nic_isr_pref);
5270 }
5271
5272 /*
5273  * Support for dumping the error log seemed like a good idea ...
5274  * but it's mostly hex junk and the only sensible thing is the
5275  * hw/ucode revision (which we know anyway).  Since it's here,
5276  * I'll just leave it in, just in case e.g. the Intel guys want to
5277  * help us decipher some "ADVANCED_SYSASSERT" later.
5278  */
5279 static void
5280 iwm_nic_error(struct iwm_softc *sc)
5281 {
5282         struct iwm_error_event_table table;
5283         uint32_t base;
5284
5285         device_printf(sc->sc_dev, "dumping device error log\n");
5286         base = sc->error_event_table;
5287         if (base < 0x800000) {
5288                 device_printf(sc->sc_dev,
5289                     "Invalid error log pointer 0x%08x\n", base);
5290                 return;
5291         }
5292
5293         if (iwm_read_mem(sc, base, &table, sizeof(table)/sizeof(uint32_t))) {
5294                 device_printf(sc->sc_dev, "reading errlog failed\n");
5295                 return;
5296         }
5297
5298         if (!table.valid) {
5299                 device_printf(sc->sc_dev, "errlog not found, skipping\n");
5300                 return;
5301         }
5302
5303         if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) {
5304                 device_printf(sc->sc_dev, "Start Error Log Dump:\n");
5305                 device_printf(sc->sc_dev, "Status: 0x%x, count: %d\n",
5306                     sc->sc_flags, table.valid);
5307         }
5308
5309         device_printf(sc->sc_dev, "0x%08X | %-28s\n", table.error_id,
5310             iwm_desc_lookup(table.error_id));
5311         device_printf(sc->sc_dev, "%08X | trm_hw_status0\n",
5312             table.trm_hw_status0);
5313         device_printf(sc->sc_dev, "%08X | trm_hw_status1\n",
5314             table.trm_hw_status1);
5315         device_printf(sc->sc_dev, "%08X | branchlink2\n", table.blink2);
5316         device_printf(sc->sc_dev, "%08X | interruptlink1\n", table.ilink1);
5317         device_printf(sc->sc_dev, "%08X | interruptlink2\n", table.ilink2);
5318         device_printf(sc->sc_dev, "%08X | data1\n", table.data1);
5319         device_printf(sc->sc_dev, "%08X | data2\n", table.data2);
5320         device_printf(sc->sc_dev, "%08X | data3\n", table.data3);
5321         device_printf(sc->sc_dev, "%08X | beacon time\n", table.bcon_time);
5322         device_printf(sc->sc_dev, "%08X | tsf low\n", table.tsf_low);
5323         device_printf(sc->sc_dev, "%08X | tsf hi\n", table.tsf_hi);
5324         device_printf(sc->sc_dev, "%08X | time gp1\n", table.gp1);
5325         device_printf(sc->sc_dev, "%08X | time gp2\n", table.gp2);
5326         device_printf(sc->sc_dev, "%08X | uCode revision type\n",
5327             table.fw_rev_type);
5328         device_printf(sc->sc_dev, "%08X | uCode version major\n", table.major);
5329         device_printf(sc->sc_dev, "%08X | uCode version minor\n", table.minor);
5330         device_printf(sc->sc_dev, "%08X | hw version\n", table.hw_ver);
5331         device_printf(sc->sc_dev, "%08X | board version\n", table.brd_ver);
5332         device_printf(sc->sc_dev, "%08X | hcmd\n", table.hcmd);
5333         device_printf(sc->sc_dev, "%08X | isr0\n", table.isr0);
5334         device_printf(sc->sc_dev, "%08X | isr1\n", table.isr1);
5335         device_printf(sc->sc_dev, "%08X | isr2\n", table.isr2);
5336         device_printf(sc->sc_dev, "%08X | isr3\n", table.isr3);
5337         device_printf(sc->sc_dev, "%08X | isr4\n", table.isr4);
5338         device_printf(sc->sc_dev, "%08X | last cmd Id\n", table.last_cmd_id);
5339         device_printf(sc->sc_dev, "%08X | wait_event\n", table.wait_event);
5340         device_printf(sc->sc_dev, "%08X | l2p_control\n", table.l2p_control);
5341         device_printf(sc->sc_dev, "%08X | l2p_duration\n", table.l2p_duration);
5342         device_printf(sc->sc_dev, "%08X | l2p_mhvalid\n", table.l2p_mhvalid);
5343         device_printf(sc->sc_dev, "%08X | l2p_addr_match\n", table.l2p_addr_match);
5344         device_printf(sc->sc_dev, "%08X | lmpm_pmg_sel\n", table.lmpm_pmg_sel);
5345         device_printf(sc->sc_dev, "%08X | timestamp\n", table.u_timestamp);
5346         device_printf(sc->sc_dev, "%08X | flow_handler\n", table.flow_handler);
5347
5348         if (sc->umac_error_event_table)
5349                 iwm_nic_umac_error(sc);
5350 }
5351 #endif
5352
5353 #define ADVANCE_RXQ(sc) (sc->rxq.cur = (sc->rxq.cur + 1) % IWM_RX_RING_COUNT);
5354
5355 /*
5356  * Process an IWM_CSR_INT_BIT_FH_RX or IWM_CSR_INT_BIT_SW_RX interrupt.
5357  * Basic structure from if_iwn
5358  */
5359 static void
5360 iwm_notif_intr(struct iwm_softc *sc)
5361 {
5362         struct ieee80211com *ic = &sc->sc_ic;
5363         uint16_t hw;
5364
5365         bus_dmamap_sync(sc->rxq.stat_dma.tag, sc->rxq.stat_dma.map,
5366             BUS_DMASYNC_POSTREAD);
5367
5368         hw = le16toh(sc->rxq.stat->closed_rb_num) & 0xfff;
5369
5370         /*
5371          * Process responses
5372          */
5373         while (sc->rxq.cur != hw) {
5374                 struct iwm_rx_ring *ring = &sc->rxq;
5375                 struct iwm_rx_data *data = &ring->data[ring->cur];
5376                 struct iwm_rx_packet *pkt;
5377                 struct iwm_cmd_response *cresp;
5378                 int qid, idx, code;
5379
5380                 bus_dmamap_sync(ring->data_dmat, data->map,
5381                     BUS_DMASYNC_POSTREAD);
5382                 pkt = mtod(data->m, struct iwm_rx_packet *);
5383
5384                 qid = pkt->hdr.qid & ~0x80;
5385                 idx = pkt->hdr.idx;
5386
5387                 code = IWM_WIDE_ID(pkt->hdr.flags, pkt->hdr.code);
5388                 IWM_DPRINTF(sc, IWM_DEBUG_INTR,
5389                     "rx packet qid=%d idx=%d type=%x %d %d\n",
5390                     pkt->hdr.qid & ~0x80, pkt->hdr.idx, code, ring->cur, hw);
5391
5392                 /*
5393                  * randomly get these from the firmware, no idea why.
5394                  * they at least seem harmless, so just ignore them for now
5395                  */
5396                 if (__predict_false((pkt->hdr.code == 0 && qid == 0 && idx == 0)
5397                     || pkt->len_n_flags == htole32(0x55550000))) {
5398                         ADVANCE_RXQ(sc);
5399                         continue;
5400                 }
5401
5402                 iwm_notification_wait_notify(sc->sc_notif_wait, code, pkt);
5403
5404                 switch (code) {
5405                 case IWM_REPLY_RX_PHY_CMD:
5406                         iwm_mvm_rx_rx_phy_cmd(sc, pkt, data);
5407                         break;
5408
5409                 case IWM_REPLY_RX_MPDU_CMD:
5410                         iwm_mvm_rx_rx_mpdu(sc, data->m);
5411                         break;
5412
5413                 case IWM_TX_CMD:
5414                         iwm_mvm_rx_tx_cmd(sc, pkt, data);
5415                         break;
5416
5417                 case IWM_MISSED_BEACONS_NOTIFICATION: {
5418                         struct iwm_missed_beacons_notif *resp;
5419                         int missed;
5420
5421                         /* XXX look at mac_id to determine interface ID */
5422                         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
5423
5424                         resp = (void *)pkt->data;
5425                         missed = le32toh(resp->consec_missed_beacons);
5426
5427                         IWM_DPRINTF(sc, IWM_DEBUG_BEACON | IWM_DEBUG_STATE,
5428                             "%s: MISSED_BEACON: mac_id=%d, "
5429                             "consec_since_last_rx=%d, consec=%d, num_expect=%d "
5430                             "num_rx=%d\n",
5431                             __func__,
5432                             le32toh(resp->mac_id),
5433                             le32toh(resp->consec_missed_beacons_since_last_rx),
5434                             le32toh(resp->consec_missed_beacons),
5435                             le32toh(resp->num_expected_beacons),
5436                             le32toh(resp->num_recvd_beacons));
5437
5438                         /* Be paranoid */
5439                         if (vap == NULL)
5440                                 break;
5441
5442                         /* XXX no net80211 locking? */
5443                         if (vap->iv_state == IEEE80211_S_RUN &&
5444                             (ic->ic_flags & IEEE80211_F_SCAN) == 0) {
5445                                 if (missed > vap->iv_bmissthreshold) {
5446                                         /* XXX bad locking; turn into task */
5447                                         IWM_UNLOCK(sc);
5448                                         ieee80211_beacon_miss(ic);
5449                                         IWM_LOCK(sc);
5450                                 }
5451                         }
5452
5453                         break;
5454                 }
5455
5456                 case IWM_MFUART_LOAD_NOTIFICATION:
5457                         break;
5458
5459                 case IWM_MVM_ALIVE:
5460                         break;
5461
5462                 case IWM_CALIB_RES_NOTIF_PHY_DB:
5463                         break;
5464
5465                 case IWM_STATISTICS_NOTIFICATION: {
5466                         struct iwm_notif_statistics *stats;
5467                         stats = (void *)pkt->data;
5468                         memcpy(&sc->sc_stats, stats, sizeof(sc->sc_stats));
5469                         sc->sc_noise = iwm_get_noise(sc, &stats->rx.general);
5470                         break;
5471                 }
5472
5473                 case IWM_NVM_ACCESS_CMD:
5474                 case IWM_MCC_UPDATE_CMD:
5475                         if (sc->sc_wantresp == ((qid << 16) | idx)) {
5476                                 memcpy(sc->sc_cmd_resp,
5477                                     pkt, sizeof(sc->sc_cmd_resp));
5478                         }
5479                         break;
5480
5481                 case IWM_MCC_CHUB_UPDATE_CMD: {
5482                         struct iwm_mcc_chub_notif *notif;
5483                         notif = (void *)pkt->data;
5484
5485                         sc->sc_fw_mcc[0] = (notif->mcc & 0xff00) >> 8;
5486                         sc->sc_fw_mcc[1] = notif->mcc & 0xff;
5487                         sc->sc_fw_mcc[2] = '\0';
5488                         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
5489                             "fw source %d sent CC '%s'\n",
5490                             notif->source_id, sc->sc_fw_mcc);
5491                         break;
5492                 }
5493
5494                 case IWM_DTS_MEASUREMENT_NOTIFICATION:
5495                 case IWM_WIDE_ID(IWM_PHY_OPS_GROUP,
5496                                  IWM_DTS_MEASUREMENT_NOTIF_WIDE): {
5497                         struct iwm_dts_measurement_notif_v1 *notif;
5498
5499                         if (iwm_rx_packet_payload_len(pkt) < sizeof(*notif)) {
5500                                 device_printf(sc->sc_dev,
5501                                     "Invalid DTS_MEASUREMENT_NOTIFICATION\n");
5502                                 break;
5503                         }
5504                         notif = (void *)pkt->data;
5505                         IWM_DPRINTF(sc, IWM_DEBUG_TEMP,
5506                             "IWM_DTS_MEASUREMENT_NOTIFICATION - %d\n",
5507                             notif->temp);
5508                         break;
5509                 }
5510
5511                 case IWM_PHY_CONFIGURATION_CMD:
5512                 case IWM_TX_ANT_CONFIGURATION_CMD:
5513                 case IWM_ADD_STA:
5514                 case IWM_MAC_CONTEXT_CMD:
5515                 case IWM_REPLY_SF_CFG_CMD:
5516                 case IWM_POWER_TABLE_CMD:
5517                 case IWM_PHY_CONTEXT_CMD:
5518                 case IWM_BINDING_CONTEXT_CMD:
5519                 case IWM_TIME_EVENT_CMD:
5520                 case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_CFG_CMD):
5521                 case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_REQ_UMAC):
5522                 case IWM_SCAN_ABORT_UMAC:
5523                 case IWM_SCAN_OFFLOAD_REQUEST_CMD:
5524                 case IWM_SCAN_OFFLOAD_ABORT_CMD:
5525                 case IWM_REPLY_BEACON_FILTERING_CMD:
5526                 case IWM_MAC_PM_POWER_TABLE:
5527                 case IWM_TIME_QUOTA_CMD:
5528                 case IWM_REMOVE_STA:
5529                 case IWM_TXPATH_FLUSH:
5530                 case IWM_LQ_CMD:
5531                 case IWM_FW_PAGING_BLOCK_CMD:
5532                 case IWM_BT_CONFIG:
5533                 case IWM_REPLY_THERMAL_MNG_BACKOFF:
5534                         cresp = (void *)pkt->data;
5535                         if (sc->sc_wantresp == ((qid << 16) | idx)) {
5536                                 memcpy(sc->sc_cmd_resp,
5537                                     pkt, sizeof(*pkt)+sizeof(*cresp));
5538                         }
5539                         break;
5540
5541                 /* ignore */
5542                 case 0x6c: /* IWM_PHY_DB_CMD, no idea why it's not in fw-api.h */
5543                         break;
5544
5545                 case IWM_INIT_COMPLETE_NOTIF:
5546                         break;
5547
5548                 case IWM_SCAN_OFFLOAD_COMPLETE:
5549                         iwm_mvm_rx_lmac_scan_complete_notif(sc, pkt);
5550                         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
5551                                 sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
5552                                 ieee80211_runtask(ic, &sc->sc_es_task);
5553                         }
5554                         break;
5555
5556                 case IWM_SCAN_ITERATION_COMPLETE: {
5557                         struct iwm_lmac_scan_complete_notif *notif;
5558                         notif = (void *)pkt->data;
5559                         break;
5560                 }
5561
5562                 case IWM_SCAN_COMPLETE_UMAC:
5563                         iwm_mvm_rx_umac_scan_complete_notif(sc, pkt);
5564                         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
5565                                 sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
5566                                 ieee80211_runtask(ic, &sc->sc_es_task);
5567                         }
5568                         break;
5569
5570                 case IWM_SCAN_ITERATION_COMPLETE_UMAC: {
5571                         struct iwm_umac_scan_iter_complete_notif *notif;
5572                         notif = (void *)pkt->data;
5573
5574                         IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "UMAC scan iteration "
5575                             "complete, status=0x%x, %d channels scanned\n",
5576                             notif->status, notif->scanned_channels);
5577                         break;
5578                 }
5579
5580                 case IWM_REPLY_ERROR: {
5581                         struct iwm_error_resp *resp;
5582                         resp = (void *)pkt->data;
5583
5584                         device_printf(sc->sc_dev,
5585                             "firmware error 0x%x, cmd 0x%x\n",
5586                             le32toh(resp->error_type),
5587                             resp->cmd_id);
5588                         break;
5589                 }
5590
5591                 case IWM_TIME_EVENT_NOTIFICATION: {
5592                         struct iwm_time_event_notif *notif;
5593                         notif = (void *)pkt->data;
5594
5595                         IWM_DPRINTF(sc, IWM_DEBUG_INTR,
5596                             "TE notif status = 0x%x action = 0x%x\n",
5597                             notif->status, notif->action);
5598                         break;
5599                 }
5600
5601                 case IWM_MCAST_FILTER_CMD:
5602                         break;
5603
5604                 case IWM_SCD_QUEUE_CFG: {
5605                         struct iwm_scd_txq_cfg_rsp *rsp;
5606                         rsp = (void *)pkt->data;
5607
5608                         IWM_DPRINTF(sc, IWM_DEBUG_CMD,
5609                             "queue cfg token=0x%x sta_id=%d "
5610                             "tid=%d scd_queue=%d\n",
5611                             rsp->token, rsp->sta_id, rsp->tid,
5612                             rsp->scd_queue);
5613                         break;
5614                 }
5615
5616                 default:
5617                         device_printf(sc->sc_dev,
5618                             "frame %d/%d %x UNHANDLED (this should "
5619                             "not happen)\n", qid, idx,
5620                             pkt->len_n_flags);
5621                         break;
5622                 }
5623
5624                 /*
5625                  * Why test bit 0x80?  The Linux driver:
5626                  *
5627                  * There is one exception:  uCode sets bit 15 when it
5628                  * originates the response/notification, i.e. when the
5629                  * response/notification is not a direct response to a
5630                  * command sent by the driver.  For example, uCode issues
5631                  * IWM_REPLY_RX when it sends a received frame to the driver;
5632                  * it is not a direct response to any driver command.
5633                  *
5634                  * Ok, so since when is 7 == 15?  Well, the Linux driver
5635                  * uses a slightly different format for pkt->hdr, and "qid"
5636                  * is actually the upper byte of a two-byte field.
5637                  */
5638                 if (!(pkt->hdr.qid & (1 << 7))) {
5639                         iwm_cmd_done(sc, pkt);
5640                 }
5641
5642                 ADVANCE_RXQ(sc);
5643         }
5644
5645         /*
5646          * Tell the firmware what we have processed.
5647          * Seems like the hardware gets upset unless we align
5648          * the write by 8??
5649          */
5650         hw = (hw == 0) ? IWM_RX_RING_COUNT - 1 : hw - 1;
5651         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_WPTR, hw & ~7);
5652 }
5653
5654 static void
5655 iwm_intr(void *arg)
5656 {
5657         struct iwm_softc *sc = arg;
5658         int handled = 0;
5659         int r1, r2, rv = 0;
5660         int isperiodic = 0;
5661
5662         IWM_LOCK(sc);
5663         IWM_WRITE(sc, IWM_CSR_INT_MASK, 0);
5664
5665         if (sc->sc_flags & IWM_FLAG_USE_ICT) {
5666                 uint32_t *ict = sc->ict_dma.vaddr;
5667                 int tmp;
5668
5669                 tmp = htole32(ict[sc->ict_cur]);
5670                 if (!tmp)
5671                         goto out_ena;
5672
5673                 /*
5674                  * ok, there was something.  keep plowing until we have all.
5675                  */
5676                 r1 = r2 = 0;
5677                 while (tmp) {
5678                         r1 |= tmp;
5679                         ict[sc->ict_cur] = 0;
5680                         sc->ict_cur = (sc->ict_cur+1) % IWM_ICT_COUNT;
5681                         tmp = htole32(ict[sc->ict_cur]);
5682                 }
5683
5684                 /* this is where the fun begins.  don't ask */
5685                 if (r1 == 0xffffffff)
5686                         r1 = 0;
5687
5688                 /* i am not expected to understand this */
5689                 if (r1 & 0xc0000)
5690                         r1 |= 0x8000;
5691                 r1 = (0xff & r1) | ((0xff00 & r1) << 16);
5692         } else {
5693                 r1 = IWM_READ(sc, IWM_CSR_INT);
5694                 /* "hardware gone" (where, fishing?) */
5695                 if (r1 == 0xffffffff || (r1 & 0xfffffff0) == 0xa5a5a5a0)
5696                         goto out;
5697                 r2 = IWM_READ(sc, IWM_CSR_FH_INT_STATUS);
5698         }
5699         if (r1 == 0 && r2 == 0) {
5700                 goto out_ena;
5701         }
5702
5703         IWM_WRITE(sc, IWM_CSR_INT, r1 | ~sc->sc_intmask);
5704
5705         /* Safely ignore these bits for debug checks below */
5706         r1 &= ~(IWM_CSR_INT_BIT_ALIVE | IWM_CSR_INT_BIT_SCD);
5707
5708         if (r1 & IWM_CSR_INT_BIT_SW_ERR) {
5709                 int i;
5710                 struct ieee80211com *ic = &sc->sc_ic;
5711                 struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
5712
5713 #ifdef IWM_DEBUG
5714                 iwm_nic_error(sc);
5715 #endif
5716                 /* Dump driver status (TX and RX rings) while we're here. */
5717                 device_printf(sc->sc_dev, "driver status:\n");
5718                 for (i = 0; i < IWM_MVM_MAX_QUEUES; i++) {
5719                         struct iwm_tx_ring *ring = &sc->txq[i];
5720                         device_printf(sc->sc_dev,
5721                             "  tx ring %2d: qid=%-2d cur=%-3d "
5722                             "queued=%-3d\n",
5723                             i, ring->qid, ring->cur, ring->queued);
5724                 }
5725                 device_printf(sc->sc_dev,
5726                     "  rx ring: cur=%d\n", sc->rxq.cur);
5727                 device_printf(sc->sc_dev,
5728                     "  802.11 state %d\n", (vap == NULL) ? -1 : vap->iv_state);
5729
5730                 /* Don't stop the device; just do a VAP restart */
5731                 IWM_UNLOCK(sc);
5732
5733                 if (vap == NULL) {
5734                         printf("%s: null vap\n", __func__);
5735                         return;
5736                 }
5737
5738                 device_printf(sc->sc_dev, "%s: controller panicked, iv_state = %d; "
5739                     "restarting\n", __func__, vap->iv_state);
5740
5741                 /* XXX TODO: turn this into a callout/taskqueue */
5742                 ieee80211_restart_all(ic);
5743                 return;
5744         }
5745
5746         if (r1 & IWM_CSR_INT_BIT_HW_ERR) {
5747                 handled |= IWM_CSR_INT_BIT_HW_ERR;
5748                 device_printf(sc->sc_dev, "hardware error, stopping device\n");
5749                 iwm_stop(sc);
5750                 rv = 1;
5751                 goto out;
5752         }
5753
5754         /* firmware chunk loaded */
5755         if (r1 & IWM_CSR_INT_BIT_FH_TX) {
5756                 IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, IWM_CSR_FH_INT_TX_MASK);
5757                 handled |= IWM_CSR_INT_BIT_FH_TX;
5758                 sc->sc_fw_chunk_done = 1;
5759                 wakeup(&sc->sc_fw);
5760         }
5761
5762         if (r1 & IWM_CSR_INT_BIT_RF_KILL) {
5763                 handled |= IWM_CSR_INT_BIT_RF_KILL;
5764                 if (iwm_check_rfkill(sc)) {
5765                         device_printf(sc->sc_dev,
5766                             "%s: rfkill switch, disabling interface\n",
5767                             __func__);
5768                         iwm_stop(sc);
5769                 }
5770         }
5771
5772         /*
5773          * The Linux driver uses periodic interrupts to avoid races.
5774          * We cargo-cult like it's going out of fashion.
5775          */
5776         if (r1 & IWM_CSR_INT_BIT_RX_PERIODIC) {
5777                 handled |= IWM_CSR_INT_BIT_RX_PERIODIC;
5778                 IWM_WRITE(sc, IWM_CSR_INT, IWM_CSR_INT_BIT_RX_PERIODIC);
5779                 if ((r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX)) == 0)
5780                         IWM_WRITE_1(sc,
5781                             IWM_CSR_INT_PERIODIC_REG, IWM_CSR_INT_PERIODIC_DIS);
5782                 isperiodic = 1;
5783         }
5784
5785         if ((r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX)) || isperiodic) {
5786                 handled |= (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX);
5787                 IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, IWM_CSR_FH_INT_RX_MASK);
5788
5789                 iwm_notif_intr(sc);
5790
5791                 /* enable periodic interrupt, see above */
5792                 if (r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX) && !isperiodic)
5793                         IWM_WRITE_1(sc, IWM_CSR_INT_PERIODIC_REG,
5794                             IWM_CSR_INT_PERIODIC_ENA);
5795         }
5796
5797         if (__predict_false(r1 & ~handled))
5798                 IWM_DPRINTF(sc, IWM_DEBUG_INTR,
5799                     "%s: unhandled interrupts: %x\n", __func__, r1);
5800         rv = 1;
5801
5802  out_ena:
5803         iwm_restore_interrupts(sc);
5804  out:
5805         IWM_UNLOCK(sc);
5806         return;
5807 }
5808
5809 /*
5810  * Autoconf glue-sniffing
5811  */
5812 #define PCI_VENDOR_INTEL                0x8086
5813 #define PCI_PRODUCT_INTEL_WL_3160_1     0x08b3
5814 #define PCI_PRODUCT_INTEL_WL_3160_2     0x08b4
5815 #define PCI_PRODUCT_INTEL_WL_3165_1     0x3165
5816 #define PCI_PRODUCT_INTEL_WL_3165_2     0x3166
5817 #define PCI_PRODUCT_INTEL_WL_7260_1     0x08b1
5818 #define PCI_PRODUCT_INTEL_WL_7260_2     0x08b2
5819 #define PCI_PRODUCT_INTEL_WL_7265_1     0x095a
5820 #define PCI_PRODUCT_INTEL_WL_7265_2     0x095b
5821 #define PCI_PRODUCT_INTEL_WL_8260_1     0x24f3
5822 #define PCI_PRODUCT_INTEL_WL_8260_2     0x24f4
5823
5824 static const struct iwm_devices {
5825         uint16_t                device;
5826         const struct iwm_cfg    *cfg;
5827 } iwm_devices[] = {
5828         { PCI_PRODUCT_INTEL_WL_3160_1, &iwm3160_cfg },
5829         { PCI_PRODUCT_INTEL_WL_3160_2, &iwm3160_cfg },
5830         { PCI_PRODUCT_INTEL_WL_3165_1, &iwm3165_cfg },
5831         { PCI_PRODUCT_INTEL_WL_3165_2, &iwm3165_cfg },
5832         { PCI_PRODUCT_INTEL_WL_7260_1, &iwm7260_cfg },
5833         { PCI_PRODUCT_INTEL_WL_7260_2, &iwm7260_cfg },
5834         { PCI_PRODUCT_INTEL_WL_7265_1, &iwm7265_cfg },
5835         { PCI_PRODUCT_INTEL_WL_7265_2, &iwm7265_cfg },
5836         { PCI_PRODUCT_INTEL_WL_8260_1, &iwm8260_cfg },
5837         { PCI_PRODUCT_INTEL_WL_8260_2, &iwm8260_cfg },
5838 };
5839
5840 static int
5841 iwm_probe(device_t dev)
5842 {
5843         int i;
5844
5845         for (i = 0; i < nitems(iwm_devices); i++) {
5846                 if (pci_get_vendor(dev) == PCI_VENDOR_INTEL &&
5847                     pci_get_device(dev) == iwm_devices[i].device) {
5848                         device_set_desc(dev, iwm_devices[i].cfg->name);
5849                         return (BUS_PROBE_DEFAULT);
5850                 }
5851         }
5852
5853         return (ENXIO);
5854 }
5855
5856 static int
5857 iwm_dev_check(device_t dev)
5858 {
5859         struct iwm_softc *sc;
5860         uint16_t devid;
5861         int i;
5862
5863         sc = device_get_softc(dev);
5864
5865         devid = pci_get_device(dev);
5866         for (i = 0; i < nitems(iwm_devices); i++) {
5867                 if (iwm_devices[i].device == devid) {
5868                         sc->cfg = iwm_devices[i].cfg;
5869                         return (0);
5870                 }
5871         }
5872         device_printf(dev, "unknown adapter type\n");
5873         return ENXIO;
5874 }
5875
5876 /* PCI registers */
5877 #define PCI_CFG_RETRY_TIMEOUT   0x041
5878
5879 static int
5880 iwm_pci_attach(device_t dev)
5881 {
5882         struct iwm_softc *sc;
5883         int count, error, rid;
5884         uint16_t reg;
5885
5886         sc = device_get_softc(dev);
5887
5888         /* We disable the RETRY_TIMEOUT register (0x41) to keep
5889          * PCI Tx retries from interfering with C3 CPU state */
5890         pci_write_config(dev, PCI_CFG_RETRY_TIMEOUT, 0x00, 1);
5891
5892         /* Enable bus-mastering and hardware bug workaround. */
5893         pci_enable_busmaster(dev);
5894         reg = pci_read_config(dev, PCIR_STATUS, sizeof(reg));
5895         /* if !MSI */
5896         if (reg & PCIM_STATUS_INTxSTATE) {
5897                 reg &= ~PCIM_STATUS_INTxSTATE;
5898         }
5899         pci_write_config(dev, PCIR_STATUS, reg, sizeof(reg));
5900
5901         rid = PCIR_BAR(0);
5902         sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
5903             RF_ACTIVE);
5904         if (sc->sc_mem == NULL) {
5905                 device_printf(sc->sc_dev, "can't map mem space\n");
5906                 return (ENXIO);
5907         }
5908         sc->sc_st = rman_get_bustag(sc->sc_mem);
5909         sc->sc_sh = rman_get_bushandle(sc->sc_mem);
5910
5911         /* Install interrupt handler. */
5912         count = 1;
5913         rid = 0;
5914         if (pci_alloc_msi(dev, &count) == 0)
5915                 rid = 1;
5916         sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE |
5917             (rid != 0 ? 0 : RF_SHAREABLE));
5918         if (sc->sc_irq == NULL) {
5919                 device_printf(dev, "can't map interrupt\n");
5920                         return (ENXIO);
5921         }
5922         error = bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
5923             NULL, iwm_intr, sc, &sc->sc_ih);
5924         if (sc->sc_ih == NULL) {
5925                 device_printf(dev, "can't establish interrupt");
5926                         return (ENXIO);
5927         }
5928         sc->sc_dmat = bus_get_dma_tag(sc->sc_dev);
5929
5930         return (0);
5931 }
5932
5933 static void
5934 iwm_pci_detach(device_t dev)
5935 {
5936         struct iwm_softc *sc = device_get_softc(dev);
5937
5938         if (sc->sc_irq != NULL) {
5939                 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
5940                 bus_release_resource(dev, SYS_RES_IRQ,
5941                     rman_get_rid(sc->sc_irq), sc->sc_irq);
5942                 pci_release_msi(dev);
5943         }
5944         if (sc->sc_mem != NULL)
5945                 bus_release_resource(dev, SYS_RES_MEMORY,
5946                     rman_get_rid(sc->sc_mem), sc->sc_mem);
5947 }
5948
5949
5950
5951 static int
5952 iwm_attach(device_t dev)
5953 {
5954         struct iwm_softc *sc = device_get_softc(dev);
5955         struct ieee80211com *ic = &sc->sc_ic;
5956         int error;
5957         int txq_i, i;
5958
5959         sc->sc_dev = dev;
5960         sc->sc_attached = 1;
5961         IWM_LOCK_INIT(sc);
5962         mbufq_init(&sc->sc_snd, ifqmaxlen);
5963         callout_init_mtx(&sc->sc_watchdog_to, &sc->sc_mtx, 0);
5964         callout_init_mtx(&sc->sc_led_blink_to, &sc->sc_mtx, 0);
5965         TASK_INIT(&sc->sc_es_task, 0, iwm_endscan_cb, sc);
5966
5967         sc->sc_notif_wait = iwm_notification_wait_init(sc);
5968         if (sc->sc_notif_wait == NULL) {
5969                 device_printf(dev, "failed to init notification wait struct\n");
5970                 goto fail;
5971         }
5972
5973         /* Init phy db */
5974         sc->sc_phy_db = iwm_phy_db_init(sc);
5975         if (!sc->sc_phy_db) {
5976                 device_printf(dev, "Cannot init phy_db\n");
5977                 goto fail;
5978         }
5979
5980         /* Set EBS as successful as long as not stated otherwise by the FW. */
5981         sc->last_ebs_successful = TRUE;
5982
5983         /* PCI attach */
5984         error = iwm_pci_attach(dev);
5985         if (error != 0)
5986                 goto fail;
5987
5988         sc->sc_wantresp = -1;
5989
5990         /* Check device type */
5991         error = iwm_dev_check(dev);
5992         if (error != 0)
5993                 goto fail;
5994
5995         sc->sc_hw_rev = IWM_READ(sc, IWM_CSR_HW_REV);
5996         /*
5997          * In the 8000 HW family the format of the 4 bytes of CSR_HW_REV have
5998          * changed, and now the revision step also includes bit 0-1 (no more
5999          * "dash" value). To keep hw_rev backwards compatible - we'll store it
6000          * in the old format.
6001          */
6002         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000)
6003                 sc->sc_hw_rev = (sc->sc_hw_rev & 0xfff0) |
6004                                 (IWM_CSR_HW_REV_STEP(sc->sc_hw_rev << 2) << 2);
6005
6006         if (iwm_prepare_card_hw(sc) != 0) {
6007                 device_printf(dev, "could not initialize hardware\n");
6008                 goto fail;
6009         }
6010
6011         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) {
6012                 int ret;
6013                 uint32_t hw_step;
6014
6015                 /*
6016                  * In order to recognize C step the driver should read the
6017                  * chip version id located at the AUX bus MISC address.
6018                  */
6019                 IWM_SETBITS(sc, IWM_CSR_GP_CNTRL,
6020                             IWM_CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
6021                 DELAY(2);
6022
6023                 ret = iwm_poll_bit(sc, IWM_CSR_GP_CNTRL,
6024                                    IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
6025                                    IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
6026                                    25000);
6027                 if (!ret) {
6028                         device_printf(sc->sc_dev,
6029                             "Failed to wake up the nic\n");
6030                         goto fail;
6031                 }
6032
6033                 if (iwm_nic_lock(sc)) {
6034                         hw_step = iwm_read_prph(sc, IWM_WFPM_CTRL_REG);
6035                         hw_step |= IWM_ENABLE_WFPM;
6036                         iwm_write_prph(sc, IWM_WFPM_CTRL_REG, hw_step);
6037                         hw_step = iwm_read_prph(sc, IWM_AUX_MISC_REG);
6038                         hw_step = (hw_step >> IWM_HW_STEP_LOCATION_BITS) & 0xF;
6039                         if (hw_step == 0x3)
6040                                 sc->sc_hw_rev = (sc->sc_hw_rev & 0xFFFFFFF3) |
6041                                                 (IWM_SILICON_C_STEP << 2);
6042                         iwm_nic_unlock(sc);
6043                 } else {
6044                         device_printf(sc->sc_dev, "Failed to lock the nic\n");
6045                         goto fail;
6046                 }
6047         }
6048
6049         /* special-case 7265D, it has the same PCI IDs. */
6050         if (sc->cfg == &iwm7265_cfg &&
6051             (sc->sc_hw_rev & IWM_CSR_HW_REV_TYPE_MSK) == IWM_CSR_HW_REV_TYPE_7265D) {
6052                 sc->cfg = &iwm7265d_cfg;
6053         }
6054
6055         /* Allocate DMA memory for firmware transfers. */
6056         if ((error = iwm_alloc_fwmem(sc)) != 0) {
6057                 device_printf(dev, "could not allocate memory for firmware\n");
6058                 goto fail;
6059         }
6060
6061         /* Allocate "Keep Warm" page. */
6062         if ((error = iwm_alloc_kw(sc)) != 0) {
6063                 device_printf(dev, "could not allocate keep warm page\n");
6064                 goto fail;
6065         }
6066
6067         /* We use ICT interrupts */
6068         if ((error = iwm_alloc_ict(sc)) != 0) {
6069                 device_printf(dev, "could not allocate ICT table\n");
6070                 goto fail;
6071         }
6072
6073         /* Allocate TX scheduler "rings". */
6074         if ((error = iwm_alloc_sched(sc)) != 0) {
6075                 device_printf(dev, "could not allocate TX scheduler rings\n");
6076                 goto fail;
6077         }
6078
6079         /* Allocate TX rings */
6080         for (txq_i = 0; txq_i < nitems(sc->txq); txq_i++) {
6081                 if ((error = iwm_alloc_tx_ring(sc,
6082                     &sc->txq[txq_i], txq_i)) != 0) {
6083                         device_printf(dev,
6084                             "could not allocate TX ring %d\n",
6085                             txq_i);
6086                         goto fail;
6087                 }
6088         }
6089
6090         /* Allocate RX ring. */
6091         if ((error = iwm_alloc_rx_ring(sc, &sc->rxq)) != 0) {
6092                 device_printf(dev, "could not allocate RX ring\n");
6093                 goto fail;
6094         }
6095
6096         /* Clear pending interrupts. */
6097         IWM_WRITE(sc, IWM_CSR_INT, 0xffffffff);
6098
6099         ic->ic_softc = sc;
6100         ic->ic_name = device_get_nameunit(sc->sc_dev);
6101         ic->ic_phytype = IEEE80211_T_OFDM;      /* not only, but not used */
6102         ic->ic_opmode = IEEE80211_M_STA;        /* default to BSS mode */
6103
6104         /* Set device capabilities. */
6105         ic->ic_caps =
6106             IEEE80211_C_STA |
6107             IEEE80211_C_WPA |           /* WPA/RSN */
6108             IEEE80211_C_WME |
6109             IEEE80211_C_SHSLOT |        /* short slot time supported */
6110             IEEE80211_C_SHPREAMBLE      /* short preamble supported */
6111 //          IEEE80211_C_BGSCAN          /* capable of bg scanning */
6112             ;
6113         /* Advertise full-offload scanning */
6114         ic->ic_flags_ext = IEEE80211_FEXT_SCAN_OFFLOAD;
6115         for (i = 0; i < nitems(sc->sc_phyctxt); i++) {
6116                 sc->sc_phyctxt[i].id = i;
6117                 sc->sc_phyctxt[i].color = 0;
6118                 sc->sc_phyctxt[i].ref = 0;
6119                 sc->sc_phyctxt[i].channel = NULL;
6120         }
6121
6122         /* Default noise floor */
6123         sc->sc_noise = -96;
6124
6125         /* Max RSSI */
6126         sc->sc_max_rssi = IWM_MAX_DBM - IWM_MIN_DBM;
6127
6128         sc->sc_preinit_hook.ich_func = iwm_preinit;
6129         sc->sc_preinit_hook.ich_arg = sc;
6130         if (config_intrhook_establish(&sc->sc_preinit_hook) != 0) {
6131                 device_printf(dev, "config_intrhook_establish failed\n");
6132                 goto fail;
6133         }
6134
6135 #ifdef IWM_DEBUG
6136         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
6137             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug",
6138             CTLFLAG_RW, &sc->sc_debug, 0, "control debugging");
6139 #endif
6140
6141         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6142             "<-%s\n", __func__);
6143
6144         return 0;
6145
6146         /* Free allocated memory if something failed during attachment. */
6147 fail:
6148         iwm_detach_local(sc, 0);
6149
6150         return ENXIO;
6151 }
6152
6153 static int
6154 iwm_is_valid_ether_addr(uint8_t *addr)
6155 {
6156         char zero_addr[IEEE80211_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 };
6157
6158         if ((addr[0] & 1) || IEEE80211_ADDR_EQ(zero_addr, addr))
6159                 return (FALSE);
6160
6161         return (TRUE);
6162 }
6163
6164 static int
6165 iwm_update_edca(struct ieee80211com *ic)
6166 {
6167         struct iwm_softc *sc = ic->ic_softc;
6168
6169         device_printf(sc->sc_dev, "%s: called\n", __func__);
6170         return (0);
6171 }
6172
6173 static void
6174 iwm_preinit(void *arg)
6175 {
6176         struct iwm_softc *sc = arg;
6177         device_t dev = sc->sc_dev;
6178         struct ieee80211com *ic = &sc->sc_ic;
6179         int error;
6180
6181         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6182             "->%s\n", __func__);
6183
6184         IWM_LOCK(sc);
6185         if ((error = iwm_start_hw(sc)) != 0) {
6186                 device_printf(dev, "could not initialize hardware\n");
6187                 IWM_UNLOCK(sc);
6188                 goto fail;
6189         }
6190
6191         error = iwm_run_init_mvm_ucode(sc, 1);
6192         iwm_stop_device(sc);
6193         if (error) {
6194                 IWM_UNLOCK(sc);
6195                 goto fail;
6196         }
6197         device_printf(dev,
6198             "hw rev 0x%x, fw ver %s, address %s\n",
6199             sc->sc_hw_rev & IWM_CSR_HW_REV_TYPE_MSK,
6200             sc->sc_fwver, ether_sprintf(sc->nvm_data->hw_addr));
6201
6202         /* not all hardware can do 5GHz band */
6203         if (!sc->nvm_data->sku_cap_band_52GHz_enable)
6204                 memset(&ic->ic_sup_rates[IEEE80211_MODE_11A], 0,
6205                     sizeof(ic->ic_sup_rates[IEEE80211_MODE_11A]));
6206         IWM_UNLOCK(sc);
6207
6208         iwm_init_channel_map(ic, IEEE80211_CHAN_MAX, &ic->ic_nchans,
6209             ic->ic_channels);
6210
6211         /*
6212          * At this point we've committed - if we fail to do setup,
6213          * we now also have to tear down the net80211 state.
6214          */
6215         ieee80211_ifattach(ic);
6216         ic->ic_vap_create = iwm_vap_create;
6217         ic->ic_vap_delete = iwm_vap_delete;
6218         ic->ic_raw_xmit = iwm_raw_xmit;
6219         ic->ic_node_alloc = iwm_node_alloc;
6220         ic->ic_scan_start = iwm_scan_start;
6221         ic->ic_scan_end = iwm_scan_end;
6222         ic->ic_update_mcast = iwm_update_mcast;
6223         ic->ic_getradiocaps = iwm_init_channel_map;
6224         ic->ic_set_channel = iwm_set_channel;
6225         ic->ic_scan_curchan = iwm_scan_curchan;
6226         ic->ic_scan_mindwell = iwm_scan_mindwell;
6227         ic->ic_wme.wme_update = iwm_update_edca;
6228         ic->ic_parent = iwm_parent;
6229         ic->ic_transmit = iwm_transmit;
6230         iwm_radiotap_attach(sc);
6231         if (bootverbose)
6232                 ieee80211_announce(ic);
6233
6234         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6235             "<-%s\n", __func__);
6236         config_intrhook_disestablish(&sc->sc_preinit_hook);
6237
6238         return;
6239 fail:
6240         config_intrhook_disestablish(&sc->sc_preinit_hook);
6241         iwm_detach_local(sc, 0);
6242 }
6243
6244 /*
6245  * Attach the interface to 802.11 radiotap.
6246  */
6247 static void
6248 iwm_radiotap_attach(struct iwm_softc *sc)
6249 {
6250         struct ieee80211com *ic = &sc->sc_ic;
6251
6252         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6253             "->%s begin\n", __func__);
6254         ieee80211_radiotap_attach(ic,
6255             &sc->sc_txtap.wt_ihdr, sizeof(sc->sc_txtap),
6256                 IWM_TX_RADIOTAP_PRESENT,
6257             &sc->sc_rxtap.wr_ihdr, sizeof(sc->sc_rxtap),
6258                 IWM_RX_RADIOTAP_PRESENT);
6259         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6260             "->%s end\n", __func__);
6261 }
6262
6263 static struct ieee80211vap *
6264 iwm_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], int unit,
6265     enum ieee80211_opmode opmode, int flags,
6266     const uint8_t bssid[IEEE80211_ADDR_LEN],
6267     const uint8_t mac[IEEE80211_ADDR_LEN])
6268 {
6269         struct iwm_vap *ivp;
6270         struct ieee80211vap *vap;
6271
6272         if (!TAILQ_EMPTY(&ic->ic_vaps))         /* only one at a time */
6273                 return NULL;
6274         ivp = malloc(sizeof(struct iwm_vap), M_80211_VAP, M_WAITOK | M_ZERO);
6275         vap = &ivp->iv_vap;
6276         ieee80211_vap_setup(ic, vap, name, unit, opmode, flags, bssid);
6277         vap->iv_bmissthreshold = 10;            /* override default */
6278         /* Override with driver methods. */
6279         ivp->iv_newstate = vap->iv_newstate;
6280         vap->iv_newstate = iwm_newstate;
6281
6282         ieee80211_ratectl_init(vap);
6283         /* Complete setup. */
6284         ieee80211_vap_attach(vap, iwm_media_change, ieee80211_media_status,
6285             mac);
6286         ic->ic_opmode = opmode;
6287
6288         return vap;
6289 }
6290
6291 static void
6292 iwm_vap_delete(struct ieee80211vap *vap)
6293 {
6294         struct iwm_vap *ivp = IWM_VAP(vap);
6295
6296         ieee80211_ratectl_deinit(vap);
6297         ieee80211_vap_detach(vap);
6298         free(ivp, M_80211_VAP);
6299 }
6300
6301 static void
6302 iwm_scan_start(struct ieee80211com *ic)
6303 {
6304         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
6305         struct iwm_softc *sc = ic->ic_softc;
6306         int error;
6307
6308         IWM_LOCK(sc);
6309         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
6310                 /* This should not be possible */
6311                 device_printf(sc->sc_dev,
6312                     "%s: Previous scan not completed yet\n", __func__);
6313         }
6314         if (fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_UMAC_SCAN))
6315                 error = iwm_mvm_umac_scan(sc);
6316         else
6317                 error = iwm_mvm_lmac_scan(sc);
6318         if (error != 0) {
6319                 device_printf(sc->sc_dev, "could not initiate scan\n");
6320                 IWM_UNLOCK(sc);
6321                 ieee80211_cancel_scan(vap);
6322         } else {
6323                 sc->sc_flags |= IWM_FLAG_SCAN_RUNNING;
6324                 iwm_led_blink_start(sc);
6325                 IWM_UNLOCK(sc);
6326         }
6327 }
6328
6329 static void
6330 iwm_scan_end(struct ieee80211com *ic)
6331 {
6332         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
6333         struct iwm_softc *sc = ic->ic_softc;
6334
6335         IWM_LOCK(sc);
6336         iwm_led_blink_stop(sc);
6337         if (vap->iv_state == IEEE80211_S_RUN)
6338                 iwm_mvm_led_enable(sc);
6339         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
6340                 /*
6341                  * Removing IWM_FLAG_SCAN_RUNNING now, is fine because
6342                  * both iwm_scan_end and iwm_scan_start run in the ic->ic_tq
6343                  * taskqueue.
6344                  */
6345                 sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
6346                 iwm_mvm_scan_stop_wait(sc);
6347         }
6348         IWM_UNLOCK(sc);
6349
6350         /*
6351          * Make sure we don't race, if sc_es_task is still enqueued here.
6352          * This is to make sure that it won't call ieee80211_scan_done
6353          * when we have already started the next scan.
6354          */
6355         taskqueue_cancel(ic->ic_tq, &sc->sc_es_task, NULL);
6356 }
6357
6358 static void
6359 iwm_update_mcast(struct ieee80211com *ic)
6360 {
6361 }
6362
6363 static void
6364 iwm_set_channel(struct ieee80211com *ic)
6365 {
6366 }
6367
6368 static void
6369 iwm_scan_curchan(struct ieee80211_scan_state *ss, unsigned long maxdwell)
6370 {
6371 }
6372
6373 static void
6374 iwm_scan_mindwell(struct ieee80211_scan_state *ss)
6375 {
6376         return;
6377 }
6378
6379 void
6380 iwm_init_task(void *arg1)
6381 {
6382         struct iwm_softc *sc = arg1;
6383
6384         IWM_LOCK(sc);
6385         while (sc->sc_flags & IWM_FLAG_BUSY)
6386                 msleep(&sc->sc_flags, &sc->sc_mtx, 0, "iwmpwr", 0);
6387         sc->sc_flags |= IWM_FLAG_BUSY;
6388         iwm_stop(sc);
6389         if (sc->sc_ic.ic_nrunning > 0)
6390                 iwm_init(sc);
6391         sc->sc_flags &= ~IWM_FLAG_BUSY;
6392         wakeup(&sc->sc_flags);
6393         IWM_UNLOCK(sc);
6394 }
6395
6396 static int
6397 iwm_resume(device_t dev)
6398 {
6399         struct iwm_softc *sc = device_get_softc(dev);
6400         int do_reinit = 0;
6401
6402         /*
6403          * We disable the RETRY_TIMEOUT register (0x41) to keep
6404          * PCI Tx retries from interfering with C3 CPU state.
6405          */
6406         pci_write_config(dev, PCI_CFG_RETRY_TIMEOUT, 0x00, 1);
6407         iwm_init_task(device_get_softc(dev));
6408
6409         IWM_LOCK(sc);
6410         if (sc->sc_flags & IWM_FLAG_SCANNING) {
6411                 sc->sc_flags &= ~IWM_FLAG_SCANNING;
6412                 do_reinit = 1;
6413         }
6414         IWM_UNLOCK(sc);
6415
6416         if (do_reinit)
6417                 ieee80211_resume_all(&sc->sc_ic);
6418
6419         return 0;
6420 }
6421
6422 static int
6423 iwm_suspend(device_t dev)
6424 {
6425         int do_stop = 0;
6426         struct iwm_softc *sc = device_get_softc(dev);
6427
6428         do_stop = !! (sc->sc_ic.ic_nrunning > 0);
6429
6430         ieee80211_suspend_all(&sc->sc_ic);
6431
6432         if (do_stop) {
6433                 IWM_LOCK(sc);
6434                 iwm_stop(sc);
6435                 sc->sc_flags |= IWM_FLAG_SCANNING;
6436                 IWM_UNLOCK(sc);
6437         }
6438
6439         return (0);
6440 }
6441
6442 static int
6443 iwm_detach_local(struct iwm_softc *sc, int do_net80211)
6444 {
6445         struct iwm_fw_info *fw = &sc->sc_fw;
6446         device_t dev = sc->sc_dev;
6447         int i;
6448
6449         if (!sc->sc_attached)
6450                 return 0;
6451         sc->sc_attached = 0;
6452
6453         if (do_net80211)
6454                 ieee80211_draintask(&sc->sc_ic, &sc->sc_es_task);
6455
6456         callout_drain(&sc->sc_led_blink_to);
6457         callout_drain(&sc->sc_watchdog_to);
6458         iwm_stop_device(sc);
6459         if (do_net80211) {
6460                 ieee80211_ifdetach(&sc->sc_ic);
6461         }
6462
6463         iwm_phy_db_free(sc->sc_phy_db);
6464         sc->sc_phy_db = NULL;
6465
6466         iwm_free_nvm_data(sc->nvm_data);
6467
6468         /* Free descriptor rings */
6469         iwm_free_rx_ring(sc, &sc->rxq);
6470         for (i = 0; i < nitems(sc->txq); i++)
6471                 iwm_free_tx_ring(sc, &sc->txq[i]);
6472
6473         /* Free firmware */
6474         if (fw->fw_fp != NULL)
6475                 iwm_fw_info_free(fw);
6476
6477         /* Free scheduler */
6478         iwm_dma_contig_free(&sc->sched_dma);
6479         iwm_dma_contig_free(&sc->ict_dma);
6480         iwm_dma_contig_free(&sc->kw_dma);
6481         iwm_dma_contig_free(&sc->fw_dma);
6482
6483         iwm_free_fw_paging(sc);
6484
6485         /* Finished with the hardware - detach things */
6486         iwm_pci_detach(dev);
6487
6488         if (sc->sc_notif_wait != NULL) {
6489                 iwm_notification_wait_free(sc->sc_notif_wait);
6490                 sc->sc_notif_wait = NULL;
6491         }
6492
6493         mbufq_drain(&sc->sc_snd);
6494         IWM_LOCK_DESTROY(sc);
6495
6496         return (0);
6497 }
6498
6499 static int
6500 iwm_detach(device_t dev)
6501 {
6502         struct iwm_softc *sc = device_get_softc(dev);
6503
6504         return (iwm_detach_local(sc, 1));
6505 }
6506
6507 static device_method_t iwm_pci_methods[] = {
6508         /* Device interface */
6509         DEVMETHOD(device_probe,         iwm_probe),
6510         DEVMETHOD(device_attach,        iwm_attach),
6511         DEVMETHOD(device_detach,        iwm_detach),
6512         DEVMETHOD(device_suspend,       iwm_suspend),
6513         DEVMETHOD(device_resume,        iwm_resume),
6514
6515         DEVMETHOD_END
6516 };
6517
6518 static driver_t iwm_pci_driver = {
6519         "iwm",
6520         iwm_pci_methods,
6521         sizeof (struct iwm_softc)
6522 };
6523
6524 static devclass_t iwm_devclass;
6525
6526 DRIVER_MODULE(iwm, pci, iwm_pci_driver, iwm_devclass, NULL, NULL);
6527 MODULE_DEPEND(iwm, firmware, 1, 1, 1);
6528 MODULE_DEPEND(iwm, pci, 1, 1, 1);
6529 MODULE_DEPEND(iwm, wlan, 1, 1, 1);