sys/dev/iwm/if_iwm.c

   1 /*      $OpenBSD: if_iwm.c,v 1.42 2015/05/30 02:49:23 deraadt Exp $     */
   2
   3 /*
   4  * Copyright (c) 2014 genua mbh <info@genua.de>
   5  * Copyright (c) 2014 Fixup Software Ltd.
   6  *
   7  * Permission to use, copy, modify, and distribute this software for any
   8  * purpose with or without fee is hereby granted, provided that the above
   9  * copyright notice and this permission notice appear in all copies.
  10  *
  11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  18  */
  19
  20 /*-
  21  * Based on BSD-licensed source modules in the Linux iwlwifi driver,
  22  * which were used as the reference documentation for this implementation.
  23  *
  24  * Driver version we are currently based off of is
  25  * Linux 3.14.3 (tag id a2df521e42b1d9a23f620ac79dbfe8655a8391dd)
  26  *
  27  ***********************************************************************
  28  *
  29  * This file is provided under a dual BSD/GPLv2 license.  When using or
  30  * redistributing this file, you may do so under either license.
  31  *
  32  * GPL LICENSE SUMMARY
  33  *
  34  * Copyright(c) 2007 - 2013 Intel Corporation. All rights reserved.
  35  *
  36  * This program is free software; you can redistribute it and/or modify
  37  * it under the terms of version 2 of the GNU General Public License as
  38  * published by the Free Software Foundation.
  39  *
  40  * This program is distributed in the hope that it will be useful, but
  41  * WITHOUT ANY WARRANTY; without even the implied warranty of
  42  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  43  * General Public License for more details.
  44  *
  45  * You should have received a copy of the GNU General Public License
  46  * along with this program; if not, write to the Free Software
  47  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
  48  * USA
  49  *
  50  * The full GNU General Public License is included in this distribution
  51  * in the file called COPYING.
  52  *
  53  * Contact Information:
  54  *  Intel Linux Wireless <ilw@linux.intel.com>
  55  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  56  *
  57  *
  58  * BSD LICENSE
  59  *
  60  * Copyright(c) 2005 - 2013 Intel Corporation. All rights reserved.
  61  * All rights reserved.
  62  *
  63  * Redistribution and use in source and binary forms, with or without
  64  * modification, are permitted provided that the following conditions
  65  * are met:
  66  *
  67  *  * Redistributions of source code must retain the above copyright
  68  *    notice, this list of conditions and the following disclaimer.
  69  *  * Redistributions in binary form must reproduce the above copyright
  70  *    notice, this list of conditions and the following disclaimer in
  71  *    the documentation and/or other materials provided with the
  72  *    distribution.
  73  *  * Neither the name Intel Corporation nor the names of its
  74  *    contributors may be used to endorse or promote products derived
  75  *    from this software without specific prior written permission.
  76  *
  77  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  78  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  79  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  80  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  81  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  82  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  83  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  84  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  85  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  86  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  87  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  88  */
  89
  90 /*-
  91  * Copyright (c) 2007-2010 Damien Bergamini <damien.bergamini@free.fr>
  92  *
  93  * Permission to use, copy, modify, and distribute this software for any
  94  * purpose with or without fee is hereby granted, provided that the above
  95  * copyright notice and this permission notice appear in all copies.
  96  *
  97  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  98  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  99  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 100  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 101  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 102  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 103  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 104  */
 105 #include <sys/cdefs.h>
 106 __FBSDID("$FreeBSD$");
 107
 108 #include "opt_wlan.h"
 109
 110 #include <sys/param.h>
 111 #include <sys/bus.h>
 112 #include <sys/conf.h>
 113 #include <sys/endian.h>
 114 #include <sys/firmware.h>
 115 #include <sys/kernel.h>
 116 #include <sys/malloc.h>
 117 #include <sys/mbuf.h>
 118 #include <sys/mutex.h>
 119 #include <sys/module.h>
 120 #include <sys/proc.h>
 121 #include <sys/rman.h>
 122 #include <sys/socket.h>
 123 #include <sys/sockio.h>
 124 #include <sys/sysctl.h>
 125 #include <sys/linker.h>
 126
 127 #include <machine/bus.h>
 128 #include <machine/endian.h>
 129 #include <machine/resource.h>
 130
 131 #include <dev/pci/pcivar.h>
 132 #include <dev/pci/pcireg.h>
 133
 134 #include <net/bpf.h>
 135
 136 #include <net/if.h>
 137 #include <net/if_var.h>
 138 #include <net/if_arp.h>
 139 #include <net/if_dl.h>
 140 #include <net/if_media.h>
 141 #include <net/if_types.h>
 142
 143 #include <netinet/in.h>
 144 #include <netinet/in_systm.h>
 145 #include <netinet/if_ether.h>
 146 #include <netinet/ip.h>
 147
 148 #include <net80211/ieee80211_var.h>
 149 #include <net80211/ieee80211_regdomain.h>
 150 #include <net80211/ieee80211_ratectl.h>
 151 #include <net80211/ieee80211_radiotap.h>
 152
 153 #include <dev/iwm/if_iwmreg.h>
 154 #include <dev/iwm/if_iwmvar.h>
 155 #include <dev/iwm/if_iwm_config.h>
 156 #include <dev/iwm/if_iwm_debug.h>
 157 #include <dev/iwm/if_iwm_notif_wait.h>
 158 #include <dev/iwm/if_iwm_util.h>
 159 #include <dev/iwm/if_iwm_binding.h>
 160 #include <dev/iwm/if_iwm_phy_db.h>
 161 #include <dev/iwm/if_iwm_mac_ctxt.h>
 162 #include <dev/iwm/if_iwm_phy_ctxt.h>
 163 #include <dev/iwm/if_iwm_time_event.h>
 164 #include <dev/iwm/if_iwm_power.h>
 165 #include <dev/iwm/if_iwm_scan.h>
 166
 167 #include <dev/iwm/if_iwm_pcie_trans.h>
 168 #include <dev/iwm/if_iwm_led.h>
 169 #include <dev/iwm/if_iwm_fw.h>
 170
 171 const uint8_t iwm_nvm_channels[] = {
 172         /* 2.4 GHz */
 173         1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
 174         /* 5 GHz */
 175         36, 40, 44, 48, 52, 56, 60, 64,
 176         100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144,
 177         149, 153, 157, 161, 165
 178 };
 179 _Static_assert(nitems(iwm_nvm_channels) <= IWM_NUM_CHANNELS,
 180     "IWM_NUM_CHANNELS is too small");
 181
 182 const uint8_t iwm_nvm_channels_8000[] = {
 183         /* 2.4 GHz */
 184         1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
 185         /* 5 GHz */
 186         36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92,
 187         96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144,
 188         149, 153, 157, 161, 165, 169, 173, 177, 181
 189 };
 190 _Static_assert(nitems(iwm_nvm_channels_8000) <= IWM_NUM_CHANNELS_8000,
 191     "IWM_NUM_CHANNELS_8000 is too small");
 192
 193 #define IWM_NUM_2GHZ_CHANNELS   14
 194 #define IWM_N_HW_ADDR_MASK      0xF
 195
 196 /*
 197  * XXX For now, there's simply a fixed set of rate table entries
 198  * that are populated.
 199  */
 200 const struct iwm_rate {
 201         uint8_t rate;
 202         uint8_t plcp;
 203 } iwm_rates[] = {
 204         {   2,  IWM_RATE_1M_PLCP  },
 205         {   4,  IWM_RATE_2M_PLCP  },
 206         {  11,  IWM_RATE_5M_PLCP  },
 207         {  22,  IWM_RATE_11M_PLCP },
 208         {  12,  IWM_RATE_6M_PLCP  },
 209         {  18,  IWM_RATE_9M_PLCP  },
 210         {  24,  IWM_RATE_12M_PLCP },
 211         {  36,  IWM_RATE_18M_PLCP },
 212         {  48,  IWM_RATE_24M_PLCP },
 213         {  72,  IWM_RATE_36M_PLCP },
 214         {  96,  IWM_RATE_48M_PLCP },
 215         { 108,  IWM_RATE_54M_PLCP },
 216 };
 217 #define IWM_RIDX_CCK    0
 218 #define IWM_RIDX_OFDM   4
 219 #define IWM_RIDX_MAX    (nitems(iwm_rates)-1)
 220 #define IWM_RIDX_IS_CCK(_i_) ((_i_) < IWM_RIDX_OFDM)
 221 #define IWM_RIDX_IS_OFDM(_i_) ((_i_) >= IWM_RIDX_OFDM)
 222
 223 struct iwm_nvm_section {
 224         uint16_t length;
 225         uint8_t *data;
 226 };
 227
 228 #define IWM_MVM_UCODE_ALIVE_TIMEOUT     hz
 229 #define IWM_MVM_UCODE_CALIB_TIMEOUT     (2*hz)
 230
 231 struct iwm_mvm_alive_data {
 232         int valid;
 233         uint32_t scd_base_addr;
 234 };
 235
 236 static int      iwm_store_cscheme(struct iwm_softc *, const uint8_t *, size_t);
 237 static int      iwm_firmware_store_section(struct iwm_softc *,
 238                                            enum iwm_ucode_type,
 239                                            const uint8_t *, size_t);
 240 static int      iwm_set_default_calib(struct iwm_softc *, const void *);
 241 static void     iwm_fw_info_free(struct iwm_fw_info *);
 242 static int      iwm_read_firmware(struct iwm_softc *, enum iwm_ucode_type);
 243 static int      iwm_alloc_fwmem(struct iwm_softc *);
 244 static int      iwm_alloc_sched(struct iwm_softc *);
 245 static int      iwm_alloc_kw(struct iwm_softc *);
 246 static int      iwm_alloc_ict(struct iwm_softc *);
 247 static int      iwm_alloc_rx_ring(struct iwm_softc *, struct iwm_rx_ring *);
 248 static void     iwm_reset_rx_ring(struct iwm_softc *, struct iwm_rx_ring *);
 249 static void     iwm_free_rx_ring(struct iwm_softc *, struct iwm_rx_ring *);
 250 static int      iwm_alloc_tx_ring(struct iwm_softc *, struct iwm_tx_ring *,
 251                                   int);
 252 static void     iwm_reset_tx_ring(struct iwm_softc *, struct iwm_tx_ring *);
 253 static void     iwm_free_tx_ring(struct iwm_softc *, struct iwm_tx_ring *);
 254 static void     iwm_enable_interrupts(struct iwm_softc *);
 255 static void     iwm_restore_interrupts(struct iwm_softc *);
 256 static void     iwm_disable_interrupts(struct iwm_softc *);
 257 static void     iwm_ict_reset(struct iwm_softc *);
 258 static int      iwm_allow_mcast(struct ieee80211vap *, struct iwm_softc *);
 259 static void     iwm_stop_device(struct iwm_softc *);
 260 static void     iwm_mvm_nic_config(struct iwm_softc *);
 261 static int      iwm_nic_rx_init(struct iwm_softc *);
 262 static int      iwm_nic_tx_init(struct iwm_softc *);
 263 static int      iwm_nic_init(struct iwm_softc *);
 264 static int      iwm_enable_txq(struct iwm_softc *, int, int, int);
 265 static int      iwm_trans_pcie_fw_alive(struct iwm_softc *, uint32_t);
 266 static int      iwm_nvm_read_chunk(struct iwm_softc *, uint16_t, uint16_t,
 267                                    uint16_t, uint8_t *, uint16_t *);
 268 static int      iwm_nvm_read_section(struct iwm_softc *, uint16_t, uint8_t *,
 269                                      uint16_t *, uint32_t);
 270 static uint32_t iwm_eeprom_channel_flags(uint16_t);
 271 static void     iwm_add_channel_band(struct iwm_softc *,
 272                     struct ieee80211_channel[], int, int *, int, size_t,
 273                     const uint8_t[]);
 274 static void     iwm_init_channel_map(struct ieee80211com *, int, int *,
 275                     struct ieee80211_channel[]);
 276 static struct iwm_nvm_data *
 277         iwm_parse_nvm_data(struct iwm_softc *, const uint16_t *,
 278                            const uint16_t *, const uint16_t *,
 279                            const uint16_t *, const uint16_t *,
 280                            const uint16_t *);
 281 static void     iwm_free_nvm_data(struct iwm_nvm_data *);
 282 static void     iwm_set_hw_address_family_8000(struct iwm_softc *,
 283                                                struct iwm_nvm_data *,
 284                                                const uint16_t *,
 285                                                const uint16_t *);
 286 static int      iwm_get_sku(const struct iwm_softc *, const uint16_t *,
 287                             const uint16_t *);
 288 static int      iwm_get_nvm_version(const struct iwm_softc *, const uint16_t *);
 289 static int      iwm_get_radio_cfg(const struct iwm_softc *, const uint16_t *,
 290                                   const uint16_t *);
 291 static int      iwm_get_n_hw_addrs(const struct iwm_softc *,
 292                                    const uint16_t *);
 293 static void     iwm_set_radio_cfg(const struct iwm_softc *,
 294                                   struct iwm_nvm_data *, uint32_t);
 295 static struct iwm_nvm_data *
 296         iwm_parse_nvm_sections(struct iwm_softc *, struct iwm_nvm_section *);
 297 static int      iwm_nvm_init(struct iwm_softc *);
 298 static int      iwm_pcie_load_section(struct iwm_softc *, uint8_t,
 299                                       const struct iwm_fw_desc *);
 300 static int      iwm_pcie_load_firmware_chunk(struct iwm_softc *, uint32_t,
 301                                              bus_addr_t, uint32_t);
 302 static int      iwm_pcie_load_cpu_sections_8000(struct iwm_softc *sc,
 303                                                 const struct iwm_fw_sects *,
 304                                                 int, int *);
 305 static int      iwm_pcie_load_cpu_sections(struct iwm_softc *,
 306                                            const struct iwm_fw_sects *,
 307                                            int, int *);
 308 static int      iwm_pcie_load_given_ucode_8000(struct iwm_softc *,
 309                                                const struct iwm_fw_sects *);
 310 static int      iwm_pcie_load_given_ucode(struct iwm_softc *,
 311                                           const struct iwm_fw_sects *);
 312 static int      iwm_start_fw(struct iwm_softc *, const struct iwm_fw_sects *);
 313 static int      iwm_send_tx_ant_cfg(struct iwm_softc *, uint8_t);
 314 static int      iwm_send_phy_cfg_cmd(struct iwm_softc *);
 315 static int      iwm_mvm_load_ucode_wait_alive(struct iwm_softc *,
 316                                               enum iwm_ucode_type);
 317 static int      iwm_run_init_mvm_ucode(struct iwm_softc *, int);
 318 static int      iwm_rx_addbuf(struct iwm_softc *, int, int);
 319 static int      iwm_mvm_get_signal_strength(struct iwm_softc *,
 320                                             struct iwm_rx_phy_info *);
 321 static void     iwm_mvm_rx_rx_phy_cmd(struct iwm_softc *,
 322                                       struct iwm_rx_packet *,
 323                                       struct iwm_rx_data *);
 324 static int      iwm_get_noise(struct iwm_softc *sc,
 325                     const struct iwm_mvm_statistics_rx_non_phy *);
 326 static void     iwm_mvm_rx_rx_mpdu(struct iwm_softc *, struct mbuf *);
 327 static int      iwm_mvm_rx_tx_cmd_single(struct iwm_softc *,
 328                                          struct iwm_rx_packet *,
 329                                          struct iwm_node *);
 330 static void     iwm_mvm_rx_tx_cmd(struct iwm_softc *, struct iwm_rx_packet *,
 331                                   struct iwm_rx_data *);
 332 static void     iwm_cmd_done(struct iwm_softc *, struct iwm_rx_packet *);
 333 #if 0
 334 static void     iwm_update_sched(struct iwm_softc *, int, int, uint8_t,
 335                                  uint16_t);
 336 #endif
 337 static const struct iwm_rate *
 338         iwm_tx_fill_cmd(struct iwm_softc *, struct iwm_node *,
 339                         struct mbuf *, struct iwm_tx_cmd *);
 340 static int      iwm_tx(struct iwm_softc *, struct mbuf *,
 341                        struct ieee80211_node *, int);
 342 static int      iwm_raw_xmit(struct ieee80211_node *, struct mbuf *,
 343                              const struct ieee80211_bpf_params *);
 344 static int      iwm_mvm_flush_tx_path(struct iwm_softc *sc,
 345                                       uint32_t tfd_msk, uint32_t flags);
 346 static int      iwm_mvm_send_add_sta_cmd_status(struct iwm_softc *,
 347                                                 struct iwm_mvm_add_sta_cmd *,
 348                                                 int *);
 349 static int      iwm_mvm_sta_send_to_fw(struct iwm_softc *, struct iwm_node *,
 350                                        int);
 351 static int      iwm_mvm_add_sta(struct iwm_softc *, struct iwm_node *);
 352 static int      iwm_mvm_update_sta(struct iwm_softc *, struct iwm_node *);
 353 static int      iwm_mvm_add_int_sta_common(struct iwm_softc *,
 354                                            struct iwm_int_sta *,
 355                                            const uint8_t *, uint16_t, uint16_t);
 356 static int      iwm_mvm_add_aux_sta(struct iwm_softc *);
 357 static int      iwm_mvm_update_quotas(struct iwm_softc *, struct iwm_node *);
 358 static int      iwm_auth(struct ieee80211vap *, struct iwm_softc *);
 359 static int      iwm_assoc(struct ieee80211vap *, struct iwm_softc *);
 360 static int      iwm_release(struct iwm_softc *, struct iwm_node *);
 361 static struct ieee80211_node *
 362                 iwm_node_alloc(struct ieee80211vap *,
 363                                const uint8_t[IEEE80211_ADDR_LEN]);
 364 static void     iwm_setrates(struct iwm_softc *, struct iwm_node *);
 365 static int      iwm_media_change(struct ifnet *);
 366 static int      iwm_newstate(struct ieee80211vap *, enum ieee80211_state, int);
 367 static void     iwm_endscan_cb(void *, int);
 368 static void     iwm_mvm_fill_sf_command(struct iwm_softc *,
 369                                         struct iwm_sf_cfg_cmd *,
 370                                         struct ieee80211_node *);
 371 static int      iwm_mvm_sf_config(struct iwm_softc *, enum iwm_sf_state);
 372 static int      iwm_send_bt_init_conf(struct iwm_softc *);
 373 static int      iwm_send_update_mcc_cmd(struct iwm_softc *, const char *);
 374 static void     iwm_mvm_tt_tx_backoff(struct iwm_softc *, uint32_t);
 375 static int      iwm_init_hw(struct iwm_softc *);
 376 static void     iwm_init(struct iwm_softc *);
 377 static void     iwm_start(struct iwm_softc *);
 378 static void     iwm_stop(struct iwm_softc *);
 379 static void     iwm_watchdog(void *);
 380 static void     iwm_parent(struct ieee80211com *);
 381 #ifdef IWM_DEBUG
 382 static const char *
 383                 iwm_desc_lookup(uint32_t);
 384 static void     iwm_nic_error(struct iwm_softc *);
 385 static void     iwm_nic_umac_error(struct iwm_softc *);
 386 #endif
 387 static void     iwm_notif_intr(struct iwm_softc *);
 388 static void     iwm_intr(void *);
 389 static int      iwm_attach(device_t);
 390 static int      iwm_is_valid_ether_addr(uint8_t *);
 391 static void     iwm_preinit(void *);
 392 static int      iwm_detach_local(struct iwm_softc *sc, int);
 393 static void     iwm_init_task(void *);
 394 static void     iwm_radiotap_attach(struct iwm_softc *);
 395 static struct ieee80211vap *
 396                 iwm_vap_create(struct ieee80211com *,
 397                                const char [IFNAMSIZ], int,
 398                                enum ieee80211_opmode, int,
 399                                const uint8_t [IEEE80211_ADDR_LEN],
 400                                const uint8_t [IEEE80211_ADDR_LEN]);
 401 static void     iwm_vap_delete(struct ieee80211vap *);
 402 static void     iwm_scan_start(struct ieee80211com *);
 403 static void     iwm_scan_end(struct ieee80211com *);
 404 static void     iwm_update_mcast(struct ieee80211com *);
 405 static void     iwm_set_channel(struct ieee80211com *);
 406 static void     iwm_scan_curchan(struct ieee80211_scan_state *, unsigned long);
 407 static void     iwm_scan_mindwell(struct ieee80211_scan_state *);
 408 static int      iwm_detach(device_t);
 409
 410 /*
 411  * Firmware parser.
 412  */
 413
 414 static int
 415 iwm_store_cscheme(struct iwm_softc *sc, const uint8_t *data, size_t dlen)
 416 {
 417         const struct iwm_fw_cscheme_list *l = (const void *)data;
 418
 419         if (dlen < sizeof(*l) ||
 420             dlen < sizeof(l->size) + l->size * sizeof(*l->cs))
 421                 return EINVAL;
 422
 423         /* we don't actually store anything for now, always use s/w crypto */
 424
 425         return 0;
 426 }
 427
 428 static int
 429 iwm_firmware_store_section(struct iwm_softc *sc,
 430     enum iwm_ucode_type type, const uint8_t *data, size_t dlen)
 431 {
 432         struct iwm_fw_sects *fws;
 433         struct iwm_fw_desc *fwone;
 434
 435         if (type >= IWM_UCODE_TYPE_MAX)
 436                 return EINVAL;
 437         if (dlen < sizeof(uint32_t))
 438                 return EINVAL;
 439
 440         fws = &sc->sc_fw.fw_sects[type];
 441         if (fws->fw_count >= IWM_UCODE_SECTION_MAX)
 442                 return EINVAL;
 443
 444         fwone = &fws->fw_sect[fws->fw_count];
 445
 446         /* first 32bit are device load offset */
 447         memcpy(&fwone->offset, data, sizeof(uint32_t));
 448
 449         /* rest is data */
 450         fwone->data = data + sizeof(uint32_t);
 451         fwone->len = dlen - sizeof(uint32_t);
 452
 453         fws->fw_count++;
 454
 455         return 0;
 456 }
 457
 458 #define IWM_DEFAULT_SCAN_CHANNELS 40
 459
 460 /* iwlwifi: iwl-drv.c */
 461 struct iwm_tlv_calib_data {
 462         uint32_t ucode_type;
 463         struct iwm_tlv_calib_ctrl calib;
 464 } __packed;
 465
 466 static int
 467 iwm_set_default_calib(struct iwm_softc *sc, const void *data)
 468 {
 469         const struct iwm_tlv_calib_data *def_calib = data;
 470         uint32_t ucode_type = le32toh(def_calib->ucode_type);
 471
 472         if (ucode_type >= IWM_UCODE_TYPE_MAX) {
 473                 device_printf(sc->sc_dev,
 474                     "Wrong ucode_type %u for default "
 475                     "calibration.\n", ucode_type);
 476                 return EINVAL;
 477         }
 478
 479         sc->sc_default_calib[ucode_type].flow_trigger =
 480             def_calib->calib.flow_trigger;
 481         sc->sc_default_calib[ucode_type].event_trigger =
 482             def_calib->calib.event_trigger;
 483
 484         return 0;
 485 }
 486
 487 static int
 488 iwm_set_ucode_api_flags(struct iwm_softc *sc, const uint8_t *data,
 489                         struct iwm_ucode_capabilities *capa)
 490 {
 491         const struct iwm_ucode_api *ucode_api = (const void *)data;
 492         uint32_t api_index = le32toh(ucode_api->api_index);
 493         uint32_t api_flags = le32toh(ucode_api->api_flags);
 494         int i;
 495
 496         if (api_index >= howmany(IWM_NUM_UCODE_TLV_API, 32)) {
 497                 device_printf(sc->sc_dev,
 498                     "api flags index %d larger than supported by driver\n",
 499                     api_index);
 500                 /* don't return an error so we can load FW that has more bits */
 501                 return 0;
 502         }
 503
 504         for (i = 0; i < 32; i++) {
 505                 if (api_flags & (1U << i))
 506                         setbit(capa->enabled_api, i + 32 * api_index);
 507         }
 508
 509         return 0;
 510 }
 511
 512 static int
 513 iwm_set_ucode_capabilities(struct iwm_softc *sc, const uint8_t *data,
 514                            struct iwm_ucode_capabilities *capa)
 515 {
 516         const struct iwm_ucode_capa *ucode_capa = (const void *)data;
 517         uint32_t api_index = le32toh(ucode_capa->api_index);
 518         uint32_t api_flags = le32toh(ucode_capa->api_capa);
 519         int i;
 520
 521         if (api_index >= howmany(IWM_NUM_UCODE_TLV_CAPA, 32)) {
 522                 device_printf(sc->sc_dev,
 523                     "capa flags index %d larger than supported by driver\n",
 524                     api_index);
 525                 /* don't return an error so we can load FW that has more bits */
 526                 return 0;
 527         }
 528
 529         for (i = 0; i < 32; i++) {
 530                 if (api_flags & (1U << i))
 531                         setbit(capa->enabled_capa, i + 32 * api_index);
 532         }
 533
 534         return 0;
 535 }
 536
 537 static void
 538 iwm_fw_info_free(struct iwm_fw_info *fw)
 539 {
 540         firmware_put(fw->fw_fp, FIRMWARE_UNLOAD);
 541         fw->fw_fp = NULL;
 542         /* don't touch fw->fw_status */
 543         memset(fw->fw_sects, 0, sizeof(fw->fw_sects));
 544 }
 545
 546 static int
 547 iwm_read_firmware(struct iwm_softc *sc, enum iwm_ucode_type ucode_type)
 548 {
 549         struct iwm_fw_info *fw = &sc->sc_fw;
 550         const struct iwm_tlv_ucode_header *uhdr;
 551         struct iwm_ucode_tlv tlv;
 552         struct iwm_ucode_capabilities *capa = &sc->ucode_capa;
 553         enum iwm_ucode_tlv_type tlv_type;
 554         const struct firmware *fwp;
 555         const uint8_t *data;
 556         uint32_t usniffer_img;
 557         uint32_t paging_mem_size;
 558         int num_of_cpus;
 559         int error = 0;
 560         size_t len;
 561
 562         if (fw->fw_status == IWM_FW_STATUS_DONE &&
 563             ucode_type != IWM_UCODE_INIT)
 564                 return 0;
 565
 566         while (fw->fw_status == IWM_FW_STATUS_INPROGRESS)
 567                 msleep(&sc->sc_fw, &sc->sc_mtx, 0, "iwmfwp", 0);
 568         fw->fw_status = IWM_FW_STATUS_INPROGRESS;
 569
 570         if (fw->fw_fp != NULL)
 571                 iwm_fw_info_free(fw);
 572
 573         /*
 574          * Load firmware into driver memory.
 575          * fw_fp will be set.
 576          */
 577         IWM_UNLOCK(sc);
 578         fwp = firmware_get(sc->cfg->fw_name);
 579         IWM_LOCK(sc);
 580         if (fwp == NULL) {
 581                 device_printf(sc->sc_dev,
 582                     "could not read firmware %s (error %d)\n",
 583                     sc->cfg->fw_name, error);
 584                 goto out;
 585         }
 586         fw->fw_fp = fwp;
 587
 588         /* (Re-)Initialize default values. */
 589         capa->flags = 0;
 590         capa->max_probe_length = IWM_DEFAULT_MAX_PROBE_LENGTH;
 591         capa->n_scan_channels = IWM_DEFAULT_SCAN_CHANNELS;
 592         memset(capa->enabled_capa, 0, sizeof(capa->enabled_capa));
 593         memset(capa->enabled_api, 0, sizeof(capa->enabled_api));
 594         memset(sc->sc_fw_mcc, 0, sizeof(sc->sc_fw_mcc));
 595
 596         /*
 597          * Parse firmware contents
 598          */
 599
 600         uhdr = (const void *)fw->fw_fp->data;
 601         if (*(const uint32_t *)fw->fw_fp->data != 0
 602             || le32toh(uhdr->magic) != IWM_TLV_UCODE_MAGIC) {
 603                 device_printf(sc->sc_dev, "invalid firmware %s\n",
 604                     sc->cfg->fw_name);
 605                 error = EINVAL;
 606                 goto out;
 607         }
 608
 609         snprintf(sc->sc_fwver, sizeof(sc->sc_fwver), "%d.%d (API ver %d)",
 610             IWM_UCODE_MAJOR(le32toh(uhdr->ver)),
 611             IWM_UCODE_MINOR(le32toh(uhdr->ver)),
 612             IWM_UCODE_API(le32toh(uhdr->ver)));
 613         data = uhdr->data;
 614         len = fw->fw_fp->datasize - sizeof(*uhdr);
 615
 616         while (len >= sizeof(tlv)) {
 617                 size_t tlv_len;
 618                 const void *tlv_data;
 619
 620                 memcpy(&tlv, data, sizeof(tlv));
 621                 tlv_len = le32toh(tlv.length);
 622                 tlv_type = le32toh(tlv.type);
 623
 624                 len -= sizeof(tlv);
 625                 data += sizeof(tlv);
 626                 tlv_data = data;
 627
 628                 if (len < tlv_len) {
 629                         device_printf(sc->sc_dev,
 630                             "firmware too short: %zu bytes\n",
 631                             len);
 632                         error = EINVAL;
 633                         goto parse_out;
 634                 }
 635
 636                 switch ((int)tlv_type) {
 637                 case IWM_UCODE_TLV_PROBE_MAX_LEN:
 638                         if (tlv_len < sizeof(uint32_t)) {
 639                                 device_printf(sc->sc_dev,
 640                                     "%s: PROBE_MAX_LEN (%d) < sizeof(uint32_t)\n",
 641                                     __func__,
 642                                     (int) tlv_len);
 643                                 error = EINVAL;
 644                                 goto parse_out;
 645                         }
 646                         capa->max_probe_length =
 647                             le32toh(*(const uint32_t *)tlv_data);
 648                         /* limit it to something sensible */
 649                         if (capa->max_probe_length >
 650                             IWM_SCAN_OFFLOAD_PROBE_REQ_SIZE) {
 651                                 IWM_DPRINTF(sc, IWM_DEBUG_FIRMWARE_TLV,
 652                                     "%s: IWM_UCODE_TLV_PROBE_MAX_LEN "
 653                                     "ridiculous\n", __func__);
 654                                 error = EINVAL;
 655                                 goto parse_out;
 656                         }
 657                         break;
 658                 case IWM_UCODE_TLV_PAN:
 659                         if (tlv_len) {
 660                                 device_printf(sc->sc_dev,
 661                                     "%s: IWM_UCODE_TLV_PAN: tlv_len (%d) > 0\n",
 662                                     __func__,
 663                                     (int) tlv_len);
 664                                 error = EINVAL;
 665                                 goto parse_out;
 666                         }
 667                         capa->flags |= IWM_UCODE_TLV_FLAGS_PAN;
 668                         break;
 669                 case IWM_UCODE_TLV_FLAGS:
 670                         if (tlv_len < sizeof(uint32_t)) {
 671                                 device_printf(sc->sc_dev,
 672                                     "%s: IWM_UCODE_TLV_FLAGS: tlv_len (%d) < sizeof(uint32_t)\n",
 673                                     __func__,
 674                                     (int) tlv_len);
 675                                 error = EINVAL;
 676                                 goto parse_out;
 677                         }
 678                         /*
 679                          * Apparently there can be many flags, but Linux driver
 680                          * parses only the first one, and so do we.
 681                          *
 682                          * XXX: why does this override IWM_UCODE_TLV_PAN?
 683                          * Intentional or a bug?  Observations from
 684                          * current firmware file:
 685                          *  1) TLV_PAN is parsed first
 686                          *  2) TLV_FLAGS contains TLV_FLAGS_PAN
 687                          * ==> this resets TLV_PAN to itself... hnnnk
 688                          */
 689                         capa->flags = le32toh(*(const uint32_t *)tlv_data);
 690                         break;
 691                 case IWM_UCODE_TLV_CSCHEME:
 692                         if ((error = iwm_store_cscheme(sc,
 693                             tlv_data, tlv_len)) != 0) {
 694                                 device_printf(sc->sc_dev,
 695                                     "%s: iwm_store_cscheme(): returned %d\n",
 696                                     __func__,
 697                                     error);
 698                                 goto parse_out;
 699                         }
 700                         break;
 701                 case IWM_UCODE_TLV_NUM_OF_CPU:
 702                         if (tlv_len != sizeof(uint32_t)) {
 703                                 device_printf(sc->sc_dev,
 704                                     "%s: IWM_UCODE_TLV_NUM_OF_CPU: tlv_len (%d) != sizeof(uint32_t)\n",
 705                                     __func__,
 706                                     (int) tlv_len);
 707                                 error = EINVAL;
 708                                 goto parse_out;
 709                         }
 710                         num_of_cpus = le32toh(*(const uint32_t *)tlv_data);
 711                         if (num_of_cpus == 2) {
 712                                 fw->fw_sects[IWM_UCODE_REGULAR].is_dual_cpus =
 713                                         TRUE;
 714                                 fw->fw_sects[IWM_UCODE_INIT].is_dual_cpus =
 715                                         TRUE;
 716                                 fw->fw_sects[IWM_UCODE_WOWLAN].is_dual_cpus =
 717                                         TRUE;
 718                         } else if ((num_of_cpus > 2) || (num_of_cpus < 1)) {
 719                                 device_printf(sc->sc_dev,
 720                                     "%s: Driver supports only 1 or 2 CPUs\n",
 721                                     __func__);
 722                                 error = EINVAL;
 723                                 goto parse_out;
 724                         }
 725                         break;
 726                 case IWM_UCODE_TLV_SEC_RT:
 727                         if ((error = iwm_firmware_store_section(sc,
 728                             IWM_UCODE_REGULAR, tlv_data, tlv_len)) != 0) {
 729                                 device_printf(sc->sc_dev,
 730                                     "%s: IWM_UCODE_REGULAR: iwm_firmware_store_section() failed; %d\n",
 731                                     __func__,
 732                                     error);
 733                                 goto parse_out;
 734                         }
 735                         break;
 736                 case IWM_UCODE_TLV_SEC_INIT:
 737                         if ((error = iwm_firmware_store_section(sc,
 738                             IWM_UCODE_INIT, tlv_data, tlv_len)) != 0) {
 739                                 device_printf(sc->sc_dev,
 740                                     "%s: IWM_UCODE_INIT: iwm_firmware_store_section() failed; %d\n",
 741                                     __func__,
 742                                     error);
 743                                 goto parse_out;
 744                         }
 745                         break;
 746                 case IWM_UCODE_TLV_SEC_WOWLAN:
 747                         if ((error = iwm_firmware_store_section(sc,
 748                             IWM_UCODE_WOWLAN, tlv_data, tlv_len)) != 0) {
 749                                 device_printf(sc->sc_dev,
 750                                     "%s: IWM_UCODE_WOWLAN: iwm_firmware_store_section() failed; %d\n",
 751                                     __func__,
 752                                     error);
 753                                 goto parse_out;
 754                         }
 755                         break;
 756                 case IWM_UCODE_TLV_DEF_CALIB:
 757                         if (tlv_len != sizeof(struct iwm_tlv_calib_data)) {
 758                                 device_printf(sc->sc_dev,
 759                                     "%s: IWM_UCODE_TLV_DEV_CALIB: tlv_len (%d) < sizeof(iwm_tlv_calib_data) (%d)\n",
 760                                     __func__,
 761                                     (int) tlv_len,
 762                                     (int) sizeof(struct iwm_tlv_calib_data));
 763                                 error = EINVAL;
 764                                 goto parse_out;
 765                         }
 766                         if ((error = iwm_set_default_calib(sc, tlv_data)) != 0) {
 767                                 device_printf(sc->sc_dev,
 768                                     "%s: iwm_set_default_calib() failed: %d\n",
 769                                     __func__,
 770                                     error);
 771                                 goto parse_out;
 772                         }
 773                         break;
 774                 case IWM_UCODE_TLV_PHY_SKU:
 775                         if (tlv_len != sizeof(uint32_t)) {
 776                                 error = EINVAL;
 777                                 device_printf(sc->sc_dev,
 778                                     "%s: IWM_UCODE_TLV_PHY_SKU: tlv_len (%d) < sizeof(uint32_t)\n",
 779                                     __func__,
 780                                     (int) tlv_len);
 781                                 goto parse_out;
 782                         }
 783                         sc->sc_fw.phy_config =
 784                             le32toh(*(const uint32_t *)tlv_data);
 785                         sc->sc_fw.valid_tx_ant = (sc->sc_fw.phy_config &
 786                                                   IWM_FW_PHY_CFG_TX_CHAIN) >>
 787                                                   IWM_FW_PHY_CFG_TX_CHAIN_POS;
 788                         sc->sc_fw.valid_rx_ant = (sc->sc_fw.phy_config &
 789                                                   IWM_FW_PHY_CFG_RX_CHAIN) >>
 790                                                   IWM_FW_PHY_CFG_RX_CHAIN_POS;
 791                         break;
 792
 793                 case IWM_UCODE_TLV_API_CHANGES_SET: {
 794                         if (tlv_len != sizeof(struct iwm_ucode_api)) {
 795                                 error = EINVAL;
 796                                 goto parse_out;
 797                         }
 798                         if (iwm_set_ucode_api_flags(sc, tlv_data, capa)) {
 799                                 error = EINVAL;
 800                                 goto parse_out;
 801                         }
 802                         break;
 803                 }
 804
 805                 case IWM_UCODE_TLV_ENABLED_CAPABILITIES: {
 806                         if (tlv_len != sizeof(struct iwm_ucode_capa)) {
 807                                 error = EINVAL;
 808                                 goto parse_out;
 809                         }
 810                         if (iwm_set_ucode_capabilities(sc, tlv_data, capa)) {
 811                                 error = EINVAL;
 812                                 goto parse_out;
 813                         }
 814                         break;
 815                 }
 816
 817                 case 48: /* undocumented TLV */
 818                 case IWM_UCODE_TLV_SDIO_ADMA_ADDR:
 819                 case IWM_UCODE_TLV_FW_GSCAN_CAPA:
 820                         /* ignore, not used by current driver */
 821                         break;
 822
 823                 case IWM_UCODE_TLV_SEC_RT_USNIFFER:
 824                         if ((error = iwm_firmware_store_section(sc,
 825                             IWM_UCODE_REGULAR_USNIFFER, tlv_data,
 826                             tlv_len)) != 0)
 827                                 goto parse_out;
 828                         break;
 829
 830                 case IWM_UCODE_TLV_PAGING:
 831                         if (tlv_len != sizeof(uint32_t)) {
 832                                 error = EINVAL;
 833                                 goto parse_out;
 834                         }
 835                         paging_mem_size = le32toh(*(const uint32_t *)tlv_data);
 836
 837                         IWM_DPRINTF(sc, IWM_DEBUG_FIRMWARE_TLV,
 838                             "%s: Paging: paging enabled (size = %u bytes)\n",
 839                             __func__, paging_mem_size);
 840                         if (paging_mem_size > IWM_MAX_PAGING_IMAGE_SIZE) {
 841                                 device_printf(sc->sc_dev,
 842                                         "%s: Paging: driver supports up to %u bytes for paging image\n",
 843                                         __func__, IWM_MAX_PAGING_IMAGE_SIZE);
 844                                 error = EINVAL;
 845                                 goto out;
 846                         }
 847                         if (paging_mem_size & (IWM_FW_PAGING_SIZE - 1)) {
 848                                 device_printf(sc->sc_dev,
 849                                     "%s: Paging: image isn't multiple %u\n",
 850                                     __func__, IWM_FW_PAGING_SIZE);
 851                                 error = EINVAL;
 852                                 goto out;
 853                         }
 854
 855                         sc->sc_fw.fw_sects[IWM_UCODE_REGULAR].paging_mem_size =
 856                             paging_mem_size;
 857                         usniffer_img = IWM_UCODE_REGULAR_USNIFFER;
 858                         sc->sc_fw.fw_sects[usniffer_img].paging_mem_size =
 859                             paging_mem_size;
 860                         break;
 861
 862                 case IWM_UCODE_TLV_N_SCAN_CHANNELS:
 863                         if (tlv_len != sizeof(uint32_t)) {
 864                                 error = EINVAL;
 865                                 goto parse_out;
 866                         }
 867                         capa->n_scan_channels =
 868                             le32toh(*(const uint32_t *)tlv_data);
 869                         break;
 870
 871                 case IWM_UCODE_TLV_FW_VERSION:
 872                         if (tlv_len != sizeof(uint32_t) * 3) {
 873                                 error = EINVAL;
 874                                 goto parse_out;
 875                         }
 876                         snprintf(sc->sc_fwver, sizeof(sc->sc_fwver),
 877                             "%d.%d.%d",
 878                             le32toh(((const uint32_t *)tlv_data)[0]),
 879                             le32toh(((const uint32_t *)tlv_data)[1]),
 880                             le32toh(((const uint32_t *)tlv_data)[2]));
 881                         break;
 882
 883                 case IWM_UCODE_TLV_FW_MEM_SEG:
 884                         break;
 885
 886                 default:
 887                         device_printf(sc->sc_dev,
 888                             "%s: unknown firmware section %d, abort\n",
 889                             __func__, tlv_type);
 890                         error = EINVAL;
 891                         goto parse_out;
 892                 }
 893
 894                 len -= roundup(tlv_len, 4);
 895                 data += roundup(tlv_len, 4);
 896         }
 897
 898         KASSERT(error == 0, ("unhandled error"));
 899
 900  parse_out:
 901         if (error) {
 902                 device_printf(sc->sc_dev, "firmware parse error %d, "
 903                     "section type %d\n", error, tlv_type);
 904         }
 905
 906  out:
 907         if (error) {
 908                 fw->fw_status = IWM_FW_STATUS_NONE;
 909                 if (fw->fw_fp != NULL)
 910                         iwm_fw_info_free(fw);
 911         } else
 912                 fw->fw_status = IWM_FW_STATUS_DONE;
 913         wakeup(&sc->sc_fw);
 914
 915         return error;
 916 }
 917
 918 /*
 919  * DMA resource routines
 920  */
 921
 922 /* fwmem is used to load firmware onto the card */
 923 static int
 924 iwm_alloc_fwmem(struct iwm_softc *sc)
 925 {
 926         /* Must be aligned on a 16-byte boundary. */
 927         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->fw_dma,
 928             IWM_FH_MEM_TB_MAX_LENGTH, 16);
 929 }
 930
 931 /* tx scheduler rings.  not used? */
 932 static int
 933 iwm_alloc_sched(struct iwm_softc *sc)
 934 {
 935         /* TX scheduler rings must be aligned on a 1KB boundary. */
 936         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->sched_dma,
 937             nitems(sc->txq) * sizeof(struct iwm_agn_scd_bc_tbl), 1024);
 938 }
 939
 940 /* keep-warm page is used internally by the card.  see iwl-fh.h for more info */
 941 static int
 942 iwm_alloc_kw(struct iwm_softc *sc)
 943 {
 944         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->kw_dma, 4096, 4096);
 945 }
 946
 947 /* interrupt cause table */
 948 static int
 949 iwm_alloc_ict(struct iwm_softc *sc)
 950 {
 951         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->ict_dma,
 952             IWM_ICT_SIZE, 1<<IWM_ICT_PADDR_SHIFT);
 953 }
 954
 955 static int
 956 iwm_alloc_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring)
 957 {
 958         bus_size_t size;
 959         int i, error;
 960
 961         ring->cur = 0;
 962
 963         /* Allocate RX descriptors (256-byte aligned). */
 964         size = IWM_RX_RING_COUNT * sizeof(uint32_t);
 965         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->desc_dma, size, 256);
 966         if (error != 0) {
 967                 device_printf(sc->sc_dev,
 968                     "could not allocate RX ring DMA memory\n");
 969                 goto fail;
 970         }
 971         ring->desc = ring->desc_dma.vaddr;
 972
 973         /* Allocate RX status area (16-byte aligned). */
 974         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->stat_dma,
 975             sizeof(*ring->stat), 16);
 976         if (error != 0) {
 977                 device_printf(sc->sc_dev,
 978                     "could not allocate RX status DMA memory\n");
 979                 goto fail;
 980         }
 981         ring->stat = ring->stat_dma.vaddr;
 982
 983         /* Create RX buffer DMA tag. */
 984         error = bus_dma_tag_create(sc->sc_dmat, 1, 0,
 985             BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
 986             IWM_RBUF_SIZE, 1, IWM_RBUF_SIZE, 0, NULL, NULL, &ring->data_dmat);
 987         if (error != 0) {
 988                 device_printf(sc->sc_dev,
 989                     "%s: could not create RX buf DMA tag, error %d\n",
 990                     __func__, error);
 991                 goto fail;
 992         }
 993
 994         /* Allocate spare bus_dmamap_t for iwm_rx_addbuf() */
 995         error = bus_dmamap_create(ring->data_dmat, 0, &ring->spare_map);
 996         if (error != 0) {
 997                 device_printf(sc->sc_dev,
 998                     "%s: could not create RX buf DMA map, error %d\n",
 999                     __func__, error);
1000                 goto fail;
1001         }
1002         /*
1003          * Allocate and map RX buffers.
1004          */
1005         for (i = 0; i < IWM_RX_RING_COUNT; i++) {
1006                 struct iwm_rx_data *data = &ring->data[i];
1007                 error = bus_dmamap_create(ring->data_dmat, 0, &data->map);
1008                 if (error != 0) {
1009                         device_printf(sc->sc_dev,
1010                             "%s: could not create RX buf DMA map, error %d\n",
1011                             __func__, error);
1012                         goto fail;
1013                 }
1014                 data->m = NULL;
1015
1016                 if ((error = iwm_rx_addbuf(sc, IWM_RBUF_SIZE, i)) != 0) {
1017                         goto fail;
1018                 }
1019         }
1020         return 0;
1021
1022 fail:   iwm_free_rx_ring(sc, ring);
1023         return error;
1024 }
1025
1026 static void
1027 iwm_reset_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring)
1028 {
1029         /* Reset the ring state */
1030         ring->cur = 0;
1031
1032         /*
1033          * The hw rx ring index in shared memory must also be cleared,
1034          * otherwise the discrepancy can cause reprocessing chaos.
1035          */
1036         memset(sc->rxq.stat, 0, sizeof(*sc->rxq.stat));
1037 }
1038
1039 static void
1040 iwm_free_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring)
1041 {
1042         int i;
1043
1044         iwm_dma_contig_free(&ring->desc_dma);
1045         iwm_dma_contig_free(&ring->stat_dma);
1046
1047         for (i = 0; i < IWM_RX_RING_COUNT; i++) {
1048                 struct iwm_rx_data *data = &ring->data[i];
1049
1050                 if (data->m != NULL) {
1051                         bus_dmamap_sync(ring->data_dmat, data->map,
1052                             BUS_DMASYNC_POSTREAD);
1053                         bus_dmamap_unload(ring->data_dmat, data->map);
1054                         m_freem(data->m);
1055                         data->m = NULL;
1056                 }
1057                 if (data->map != NULL) {
1058                         bus_dmamap_destroy(ring->data_dmat, data->map);
1059                         data->map = NULL;
1060                 }
1061         }
1062         if (ring->spare_map != NULL) {
1063                 bus_dmamap_destroy(ring->data_dmat, ring->spare_map);
1064                 ring->spare_map = NULL;
1065         }
1066         if (ring->data_dmat != NULL) {
1067                 bus_dma_tag_destroy(ring->data_dmat);
1068                 ring->data_dmat = NULL;
1069         }
1070 }
1071
1072 static int
1073 iwm_alloc_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring, int qid)
1074 {
1075         bus_addr_t paddr;
1076         bus_size_t size;
1077         size_t maxsize;
1078         int nsegments;
1079         int i, error;
1080
1081         ring->qid = qid;
1082         ring->queued = 0;
1083         ring->cur = 0;
1084
1085         /* Allocate TX descriptors (256-byte aligned). */
1086         size = IWM_TX_RING_COUNT * sizeof (struct iwm_tfd);
1087         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->desc_dma, size, 256);
1088         if (error != 0) {
1089                 device_printf(sc->sc_dev,
1090                     "could not allocate TX ring DMA memory\n");
1091                 goto fail;
1092         }
1093         ring->desc = ring->desc_dma.vaddr;
1094
1095         /*
1096          * We only use rings 0 through 9 (4 EDCA + cmd) so there is no need
1097          * to allocate commands space for other rings.
1098          */
1099         if (qid > IWM_MVM_CMD_QUEUE)
1100                 return 0;
1101
1102         size = IWM_TX_RING_COUNT * sizeof(struct iwm_device_cmd);
1103         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->cmd_dma, size, 4);
1104         if (error != 0) {
1105                 device_printf(sc->sc_dev,
1106                     "could not allocate TX cmd DMA memory\n");
1107                 goto fail;
1108         }
1109         ring->cmd = ring->cmd_dma.vaddr;
1110
1111         /* FW commands may require more mapped space than packets. */
1112         if (qid == IWM_MVM_CMD_QUEUE) {
1113                 maxsize = IWM_RBUF_SIZE;
1114                 nsegments = 1;
1115         } else {
1116                 maxsize = MCLBYTES;
1117                 nsegments = IWM_MAX_SCATTER - 2;
1118         }
1119
1120         error = bus_dma_tag_create(sc->sc_dmat, 1, 0,
1121             BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, maxsize,
1122             nsegments, maxsize, 0, NULL, NULL, &ring->data_dmat);
1123         if (error != 0) {
1124                 device_printf(sc->sc_dev, "could not create TX buf DMA tag\n");
1125                 goto fail;
1126         }
1127
1128         paddr = ring->cmd_dma.paddr;
1129         for (i = 0; i < IWM_TX_RING_COUNT; i++) {
1130                 struct iwm_tx_data *data = &ring->data[i];
1131
1132                 data->cmd_paddr = paddr;
1133                 data->scratch_paddr = paddr + sizeof(struct iwm_cmd_header)
1134                     + offsetof(struct iwm_tx_cmd, scratch);
1135                 paddr += sizeof(struct iwm_device_cmd);
1136
1137                 error = bus_dmamap_create(ring->data_dmat, 0, &data->map);
1138                 if (error != 0) {
1139                         device_printf(sc->sc_dev,
1140                             "could not create TX buf DMA map\n");
1141                         goto fail;
1142                 }
1143         }
1144         KASSERT(paddr == ring->cmd_dma.paddr + size,
1145             ("invalid physical address"));
1146         return 0;
1147
1148 fail:   iwm_free_tx_ring(sc, ring);
1149         return error;
1150 }
1151
1152 static void
1153 iwm_reset_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring)
1154 {
1155         int i;
1156
1157         for (i = 0; i < IWM_TX_RING_COUNT; i++) {
1158                 struct iwm_tx_data *data = &ring->data[i];
1159
1160                 if (data->m != NULL) {
1161                         bus_dmamap_sync(ring->data_dmat, data->map,
1162                             BUS_DMASYNC_POSTWRITE);
1163                         bus_dmamap_unload(ring->data_dmat, data->map);
1164                         m_freem(data->m);
1165                         data->m = NULL;
1166                 }
1167         }
1168         /* Clear TX descriptors. */
1169         memset(ring->desc, 0, ring->desc_dma.size);
1170         bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map,
1171             BUS_DMASYNC_PREWRITE);
1172         sc->qfullmsk &= ~(1 << ring->qid);
1173         ring->queued = 0;
1174         ring->cur = 0;
1175
1176         if (ring->qid == IWM_MVM_CMD_QUEUE && sc->cmd_hold_nic_awake)
1177                 iwm_pcie_clear_cmd_in_flight(sc);
1178 }
1179
1180 static void
1181 iwm_free_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring)
1182 {
1183         int i;
1184
1185         iwm_dma_contig_free(&ring->desc_dma);
1186         iwm_dma_contig_free(&ring->cmd_dma);
1187
1188         for (i = 0; i < IWM_TX_RING_COUNT; i++) {
1189                 struct iwm_tx_data *data = &ring->data[i];
1190
1191                 if (data->m != NULL) {
1192                         bus_dmamap_sync(ring->data_dmat, data->map,
1193                             BUS_DMASYNC_POSTWRITE);
1194                         bus_dmamap_unload(ring->data_dmat, data->map);
1195                         m_freem(data->m);
1196                         data->m = NULL;
1197                 }
1198                 if (data->map != NULL) {
1199                         bus_dmamap_destroy(ring->data_dmat, data->map);
1200                         data->map = NULL;
1201                 }
1202         }
1203         if (ring->data_dmat != NULL) {
1204                 bus_dma_tag_destroy(ring->data_dmat);
1205                 ring->data_dmat = NULL;
1206         }
1207 }
1208
1209 /*
1210  * High-level hardware frobbing routines
1211  */
1212
1213 static void
1214 iwm_enable_interrupts(struct iwm_softc *sc)
1215 {
1216         sc->sc_intmask = IWM_CSR_INI_SET_MASK;
1217         IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask);
1218 }
1219
1220 static void
1221 iwm_restore_interrupts(struct iwm_softc *sc)
1222 {
1223         IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask);
1224 }
1225
1226 static void
1227 iwm_disable_interrupts(struct iwm_softc *sc)
1228 {
1229         /* disable interrupts */
1230         IWM_WRITE(sc, IWM_CSR_INT_MASK, 0);
1231
1232         /* acknowledge all interrupts */
1233         IWM_WRITE(sc, IWM_CSR_INT, ~0);
1234         IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, ~0);
1235 }
1236
1237 static void
1238 iwm_ict_reset(struct iwm_softc *sc)
1239 {
1240         iwm_disable_interrupts(sc);
1241
1242         /* Reset ICT table. */
1243         memset(sc->ict_dma.vaddr, 0, IWM_ICT_SIZE);
1244         sc->ict_cur = 0;
1245
1246         /* Set physical address of ICT table (4KB aligned). */
1247         IWM_WRITE(sc, IWM_CSR_DRAM_INT_TBL_REG,
1248             IWM_CSR_DRAM_INT_TBL_ENABLE
1249             | IWM_CSR_DRAM_INIT_TBL_WRITE_POINTER
1250             | IWM_CSR_DRAM_INIT_TBL_WRAP_CHECK
1251             | sc->ict_dma.paddr >> IWM_ICT_PADDR_SHIFT);
1252
1253         /* Switch to ICT interrupt mode in driver. */
1254         sc->sc_flags |= IWM_FLAG_USE_ICT;
1255
1256         /* Re-enable interrupts. */
1257         IWM_WRITE(sc, IWM_CSR_INT, ~0);
1258         iwm_enable_interrupts(sc);
1259 }
1260
1261 /* iwlwifi pcie/trans.c */
1262
1263 /*
1264  * Since this .. hard-resets things, it's time to actually
1265  * mark the first vap (if any) as having no mac context.
1266  * It's annoying, but since the driver is potentially being
1267  * stop/start'ed whilst active (thanks openbsd port!) we
1268  * have to correctly track this.
1269  */
1270 static void
1271 iwm_stop_device(struct iwm_softc *sc)
1272 {
1273         struct ieee80211com *ic = &sc->sc_ic;
1274         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
1275         int chnl, qid;
1276         uint32_t mask = 0;
1277
1278         /* tell the device to stop sending interrupts */
1279         iwm_disable_interrupts(sc);
1280
1281         /*
1282          * FreeBSD-local: mark the first vap as not-uploaded,
1283          * so the next transition through auth/assoc
1284          * will correctly populate the MAC context.
1285          */
1286         if (vap) {
1287                 struct iwm_vap *iv = IWM_VAP(vap);
1288                 iv->is_uploaded = 0;
1289         }
1290
1291         /* device going down, Stop using ICT table */
1292         sc->sc_flags &= ~IWM_FLAG_USE_ICT;
1293
1294         /* stop tx and rx.  tx and rx bits, as usual, are from if_iwn */
1295
1296         iwm_write_prph(sc, IWM_SCD_TXFACT, 0);
1297
1298         if (iwm_nic_lock(sc)) {
1299                 /* Stop each Tx DMA channel */
1300                 for (chnl = 0; chnl < IWM_FH_TCSR_CHNL_NUM; chnl++) {
1301                         IWM_WRITE(sc,
1302                             IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl), 0);
1303                         mask |= IWM_FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(chnl);
1304                 }
1305
1306                 /* Wait for DMA channels to be idle */
1307                 if (!iwm_poll_bit(sc, IWM_FH_TSSR_TX_STATUS_REG, mask, mask,
1308                     5000)) {
1309                         device_printf(sc->sc_dev,
1310                             "Failing on timeout while stopping DMA channel: [0x%08x]\n",
1311                             IWM_READ(sc, IWM_FH_TSSR_TX_STATUS_REG));
1312                 }
1313                 iwm_nic_unlock(sc);
1314         }
1315         iwm_pcie_rx_stop(sc);
1316
1317         /* Stop RX ring. */
1318         iwm_reset_rx_ring(sc, &sc->rxq);
1319
1320         /* Reset all TX rings. */
1321         for (qid = 0; qid < nitems(sc->txq); qid++)
1322                 iwm_reset_tx_ring(sc, &sc->txq[qid]);
1323
1324         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
1325                 /* Power-down device's busmaster DMA clocks */
1326                 iwm_write_prph(sc, IWM_APMG_CLK_DIS_REG,
1327                     IWM_APMG_CLK_VAL_DMA_CLK_RQT);
1328                 DELAY(5);
1329         }
1330
1331         /* Make sure (redundant) we've released our request to stay awake */
1332         IWM_CLRBITS(sc, IWM_CSR_GP_CNTRL,
1333             IWM_CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1334
1335         /* Stop the device, and put it in low power state */
1336         iwm_apm_stop(sc);
1337
1338         /* Upon stop, the APM issues an interrupt if HW RF kill is set.
1339          * Clean again the interrupt here
1340          */
1341         iwm_disable_interrupts(sc);
1342         /* stop and reset the on-board processor */
1343         IWM_WRITE(sc, IWM_CSR_RESET, IWM_CSR_RESET_REG_FLAG_SW_RESET);
1344
1345         /*
1346          * Even if we stop the HW, we still want the RF kill
1347          * interrupt
1348          */
1349         iwm_enable_rfkill_int(sc);
1350         iwm_check_rfkill(sc);
1351 }
1352
1353 /* iwlwifi: mvm/ops.c */
1354 static void
1355 iwm_mvm_nic_config(struct iwm_softc *sc)
1356 {
1357         uint8_t radio_cfg_type, radio_cfg_step, radio_cfg_dash;
1358         uint32_t reg_val = 0;
1359         uint32_t phy_config = iwm_mvm_get_phy_config(sc);
1360
1361         radio_cfg_type = (phy_config & IWM_FW_PHY_CFG_RADIO_TYPE) >>
1362             IWM_FW_PHY_CFG_RADIO_TYPE_POS;
1363         radio_cfg_step = (phy_config & IWM_FW_PHY_CFG_RADIO_STEP) >>
1364             IWM_FW_PHY_CFG_RADIO_STEP_POS;
1365         radio_cfg_dash = (phy_config & IWM_FW_PHY_CFG_RADIO_DASH) >>
1366             IWM_FW_PHY_CFG_RADIO_DASH_POS;
1367
1368         /* SKU control */
1369         reg_val |= IWM_CSR_HW_REV_STEP(sc->sc_hw_rev) <<
1370             IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_STEP;
1371         reg_val |= IWM_CSR_HW_REV_DASH(sc->sc_hw_rev) <<
1372             IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_DASH;
1373
1374         /* radio configuration */
1375         reg_val |= radio_cfg_type << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_TYPE;
1376         reg_val |= radio_cfg_step << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_STEP;
1377         reg_val |= radio_cfg_dash << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_DASH;
1378
1379         IWM_WRITE(sc, IWM_CSR_HW_IF_CONFIG_REG, reg_val);
1380
1381         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
1382             "Radio type=0x%x-0x%x-0x%x\n", radio_cfg_type,
1383             radio_cfg_step, radio_cfg_dash);
1384
1385         /*
1386          * W/A : NIC is stuck in a reset state after Early PCIe power off
1387          * (PCIe power is lost before PERST# is asserted), causing ME FW
1388          * to lose ownership and not being able to obtain it back.
1389          */
1390         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
1391                 iwm_set_bits_mask_prph(sc, IWM_APMG_PS_CTRL_REG,
1392                     IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS,
1393                     ~IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS);
1394         }
1395 }
1396
1397 static int
1398 iwm_nic_rx_init(struct iwm_softc *sc)
1399 {
1400         /*
1401          * Initialize RX ring.  This is from the iwn driver.
1402          */
1403         memset(sc->rxq.stat, 0, sizeof(*sc->rxq.stat));
1404
1405         /* Stop Rx DMA */
1406         iwm_pcie_rx_stop(sc);
1407
1408         if (!iwm_nic_lock(sc))
1409                 return EBUSY;
1410
1411         /* reset and flush pointers */
1412         IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_RBDCB_WPTR, 0);
1413         IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_FLUSH_RB_REQ, 0);
1414         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_RDPTR, 0);
1415         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_RBDCB_WPTR_REG, 0);
1416
1417         /* Set physical address of RX ring (256-byte aligned). */
1418         IWM_WRITE(sc,
1419             IWM_FH_RSCSR_CHNL0_RBDCB_BASE_REG, sc->rxq.desc_dma.paddr >> 8);
1420
1421         /* Set physical address of RX status (16-byte aligned). */
1422         IWM_WRITE(sc,
1423             IWM_FH_RSCSR_CHNL0_STTS_WPTR_REG, sc->rxq.stat_dma.paddr >> 4);
1424
1425         /* Enable RX. */
1426         IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_CONFIG_REG,
1427             IWM_FH_RCSR_RX_CONFIG_CHNL_EN_ENABLE_VAL            |
1428             IWM_FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY               |  /* HW bug */
1429             IWM_FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_INT_HOST_VAL   |
1430             IWM_FH_RCSR_CHNL0_RX_CONFIG_SINGLE_FRAME_MSK        |
1431             (IWM_RX_RB_TIMEOUT << IWM_FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS) |
1432             IWM_FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K            |
1433             IWM_RX_QUEUE_SIZE_LOG << IWM_FH_RCSR_RX_CONFIG_RBDCB_SIZE_POS);
1434
1435         IWM_WRITE_1(sc, IWM_CSR_INT_COALESCING, IWM_HOST_INT_TIMEOUT_DEF);
1436
1437         /* W/A for interrupt coalescing bug in 7260 and 3160 */
1438         if (sc->cfg->host_interrupt_operation_mode)
1439                 IWM_SETBITS(sc, IWM_CSR_INT_COALESCING, IWM_HOST_INT_OPER_MODE);
1440
1441         /*
1442          * Thus sayeth el jefe (iwlwifi) via a comment:
1443          *
1444          * This value should initially be 0 (before preparing any
1445          * RBs), should be 8 after preparing the first 8 RBs (for example)
1446          */
1447         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_WPTR, 8);
1448
1449         iwm_nic_unlock(sc);
1450
1451         return 0;
1452 }
1453
1454 static int
1455 iwm_nic_tx_init(struct iwm_softc *sc)
1456 {
1457         int qid;
1458
1459         if (!iwm_nic_lock(sc))
1460                 return EBUSY;
1461
1462         /* Deactivate TX scheduler. */
1463         iwm_write_prph(sc, IWM_SCD_TXFACT, 0);
1464
1465         /* Set physical address of "keep warm" page (16-byte aligned). */
1466         IWM_WRITE(sc, IWM_FH_KW_MEM_ADDR_REG, sc->kw_dma.paddr >> 4);
1467
1468         /* Initialize TX rings. */
1469         for (qid = 0; qid < nitems(sc->txq); qid++) {
1470                 struct iwm_tx_ring *txq = &sc->txq[qid];
1471
1472                 /* Set physical address of TX ring (256-byte aligned). */
1473                 IWM_WRITE(sc, IWM_FH_MEM_CBBC_QUEUE(qid),
1474                     txq->desc_dma.paddr >> 8);
1475                 IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
1476                     "%s: loading ring %d descriptors (%p) at %lx\n",
1477                     __func__,
1478                     qid, txq->desc,
1479                     (unsigned long) (txq->desc_dma.paddr >> 8));
1480         }
1481
1482         iwm_write_prph(sc, IWM_SCD_GP_CTRL, IWM_SCD_GP_CTRL_AUTO_ACTIVE_MODE);
1483
1484         iwm_nic_unlock(sc);
1485
1486         return 0;
1487 }
1488
1489 static int
1490 iwm_nic_init(struct iwm_softc *sc)
1491 {
1492         int error;
1493
1494         iwm_apm_init(sc);
1495         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
1496                 iwm_set_pwr(sc);
1497
1498         iwm_mvm_nic_config(sc);
1499
1500         if ((error = iwm_nic_rx_init(sc)) != 0)
1501                 return error;
1502
1503         /*
1504          * Ditto for TX, from iwn
1505          */
1506         if ((error = iwm_nic_tx_init(sc)) != 0)
1507                 return error;
1508
1509         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
1510             "%s: shadow registers enabled\n", __func__);
1511         IWM_SETBITS(sc, IWM_CSR_MAC_SHADOW_REG_CTRL, 0x800fffff);
1512
1513         return 0;
1514 }
1515
1516 const uint8_t iwm_mvm_ac_to_tx_fifo[] = {
1517         IWM_MVM_TX_FIFO_VO,
1518         IWM_MVM_TX_FIFO_VI,
1519         IWM_MVM_TX_FIFO_BE,
1520         IWM_MVM_TX_FIFO_BK,
1521 };
1522
1523 static int
1524 iwm_enable_txq(struct iwm_softc *sc, int sta_id, int qid, int fifo)
1525 {
1526         if (!iwm_nic_lock(sc)) {
1527                 device_printf(sc->sc_dev,
1528                     "%s: cannot enable txq %d\n",
1529                     __func__,
1530                     qid);
1531                 return EBUSY;
1532         }
1533
1534         IWM_WRITE(sc, IWM_HBUS_TARG_WRPTR, qid << 8 | 0);
1535
1536         if (qid == IWM_MVM_CMD_QUEUE) {
1537                 /* unactivate before configuration */
1538                 iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid),
1539                     (0 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE)
1540                     | (1 << IWM_SCD_QUEUE_STTS_REG_POS_SCD_ACT_EN));
1541
1542                 iwm_nic_unlock(sc);
1543
1544                 iwm_clear_bits_prph(sc, IWM_SCD_AGGR_SEL, (1 << qid));
1545
1546                 if (!iwm_nic_lock(sc)) {
1547                         device_printf(sc->sc_dev,
1548                             "%s: cannot enable txq %d\n", __func__, qid);
1549                         return EBUSY;
1550                 }
1551                 iwm_write_prph(sc, IWM_SCD_QUEUE_RDPTR(qid), 0);
1552                 iwm_nic_unlock(sc);
1553
1554                 iwm_write_mem32(sc, sc->scd_base_addr + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid), 0);
1555                 /* Set scheduler window size and frame limit. */
1556                 iwm_write_mem32(sc,
1557                     sc->scd_base_addr + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid) +
1558                     sizeof(uint32_t),
1559                     ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_POS) &
1560                     IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_MSK) |
1561                     ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_POS) &
1562                     IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_MSK));
1563
1564                 if (!iwm_nic_lock(sc)) {
1565                         device_printf(sc->sc_dev,
1566                             "%s: cannot enable txq %d\n", __func__, qid);
1567                         return EBUSY;
1568                 }
1569                 iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid),
1570                     (1 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE) |
1571                     (fifo << IWM_SCD_QUEUE_STTS_REG_POS_TXF) |
1572                     (1 << IWM_SCD_QUEUE_STTS_REG_POS_WSL) |
1573                     IWM_SCD_QUEUE_STTS_REG_MSK);
1574         } else {
1575                 struct iwm_scd_txq_cfg_cmd cmd;
1576                 int error;
1577
1578                 iwm_nic_unlock(sc);
1579
1580                 memset(&cmd, 0, sizeof(cmd));
1581                 cmd.scd_queue = qid;
1582                 cmd.enable = 1;
1583                 cmd.sta_id = sta_id;
1584                 cmd.tx_fifo = fifo;
1585                 cmd.aggregate = 0;
1586                 cmd.window = IWM_FRAME_LIMIT;
1587
1588                 error = iwm_mvm_send_cmd_pdu(sc, IWM_SCD_QUEUE_CFG, IWM_CMD_SYNC,
1589                     sizeof(cmd), &cmd);
1590                 if (error) {
1591                         device_printf(sc->sc_dev,
1592                             "cannot enable txq %d\n", qid);
1593                         return error;
1594                 }
1595
1596                 if (!iwm_nic_lock(sc))
1597                         return EBUSY;
1598         }
1599
1600         iwm_write_prph(sc, IWM_SCD_EN_CTRL,
1601             iwm_read_prph(sc, IWM_SCD_EN_CTRL) | qid);
1602
1603         iwm_nic_unlock(sc);
1604
1605         IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "%s: enabled txq %d FIFO %d\n",
1606             __func__, qid, fifo);
1607
1608         return 0;
1609 }
1610
1611 static int
1612 iwm_trans_pcie_fw_alive(struct iwm_softc *sc, uint32_t scd_base_addr)
1613 {
1614         int error, chnl;
1615
1616         int clear_dwords = (IWM_SCD_TRANS_TBL_MEM_UPPER_BOUND -
1617             IWM_SCD_CONTEXT_MEM_LOWER_BOUND) / sizeof(uint32_t);
1618
1619         if (!iwm_nic_lock(sc))
1620                 return EBUSY;
1621
1622         iwm_ict_reset(sc);
1623
1624         iwm_nic_unlock(sc);
1625
1626         sc->scd_base_addr = iwm_read_prph(sc, IWM_SCD_SRAM_BASE_ADDR);
1627         if (scd_base_addr != 0 &&
1628             scd_base_addr != sc->scd_base_addr) {
1629                 device_printf(sc->sc_dev,
1630                     "%s: sched addr mismatch: alive: 0x%x prph: 0x%x\n",
1631                     __func__, sc->scd_base_addr, scd_base_addr);
1632         }
1633
1634         /* reset context data, TX status and translation data */
1635         error = iwm_write_mem(sc,
1636             sc->scd_base_addr + IWM_SCD_CONTEXT_MEM_LOWER_BOUND,
1637             NULL, clear_dwords);
1638         if (error)
1639                 return EBUSY;
1640
1641         if (!iwm_nic_lock(sc))
1642                 return EBUSY;
1643
1644         /* Set physical address of TX scheduler rings (1KB aligned). */
1645         iwm_write_prph(sc, IWM_SCD_DRAM_BASE_ADDR, sc->sched_dma.paddr >> 10);
1646
1647         iwm_write_prph(sc, IWM_SCD_CHAINEXT_EN, 0);
1648
1649         iwm_nic_unlock(sc);
1650
1651         /* enable command channel */
1652         error = iwm_enable_txq(sc, 0 /* unused */, IWM_MVM_CMD_QUEUE, 7);
1653         if (error)
1654                 return error;
1655
1656         if (!iwm_nic_lock(sc))
1657                 return EBUSY;
1658
1659         iwm_write_prph(sc, IWM_SCD_TXFACT, 0xff);
1660
1661         /* Enable DMA channels. */
1662         for (chnl = 0; chnl < IWM_FH_TCSR_CHNL_NUM; chnl++) {
1663                 IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl),
1664                     IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE |
1665                     IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE);
1666         }
1667
1668         IWM_SETBITS(sc, IWM_FH_TX_CHICKEN_BITS_REG,
1669             IWM_FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN);
1670
1671         iwm_nic_unlock(sc);
1672
1673         /* Enable L1-Active */
1674         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
1675                 iwm_clear_bits_prph(sc, IWM_APMG_PCIDEV_STT_REG,
1676                     IWM_APMG_PCIDEV_STT_VAL_L1_ACT_DIS);
1677         }
1678
1679         return error;
1680 }
1681
1682 /*
1683  * NVM read access and content parsing.  We do not support
1684  * external NVM or writing NVM.
1685  * iwlwifi/mvm/nvm.c
1686  */
1687
1688 /* Default NVM size to read */
1689 #define IWM_NVM_DEFAULT_CHUNK_SIZE      (2*1024)
1690
1691 #define IWM_NVM_WRITE_OPCODE 1
1692 #define IWM_NVM_READ_OPCODE 0
1693
1694 /* load nvm chunk response */
1695 enum {
1696         IWM_READ_NVM_CHUNK_SUCCEED = 0,
1697         IWM_READ_NVM_CHUNK_NOT_VALID_ADDRESS = 1
1698 };
1699
1700 static int
1701 iwm_nvm_read_chunk(struct iwm_softc *sc, uint16_t section,
1702         uint16_t offset, uint16_t length, uint8_t *data, uint16_t *len)
1703 {
1704         struct iwm_nvm_access_cmd nvm_access_cmd = {
1705                 .offset = htole16(offset),
1706                 .length = htole16(length),
1707                 .type = htole16(section),
1708                 .op_code = IWM_NVM_READ_OPCODE,
1709         };
1710         struct iwm_nvm_access_resp *nvm_resp;
1711         struct iwm_rx_packet *pkt;
1712         struct iwm_host_cmd cmd = {
1713                 .id = IWM_NVM_ACCESS_CMD,
1714                 .flags = IWM_CMD_WANT_SKB | IWM_CMD_SEND_IN_RFKILL,
1715                 .data = { &nvm_access_cmd, },
1716         };
1717         int ret, bytes_read, offset_read;
1718         uint8_t *resp_data;
1719
1720         cmd.len[0] = sizeof(struct iwm_nvm_access_cmd);
1721
1722         ret = iwm_send_cmd(sc, &cmd);
1723         if (ret) {
1724                 device_printf(sc->sc_dev,
1725                     "Could not send NVM_ACCESS command (error=%d)\n", ret);
1726                 return ret;
1727         }
1728
1729         pkt = cmd.resp_pkt;
1730
1731         /* Extract NVM response */
1732         nvm_resp = (void *)pkt->data;
1733         ret = le16toh(nvm_resp->status);
1734         bytes_read = le16toh(nvm_resp->length);
1735         offset_read = le16toh(nvm_resp->offset);
1736         resp_data = nvm_resp->data;
1737         if (ret) {
1738                 if ((offset != 0) &&
1739                     (ret == IWM_READ_NVM_CHUNK_NOT_VALID_ADDRESS)) {
1740                         /*
1741                          * meaning of NOT_VALID_ADDRESS:
1742                          * driver try to read chunk from address that is
1743                          * multiple of 2K and got an error since addr is empty.
1744                          * meaning of (offset != 0): driver already
1745                          * read valid data from another chunk so this case
1746                          * is not an error.
1747                          */
1748                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1749                                     "NVM access command failed on offset 0x%x since that section size is multiple 2K\n",
1750                                     offset);
1751                         *len = 0;
1752                         ret = 0;
1753                 } else {
1754                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1755                                     "NVM access command failed with status %d\n", ret);
1756                         ret = EIO;
1757                 }
1758                 goto exit;
1759         }
1760
1761         if (offset_read != offset) {
1762                 device_printf(sc->sc_dev,
1763                     "NVM ACCESS response with invalid offset %d\n",
1764                     offset_read);
1765                 ret = EINVAL;
1766                 goto exit;
1767         }
1768
1769         if (bytes_read > length) {
1770                 device_printf(sc->sc_dev,
1771                     "NVM ACCESS response with too much data "
1772                     "(%d bytes requested, %d bytes received)\n",
1773                     length, bytes_read);
1774                 ret = EINVAL;
1775                 goto exit;
1776         }
1777
1778         /* Write data to NVM */
1779         memcpy(data + offset, resp_data, bytes_read);
1780         *len = bytes_read;
1781
1782  exit:
1783         iwm_free_resp(sc, &cmd);
1784         return ret;
1785 }
1786
1787 /*
1788  * Reads an NVM section completely.
1789  * NICs prior to 7000 family don't have a real NVM, but just read
1790  * section 0 which is the EEPROM. Because the EEPROM reading is unlimited
1791  * by uCode, we need to manually check in this case that we don't
1792  * overflow and try to read more than the EEPROM size.
1793  * For 7000 family NICs, we supply the maximal size we can read, and
1794  * the uCode fills the response with as much data as we can,
1795  * without overflowing, so no check is needed.
1796  */
1797 static int
1798 iwm_nvm_read_section(struct iwm_softc *sc,
1799         uint16_t section, uint8_t *data, uint16_t *len, uint32_t size_read)
1800 {
1801         uint16_t seglen, length, offset = 0;
1802         int ret;
1803
1804         /* Set nvm section read length */
1805         length = IWM_NVM_DEFAULT_CHUNK_SIZE;
1806
1807         seglen = length;
1808
1809         /* Read the NVM until exhausted (reading less than requested) */
1810         while (seglen == length) {
1811                 /* Check no memory assumptions fail and cause an overflow */
1812                 if ((size_read + offset + length) >
1813                     sc->cfg->eeprom_size) {
1814                         device_printf(sc->sc_dev,
1815                             "EEPROM size is too small for NVM\n");
1816                         return ENOBUFS;
1817                 }
1818
1819                 ret = iwm_nvm_read_chunk(sc, section, offset, length, data, &seglen);
1820                 if (ret) {
1821                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1822                                     "Cannot read NVM from section %d offset %d, length %d\n",
1823                                     section, offset, length);
1824                         return ret;
1825                 }
1826                 offset += seglen;
1827         }
1828
1829         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1830                     "NVM section %d read completed\n", section);
1831         *len = offset;
1832         return 0;
1833 }
1834
1835 /*
1836  * BEGIN IWM_NVM_PARSE
1837  */
1838
1839 /* iwlwifi/iwl-nvm-parse.c */
1840
1841 /* NVM offsets (in words) definitions */
1842 enum iwm_nvm_offsets {
1843         /* NVM HW-Section offset (in words) definitions */
1844         IWM_HW_ADDR = 0x15,
1845
1846 /* NVM SW-Section offset (in words) definitions */
1847         IWM_NVM_SW_SECTION = 0x1C0,
1848         IWM_NVM_VERSION = 0,
1849         IWM_RADIO_CFG = 1,
1850         IWM_SKU = 2,
1851         IWM_N_HW_ADDRS = 3,
1852         IWM_NVM_CHANNELS = 0x1E0 - IWM_NVM_SW_SECTION,
1853
1854 /* NVM calibration section offset (in words) definitions */
1855         IWM_NVM_CALIB_SECTION = 0x2B8,
1856         IWM_XTAL_CALIB = 0x316 - IWM_NVM_CALIB_SECTION
1857 };
1858
1859 enum iwm_8000_nvm_offsets {
1860         /* NVM HW-Section offset (in words) definitions */
1861         IWM_HW_ADDR0_WFPM_8000 = 0x12,
1862         IWM_HW_ADDR1_WFPM_8000 = 0x16,
1863         IWM_HW_ADDR0_PCIE_8000 = 0x8A,
1864         IWM_HW_ADDR1_PCIE_8000 = 0x8E,
1865         IWM_MAC_ADDRESS_OVERRIDE_8000 = 1,
1866
1867         /* NVM SW-Section offset (in words) definitions */
1868         IWM_NVM_SW_SECTION_8000 = 0x1C0,
1869         IWM_NVM_VERSION_8000 = 0,
1870         IWM_RADIO_CFG_8000 = 0,
1871         IWM_SKU_8000 = 2,
1872         IWM_N_HW_ADDRS_8000 = 3,
1873
1874         /* NVM REGULATORY -Section offset (in words) definitions */
1875         IWM_NVM_CHANNELS_8000 = 0,
1876         IWM_NVM_LAR_OFFSET_8000_OLD = 0x4C7,
1877         IWM_NVM_LAR_OFFSET_8000 = 0x507,
1878         IWM_NVM_LAR_ENABLED_8000 = 0x7,
1879
1880         /* NVM calibration section offset (in words) definitions */
1881         IWM_NVM_CALIB_SECTION_8000 = 0x2B8,
1882         IWM_XTAL_CALIB_8000 = 0x316 - IWM_NVM_CALIB_SECTION_8000
1883 };
1884
1885 /* SKU Capabilities (actual values from NVM definition) */
1886 enum nvm_sku_bits {
1887         IWM_NVM_SKU_CAP_BAND_24GHZ      = (1 << 0),
1888         IWM_NVM_SKU_CAP_BAND_52GHZ      = (1 << 1),
1889         IWM_NVM_SKU_CAP_11N_ENABLE      = (1 << 2),
1890         IWM_NVM_SKU_CAP_11AC_ENABLE     = (1 << 3),
1891 };
1892
1893 /* radio config bits (actual values from NVM definition) */
1894 #define IWM_NVM_RF_CFG_DASH_MSK(x)   (x & 0x3)         /* bits 0-1   */
1895 #define IWM_NVM_RF_CFG_STEP_MSK(x)   ((x >> 2)  & 0x3) /* bits 2-3   */
1896 #define IWM_NVM_RF_CFG_TYPE_MSK(x)   ((x >> 4)  & 0x3) /* bits 4-5   */
1897 #define IWM_NVM_RF_CFG_PNUM_MSK(x)   ((x >> 6)  & 0x3) /* bits 6-7   */
1898 #define IWM_NVM_RF_CFG_TX_ANT_MSK(x) ((x >> 8)  & 0xF) /* bits 8-11  */
1899 #define IWM_NVM_RF_CFG_RX_ANT_MSK(x) ((x >> 12) & 0xF) /* bits 12-15 */
1900
1901 #define IWM_NVM_RF_CFG_FLAVOR_MSK_8000(x)       (x & 0xF)
1902 #define IWM_NVM_RF_CFG_DASH_MSK_8000(x)         ((x >> 4) & 0xF)
1903 #define IWM_NVM_RF_CFG_STEP_MSK_8000(x)         ((x >> 8) & 0xF)
1904 #define IWM_NVM_RF_CFG_TYPE_MSK_8000(x)         ((x >> 12) & 0xFFF)
1905 #define IWM_NVM_RF_CFG_TX_ANT_MSK_8000(x)       ((x >> 24) & 0xF)
1906 #define IWM_NVM_RF_CFG_RX_ANT_MSK_8000(x)       ((x >> 28) & 0xF)
1907
1908 #define DEFAULT_MAX_TX_POWER 16
1909
1910 /**
1911  * enum iwm_nvm_channel_flags - channel flags in NVM
1912  * @IWM_NVM_CHANNEL_VALID: channel is usable for this SKU/geo
1913  * @IWM_NVM_CHANNEL_IBSS: usable as an IBSS channel
1914  * @IWM_NVM_CHANNEL_ACTIVE: active scanning allowed
1915  * @IWM_NVM_CHANNEL_RADAR: radar detection required
1916  * XXX cannot find this (DFS) flag in iwm-nvm-parse.c
1917  * @IWM_NVM_CHANNEL_DFS: dynamic freq selection candidate
1918  * @IWM_NVM_CHANNEL_WIDE: 20 MHz channel okay (?)
1919  * @IWM_NVM_CHANNEL_40MHZ: 40 MHz channel okay (?)
1920  * @IWM_NVM_CHANNEL_80MHZ: 80 MHz channel okay (?)
1921  * @IWM_NVM_CHANNEL_160MHZ: 160 MHz channel okay (?)
1922  */
1923 enum iwm_nvm_channel_flags {
1924         IWM_NVM_CHANNEL_VALID = (1 << 0),
1925         IWM_NVM_CHANNEL_IBSS = (1 << 1),
1926         IWM_NVM_CHANNEL_ACTIVE = (1 << 3),
1927         IWM_NVM_CHANNEL_RADAR = (1 << 4),
1928         IWM_NVM_CHANNEL_DFS = (1 << 7),
1929         IWM_NVM_CHANNEL_WIDE = (1 << 8),
1930         IWM_NVM_CHANNEL_40MHZ = (1 << 9),
1931         IWM_NVM_CHANNEL_80MHZ = (1 << 10),
1932         IWM_NVM_CHANNEL_160MHZ = (1 << 11),
1933 };
1934
1935 /*
1936  * Translate EEPROM flags to net80211.
1937  */
1938 static uint32_t
1939 iwm_eeprom_channel_flags(uint16_t ch_flags)
1940 {
1941         uint32_t nflags;
1942
1943         nflags = 0;
1944         if ((ch_flags & IWM_NVM_CHANNEL_ACTIVE) == 0)
1945                 nflags |= IEEE80211_CHAN_PASSIVE;
1946         if ((ch_flags & IWM_NVM_CHANNEL_IBSS) == 0)
1947                 nflags |= IEEE80211_CHAN_NOADHOC;
1948         if (ch_flags & IWM_NVM_CHANNEL_RADAR) {
1949                 nflags |= IEEE80211_CHAN_DFS;
1950                 /* Just in case. */
1951                 nflags |= IEEE80211_CHAN_NOADHOC;
1952         }
1953
1954         return (nflags);
1955 }
1956
1957 static void
1958 iwm_add_channel_band(struct iwm_softc *sc, struct ieee80211_channel chans[],
1959     int maxchans, int *nchans, int ch_idx, size_t ch_num,
1960     const uint8_t bands[])
1961 {
1962         const uint16_t * const nvm_ch_flags = sc->nvm_data->nvm_ch_flags;
1963         uint32_t nflags;
1964         uint16_t ch_flags;
1965         uint8_t ieee;
1966         int error;
1967
1968         for (; ch_idx < ch_num; ch_idx++) {
1969                 ch_flags = le16_to_cpup(nvm_ch_flags + ch_idx);
1970                 if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
1971                         ieee = iwm_nvm_channels[ch_idx];
1972                 else
1973                         ieee = iwm_nvm_channels_8000[ch_idx];
1974
1975                 if (!(ch_flags & IWM_NVM_CHANNEL_VALID)) {
1976                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM,
1977                             "Ch. %d Flags %x [%sGHz] - No traffic\n",
1978                             ieee, ch_flags,
1979                             (ch_idx >= IWM_NUM_2GHZ_CHANNELS) ?
1980                             "5.2" : "2.4");
1981                         continue;
1982                 }
1983
1984                 nflags = iwm_eeprom_channel_flags(ch_flags);
1985                 error = ieee80211_add_channel(chans, maxchans, nchans,
1986                     ieee, 0, 0, nflags, bands);
1987                 if (error != 0)
1988                         break;
1989
1990                 IWM_DPRINTF(sc, IWM_DEBUG_EEPROM,
1991                     "Ch. %d Flags %x [%sGHz] - Added\n",
1992                     ieee, ch_flags,
1993                     (ch_idx >= IWM_NUM_2GHZ_CHANNELS) ?
1994                     "5.2" : "2.4");
1995         }
1996 }
1997
1998 static void
1999 iwm_init_channel_map(struct ieee80211com *ic, int maxchans, int *nchans,
2000     struct ieee80211_channel chans[])
2001 {
2002         struct iwm_softc *sc = ic->ic_softc;
2003         struct iwm_nvm_data *data = sc->nvm_data;
2004         uint8_t bands[IEEE80211_MODE_BYTES];
2005         size_t ch_num;
2006
2007         memset(bands, 0, sizeof(bands));
2008         /* 1-13: 11b/g channels. */
2009         setbit(bands, IEEE80211_MODE_11B);
2010         setbit(bands, IEEE80211_MODE_11G);
2011         iwm_add_channel_band(sc, chans, maxchans, nchans, 0,
2012             IWM_NUM_2GHZ_CHANNELS - 1, bands);
2013
2014         /* 14: 11b channel only. */
2015         clrbit(bands, IEEE80211_MODE_11G);
2016         iwm_add_channel_band(sc, chans, maxchans, nchans,
2017             IWM_NUM_2GHZ_CHANNELS - 1, IWM_NUM_2GHZ_CHANNELS, bands);
2018
2019         if (data->sku_cap_band_52GHz_enable) {
2020                 if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
2021                         ch_num = nitems(iwm_nvm_channels);
2022                 else
2023                         ch_num = nitems(iwm_nvm_channels_8000);
2024                 memset(bands, 0, sizeof(bands));
2025                 setbit(bands, IEEE80211_MODE_11A);
2026                 iwm_add_channel_band(sc, chans, maxchans, nchans,
2027                     IWM_NUM_2GHZ_CHANNELS, ch_num, bands);
2028         }
2029 }
2030
2031 static void
2032 iwm_set_hw_address_family_8000(struct iwm_softc *sc, struct iwm_nvm_data *data,
2033         const uint16_t *mac_override, const uint16_t *nvm_hw)
2034 {
2035         const uint8_t *hw_addr;
2036
2037         if (mac_override) {
2038                 static const uint8_t reserved_mac[] = {
2039                         0x02, 0xcc, 0xaa, 0xff, 0xee, 0x00
2040                 };
2041
2042                 hw_addr = (const uint8_t *)(mac_override +
2043                                  IWM_MAC_ADDRESS_OVERRIDE_8000);
2044
2045                 /*
2046                  * Store the MAC address from MAO section.
2047                  * No byte swapping is required in MAO section
2048                  */
2049                 IEEE80211_ADDR_COPY(data->hw_addr, hw_addr);
2050
2051                 /*
2052                  * Force the use of the OTP MAC address in case of reserved MAC
2053                  * address in the NVM, or if address is given but invalid.
2054                  */
2055                 if (!IEEE80211_ADDR_EQ(reserved_mac, hw_addr) &&
2056                     !IEEE80211_ADDR_EQ(ieee80211broadcastaddr, data->hw_addr) &&
2057                     iwm_is_valid_ether_addr(data->hw_addr) &&
2058                     !IEEE80211_IS_MULTICAST(data->hw_addr))
2059                         return;
2060
2061                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2062                     "%s: mac address from nvm override section invalid\n",
2063                     __func__);
2064         }
2065
2066         if (nvm_hw) {
2067                 /* read the mac address from WFMP registers */
2068                 uint32_t mac_addr0 =
2069                     htole32(iwm_read_prph(sc, IWM_WFMP_MAC_ADDR_0));
2070                 uint32_t mac_addr1 =
2071                     htole32(iwm_read_prph(sc, IWM_WFMP_MAC_ADDR_1));
2072
2073                 hw_addr = (const uint8_t *)&mac_addr0;
2074                 data->hw_addr[0] = hw_addr[3];
2075                 data->hw_addr[1] = hw_addr[2];
2076                 data->hw_addr[2] = hw_addr[1];
2077                 data->hw_addr[3] = hw_addr[0];
2078
2079                 hw_addr = (const uint8_t *)&mac_addr1;
2080                 data->hw_addr[4] = hw_addr[1];
2081                 data->hw_addr[5] = hw_addr[0];
2082
2083                 return;
2084         }
2085
2086         device_printf(sc->sc_dev, "%s: mac address not found\n", __func__);
2087         memset(data->hw_addr, 0, sizeof(data->hw_addr));
2088 }
2089
2090 static int
2091 iwm_get_sku(const struct iwm_softc *sc, const uint16_t *nvm_sw,
2092             const uint16_t *phy_sku)
2093 {
2094         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2095                 return le16_to_cpup(nvm_sw + IWM_SKU);
2096
2097         return le32_to_cpup((const uint32_t *)(phy_sku + IWM_SKU_8000));
2098 }
2099
2100 static int
2101 iwm_get_nvm_version(const struct iwm_softc *sc, const uint16_t *nvm_sw)
2102 {
2103         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2104                 return le16_to_cpup(nvm_sw + IWM_NVM_VERSION);
2105         else
2106                 return le32_to_cpup((const uint32_t *)(nvm_sw +
2107                                                 IWM_NVM_VERSION_8000));
2108 }
2109
2110 static int
2111 iwm_get_radio_cfg(const struct iwm_softc *sc, const uint16_t *nvm_sw,
2112                   const uint16_t *phy_sku)
2113 {
2114         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2115                 return le16_to_cpup(nvm_sw + IWM_RADIO_CFG);
2116
2117         return le32_to_cpup((const uint32_t *)(phy_sku + IWM_RADIO_CFG_8000));
2118 }
2119
2120 static int
2121 iwm_get_n_hw_addrs(const struct iwm_softc *sc, const uint16_t *nvm_sw)
2122 {
2123         int n_hw_addr;
2124
2125         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2126                 return le16_to_cpup(nvm_sw + IWM_N_HW_ADDRS);
2127
2128         n_hw_addr = le32_to_cpup((const uint32_t *)(nvm_sw + IWM_N_HW_ADDRS_8000));
2129
2130         return n_hw_addr & IWM_N_HW_ADDR_MASK;
2131 }
2132
2133 static void
2134 iwm_set_radio_cfg(const struct iwm_softc *sc, struct iwm_nvm_data *data,
2135                   uint32_t radio_cfg)
2136 {
2137         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
2138                 data->radio_cfg_type = IWM_NVM_RF_CFG_TYPE_MSK(radio_cfg);
2139                 data->radio_cfg_step = IWM_NVM_RF_CFG_STEP_MSK(radio_cfg);
2140                 data->radio_cfg_dash = IWM_NVM_RF_CFG_DASH_MSK(radio_cfg);
2141                 data->radio_cfg_pnum = IWM_NVM_RF_CFG_PNUM_MSK(radio_cfg);
2142                 return;
2143         }
2144
2145         /* set the radio configuration for family 8000 */
2146         data->radio_cfg_type = IWM_NVM_RF_CFG_TYPE_MSK_8000(radio_cfg);
2147         data->radio_cfg_step = IWM_NVM_RF_CFG_STEP_MSK_8000(radio_cfg);
2148         data->radio_cfg_dash = IWM_NVM_RF_CFG_DASH_MSK_8000(radio_cfg);
2149         data->radio_cfg_pnum = IWM_NVM_RF_CFG_FLAVOR_MSK_8000(radio_cfg);
2150         data->valid_tx_ant = IWM_NVM_RF_CFG_TX_ANT_MSK_8000(radio_cfg);
2151         data->valid_rx_ant = IWM_NVM_RF_CFG_RX_ANT_MSK_8000(radio_cfg);
2152 }
2153
2154 static int
2155 iwm_set_hw_address(struct iwm_softc *sc, struct iwm_nvm_data *data,
2156                    const uint16_t *nvm_hw, const uint16_t *mac_override)
2157 {
2158 #ifdef notyet /* for FAMILY 9000 */
2159         if (cfg->mac_addr_from_csr) {
2160                 iwm_set_hw_address_from_csr(sc, data);
2161         } else
2162 #endif
2163         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
2164                 const uint8_t *hw_addr = (const uint8_t *)(nvm_hw + IWM_HW_ADDR);
2165
2166                 /* The byte order is little endian 16 bit, meaning 214365 */
2167                 data->hw_addr[0] = hw_addr[1];
2168                 data->hw_addr[1] = hw_addr[0];
2169                 data->hw_addr[2] = hw_addr[3];
2170                 data->hw_addr[3] = hw_addr[2];
2171                 data->hw_addr[4] = hw_addr[5];
2172                 data->hw_addr[5] = hw_addr[4];
2173         } else {
2174                 iwm_set_hw_address_family_8000(sc, data, mac_override, nvm_hw);
2175         }
2176
2177         if (!iwm_is_valid_ether_addr(data->hw_addr)) {
2178                 device_printf(sc->sc_dev, "no valid mac address was found\n");
2179                 return EINVAL;
2180         }
2181
2182         return 0;
2183 }
2184
2185 static struct iwm_nvm_data *
2186 iwm_parse_nvm_data(struct iwm_softc *sc,
2187                    const uint16_t *nvm_hw, const uint16_t *nvm_sw,
2188                    const uint16_t *nvm_calib, const uint16_t *mac_override,
2189                    const uint16_t *phy_sku, const uint16_t *regulatory)
2190 {
2191         struct iwm_nvm_data *data;
2192         uint32_t sku, radio_cfg;
2193
2194         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
2195                 data = malloc(sizeof(*data) +
2196                     IWM_NUM_CHANNELS * sizeof(uint16_t),
2197                     M_DEVBUF, M_NOWAIT | M_ZERO);
2198         } else {
2199                 data = malloc(sizeof(*data) +
2200                     IWM_NUM_CHANNELS_8000 * sizeof(uint16_t),
2201                     M_DEVBUF, M_NOWAIT | M_ZERO);
2202         }
2203         if (!data)
2204                 return NULL;
2205
2206         data->nvm_version = iwm_get_nvm_version(sc, nvm_sw);
2207
2208         radio_cfg = iwm_get_radio_cfg(sc, nvm_sw, phy_sku);
2209         iwm_set_radio_cfg(sc, data, radio_cfg);
2210
2211         sku = iwm_get_sku(sc, nvm_sw, phy_sku);
2212         data->sku_cap_band_24GHz_enable = sku & IWM_NVM_SKU_CAP_BAND_24GHZ;
2213         data->sku_cap_band_52GHz_enable = sku & IWM_NVM_SKU_CAP_BAND_52GHZ;
2214         data->sku_cap_11n_enable = 0;
2215
2216         data->n_hw_addrs = iwm_get_n_hw_addrs(sc, nvm_sw);
2217
2218         /* If no valid mac address was found - bail out */
2219         if (iwm_set_hw_address(sc, data, nvm_hw, mac_override)) {
2220                 free(data, M_DEVBUF);
2221                 return NULL;
2222         }
2223
2224         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
2225                 memcpy(data->nvm_ch_flags, &nvm_sw[IWM_NVM_CHANNELS],
2226                     IWM_NUM_CHANNELS * sizeof(uint16_t));
2227         } else {
2228                 memcpy(data->nvm_ch_flags, &regulatory[IWM_NVM_CHANNELS_8000],
2229                     IWM_NUM_CHANNELS_8000 * sizeof(uint16_t));
2230         }
2231
2232         return data;
2233 }
2234
2235 static void
2236 iwm_free_nvm_data(struct iwm_nvm_data *data)
2237 {
2238         if (data != NULL)
2239                 free(data, M_DEVBUF);
2240 }
2241
2242 static struct iwm_nvm_data *
2243 iwm_parse_nvm_sections(struct iwm_softc *sc, struct iwm_nvm_section *sections)
2244 {
2245         const uint16_t *hw, *sw, *calib, *regulatory, *mac_override, *phy_sku;
2246
2247         /* Checking for required sections */
2248         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
2249                 if (!sections[IWM_NVM_SECTION_TYPE_SW].data ||
2250                     !sections[sc->cfg->nvm_hw_section_num].data) {
2251                         device_printf(sc->sc_dev,
2252                             "Can't parse empty OTP/NVM sections\n");
2253                         return NULL;
2254                 }
2255         } else if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) {
2256                 /* SW and REGULATORY sections are mandatory */
2257                 if (!sections[IWM_NVM_SECTION_TYPE_SW].data ||
2258                     !sections[IWM_NVM_SECTION_TYPE_REGULATORY].data) {
2259                         device_printf(sc->sc_dev,
2260                             "Can't parse empty OTP/NVM sections\n");
2261                         return NULL;
2262                 }
2263                 /* MAC_OVERRIDE or at least HW section must exist */
2264                 if (!sections[sc->cfg->nvm_hw_section_num].data &&
2265                     !sections[IWM_NVM_SECTION_TYPE_MAC_OVERRIDE].data) {
2266                         device_printf(sc->sc_dev,
2267                             "Can't parse mac_address, empty sections\n");
2268                         return NULL;
2269                 }
2270
2271                 /* PHY_SKU section is mandatory in B0 */
2272                 if (!sections[IWM_NVM_SECTION_TYPE_PHY_SKU].data) {
2273                         device_printf(sc->sc_dev,
2274                             "Can't parse phy_sku in B0, empty sections\n");
2275                         return NULL;
2276                 }
2277         } else {
2278                 panic("unknown device family %d\n", sc->cfg->device_family);
2279         }
2280
2281         hw = (const uint16_t *) sections[sc->cfg->nvm_hw_section_num].data;
2282         sw = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_SW].data;
2283         calib = (const uint16_t *)
2284             sections[IWM_NVM_SECTION_TYPE_CALIBRATION].data;
2285         regulatory = (const uint16_t *)
2286             sections[IWM_NVM_SECTION_TYPE_REGULATORY].data;
2287         mac_override = (const uint16_t *)
2288             sections[IWM_NVM_SECTION_TYPE_MAC_OVERRIDE].data;
2289         phy_sku = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_PHY_SKU].data;
2290
2291         return iwm_parse_nvm_data(sc, hw, sw, calib, mac_override,
2292             phy_sku, regulatory);
2293 }
2294
2295 static int
2296 iwm_nvm_init(struct iwm_softc *sc)
2297 {
2298         struct iwm_nvm_section nvm_sections[IWM_NVM_MAX_NUM_SECTIONS];
2299         int i, ret, section;
2300         uint32_t size_read = 0;
2301         uint8_t *nvm_buffer, *temp;
2302         uint16_t len;
2303
2304         memset(nvm_sections, 0, sizeof(nvm_sections));
2305
2306         if (sc->cfg->nvm_hw_section_num >= IWM_NVM_MAX_NUM_SECTIONS)
2307                 return EINVAL;
2308
2309         /* load NVM values from nic */
2310         /* Read From FW NVM */
2311         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM, "Read from NVM\n");
2312
2313         nvm_buffer = malloc(sc->cfg->eeprom_size, M_DEVBUF, M_NOWAIT | M_ZERO);
2314         if (!nvm_buffer)
2315                 return ENOMEM;
2316         for (section = 0; section < IWM_NVM_MAX_NUM_SECTIONS; section++) {
2317                 /* we override the constness for initial read */
2318                 ret = iwm_nvm_read_section(sc, section, nvm_buffer,
2319                                            &len, size_read);
2320                 if (ret)
2321                         continue;
2322                 size_read += len;
2323                 temp = malloc(len, M_DEVBUF, M_NOWAIT);
2324                 if (!temp) {
2325                         ret = ENOMEM;
2326                         break;
2327                 }
2328                 memcpy(temp, nvm_buffer, len);
2329
2330                 nvm_sections[section].data = temp;
2331                 nvm_sections[section].length = len;
2332         }
2333         if (!size_read)
2334                 device_printf(sc->sc_dev, "OTP is blank\n");
2335         free(nvm_buffer, M_DEVBUF);
2336
2337         sc->nvm_data = iwm_parse_nvm_sections(sc, nvm_sections);
2338         if (!sc->nvm_data)
2339                 return EINVAL;
2340         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
2341                     "nvm version = %x\n", sc->nvm_data->nvm_version);
2342
2343         for (i = 0; i < IWM_NVM_MAX_NUM_SECTIONS; i++) {
2344                 if (nvm_sections[i].data != NULL)
2345                         free(nvm_sections[i].data, M_DEVBUF);
2346         }
2347
2348         return 0;
2349 }
2350
2351 static int
2352 iwm_pcie_load_section(struct iwm_softc *sc, uint8_t section_num,
2353         const struct iwm_fw_desc *section)
2354 {
2355         struct iwm_dma_info *dma = &sc->fw_dma;
2356         uint8_t *v_addr;
2357         bus_addr_t p_addr;
2358         uint32_t offset, chunk_sz = MIN(IWM_FH_MEM_TB_MAX_LENGTH, section->len);
2359         int ret = 0;
2360
2361         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2362                     "%s: [%d] uCode section being loaded...\n",
2363                     __func__, section_num);
2364
2365         v_addr = dma->vaddr;
2366         p_addr = dma->paddr;
2367
2368         for (offset = 0; offset < section->len; offset += chunk_sz) {
2369                 uint32_t copy_size, dst_addr;
2370                 int extended_addr = FALSE;
2371
2372                 copy_size = MIN(chunk_sz, section->len - offset);
2373                 dst_addr = section->offset + offset;
2374
2375                 if (dst_addr >= IWM_FW_MEM_EXTENDED_START &&
2376                     dst_addr <= IWM_FW_MEM_EXTENDED_END)
2377                         extended_addr = TRUE;
2378
2379                 if (extended_addr)
2380                         iwm_set_bits_prph(sc, IWM_LMPM_CHICK,
2381                                           IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE);
2382
2383                 memcpy(v_addr, (const uint8_t *)section->data + offset,
2384                     copy_size);
2385                 bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREWRITE);
2386                 ret = iwm_pcie_load_firmware_chunk(sc, dst_addr, p_addr,
2387                                                    copy_size);
2388
2389                 if (extended_addr)
2390                         iwm_clear_bits_prph(sc, IWM_LMPM_CHICK,
2391                                             IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE);
2392
2393                 if (ret) {
2394                         device_printf(sc->sc_dev,
2395                             "%s: Could not load the [%d] uCode section\n",
2396                             __func__, section_num);
2397                         break;
2398                 }
2399         }
2400
2401         return ret;
2402 }
2403
2404 /*
2405  * ucode
2406  */
2407 static int
2408 iwm_pcie_load_firmware_chunk(struct iwm_softc *sc, uint32_t dst_addr,
2409                              bus_addr_t phy_addr, uint32_t byte_cnt)
2410 {
2411         int ret;
2412
2413         sc->sc_fw_chunk_done = 0;
2414
2415         if (!iwm_nic_lock(sc))
2416                 return EBUSY;
2417
2418         IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(IWM_FH_SRVC_CHNL),
2419             IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE);
2420
2421         IWM_WRITE(sc, IWM_FH_SRVC_CHNL_SRAM_ADDR_REG(IWM_FH_SRVC_CHNL),
2422             dst_addr);
2423
2424         IWM_WRITE(sc, IWM_FH_TFDIB_CTRL0_REG(IWM_FH_SRVC_CHNL),
2425             phy_addr & IWM_FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK);
2426
2427         IWM_WRITE(sc, IWM_FH_TFDIB_CTRL1_REG(IWM_FH_SRVC_CHNL),
2428             (iwm_get_dma_hi_addr(phy_addr)
2429              << IWM_FH_MEM_TFDIB_REG1_ADDR_BITSHIFT) | byte_cnt);
2430
2431         IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_BUF_STS_REG(IWM_FH_SRVC_CHNL),
2432             1 << IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM |
2433             1 << IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX |
2434             IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID);
2435
2436         IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(IWM_FH_SRVC_CHNL),
2437             IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE    |
2438             IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE |
2439             IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD);
2440
2441         iwm_nic_unlock(sc);
2442
2443         /* wait up to 5s for this segment to load */
2444         ret = 0;
2445         while (!sc->sc_fw_chunk_done) {
2446                 ret = msleep(&sc->sc_fw, &sc->sc_mtx, 0, "iwmfw", hz);
2447                 if (ret)
2448                         break;
2449         }
2450
2451         if (ret != 0) {
2452                 device_printf(sc->sc_dev,
2453                     "fw chunk addr 0x%x len %d failed to load\n",
2454                     dst_addr, byte_cnt);
2455                 return ETIMEDOUT;
2456         }
2457
2458         return 0;
2459 }
2460
2461 static int
2462 iwm_pcie_load_cpu_sections_8000(struct iwm_softc *sc,
2463         const struct iwm_fw_sects *image, int cpu, int *first_ucode_section)
2464 {
2465         int shift_param;
2466         int i, ret = 0, sec_num = 0x1;
2467         uint32_t val, last_read_idx = 0;
2468
2469         if (cpu == 1) {
2470                 shift_param = 0;
2471                 *first_ucode_section = 0;
2472         } else {
2473                 shift_param = 16;
2474                 (*first_ucode_section)++;
2475         }
2476
2477         for (i = *first_ucode_section; i < IWM_UCODE_SECTION_MAX; i++) {
2478                 last_read_idx = i;
2479
2480                 /*
2481                  * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
2482                  * CPU1 to CPU2.
2483                  * PAGING_SEPARATOR_SECTION delimiter - separate between
2484                  * CPU2 non paged to CPU2 paging sec.
2485                  */
2486                 if (!image->fw_sect[i].data ||
2487                     image->fw_sect[i].offset == IWM_CPU1_CPU2_SEPARATOR_SECTION ||
2488                     image->fw_sect[i].offset == IWM_PAGING_SEPARATOR_SECTION) {
2489                         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2490                                     "Break since Data not valid or Empty section, sec = %d\n",
2491                                     i);
2492                         break;
2493                 }
2494                 ret = iwm_pcie_load_section(sc, i, &image->fw_sect[i]);
2495                 if (ret)
2496                         return ret;
2497
2498                 /* Notify the ucode of the loaded section number and status */
2499                 if (iwm_nic_lock(sc)) {
2500                         val = IWM_READ(sc, IWM_FH_UCODE_LOAD_STATUS);
2501                         val = val | (sec_num << shift_param);
2502                         IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, val);
2503                         sec_num = (sec_num << 1) | 0x1;
2504                         iwm_nic_unlock(sc);
2505                 }
2506         }
2507
2508         *first_ucode_section = last_read_idx;
2509
2510         iwm_enable_interrupts(sc);
2511
2512         if (iwm_nic_lock(sc)) {
2513                 if (cpu == 1)
2514                         IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, 0xFFFF);
2515                 else
2516                         IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, 0xFFFFFFFF);
2517                 iwm_nic_unlock(sc);
2518         }
2519
2520         return 0;
2521 }
2522
2523 static int
2524 iwm_pcie_load_cpu_sections(struct iwm_softc *sc,
2525         const struct iwm_fw_sects *image, int cpu, int *first_ucode_section)
2526 {
2527         int shift_param;
2528         int i, ret = 0;
2529         uint32_t last_read_idx = 0;
2530
2531         if (cpu == 1) {
2532                 shift_param = 0;
2533                 *first_ucode_section = 0;
2534         } else {
2535                 shift_param = 16;
2536                 (*first_ucode_section)++;
2537         }
2538
2539         for (i = *first_ucode_section; i < IWM_UCODE_SECTION_MAX; i++) {
2540                 last_read_idx = i;
2541
2542                 /*
2543                  * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
2544                  * CPU1 to CPU2.
2545                  * PAGING_SEPARATOR_SECTION delimiter - separate between
2546                  * CPU2 non paged to CPU2 paging sec.
2547                  */
2548                 if (!image->fw_sect[i].data ||
2549                     image->fw_sect[i].offset == IWM_CPU1_CPU2_SEPARATOR_SECTION ||
2550                     image->fw_sect[i].offset == IWM_PAGING_SEPARATOR_SECTION) {
2551                         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2552                                     "Break since Data not valid or Empty section, sec = %d\n",
2553                                      i);
2554                         break;
2555                 }
2556
2557                 ret = iwm_pcie_load_section(sc, i, &image->fw_sect[i]);
2558                 if (ret)
2559                         return ret;
2560         }
2561
2562         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000)
2563                 iwm_set_bits_prph(sc,
2564                                   IWM_CSR_UCODE_LOAD_STATUS_ADDR,
2565                                   (IWM_LMPM_CPU_UCODE_LOADING_COMPLETED |
2566                                    IWM_LMPM_CPU_HDRS_LOADING_COMPLETED |
2567                                    IWM_LMPM_CPU_UCODE_LOADING_STARTED) <<
2568                                         shift_param);
2569
2570         *first_ucode_section = last_read_idx;
2571
2572         return 0;
2573
2574 }
2575
2576 static int
2577 iwm_pcie_load_given_ucode(struct iwm_softc *sc,
2578         const struct iwm_fw_sects *image)
2579 {
2580         int ret = 0;
2581         int first_ucode_section;
2582
2583         IWM_DPRINTF(sc, IWM_DEBUG_RESET, "working with %s CPU\n",
2584                      image->is_dual_cpus ? "Dual" : "Single");
2585
2586         /* load to FW the binary non secured sections of CPU1 */
2587         ret = iwm_pcie_load_cpu_sections(sc, image, 1, &first_ucode_section);
2588         if (ret)
2589                 return ret;
2590
2591         if (image->is_dual_cpus) {
2592                 /* set CPU2 header address */
2593                 iwm_write_prph(sc,
2594                                IWM_LMPM_SECURE_UCODE_LOAD_CPU2_HDR_ADDR,
2595                                IWM_LMPM_SECURE_CPU2_HDR_MEM_SPACE);
2596
2597                 /* load to FW the binary sections of CPU2 */
2598                 ret = iwm_pcie_load_cpu_sections(sc, image, 2,
2599                                                  &first_ucode_section);
2600                 if (ret)
2601                         return ret;
2602         }
2603
2604         iwm_enable_interrupts(sc);
2605
2606         /* release CPU reset */
2607         IWM_WRITE(sc, IWM_CSR_RESET, 0);
2608
2609         return 0;
2610 }
2611
2612 int
2613 iwm_pcie_load_given_ucode_8000(struct iwm_softc *sc,
2614         const struct iwm_fw_sects *image)
2615 {
2616         int ret = 0;
2617         int first_ucode_section;
2618
2619         IWM_DPRINTF(sc, IWM_DEBUG_RESET, "working with %s CPU\n",
2620                     image->is_dual_cpus ? "Dual" : "Single");
2621
2622         /* configure the ucode to be ready to get the secured image */
2623         /* release CPU reset */
2624         iwm_write_prph(sc, IWM_RELEASE_CPU_RESET, IWM_RELEASE_CPU_RESET_BIT);
2625
2626         /* load to FW the binary Secured sections of CPU1 */
2627         ret = iwm_pcie_load_cpu_sections_8000(sc, image, 1,
2628             &first_ucode_section);
2629         if (ret)
2630                 return ret;
2631
2632         /* load to FW the binary sections of CPU2 */
2633         return iwm_pcie_load_cpu_sections_8000(sc, image, 2,
2634             &first_ucode_section);
2635 }
2636
2637 /* XXX Get rid of this definition */
2638 static inline void
2639 iwm_enable_fw_load_int(struct iwm_softc *sc)
2640 {
2641         IWM_DPRINTF(sc, IWM_DEBUG_INTR, "Enabling FW load interrupt\n");
2642         sc->sc_intmask = IWM_CSR_INT_BIT_FH_TX;
2643         IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask);
2644 }
2645
2646 /* XXX Add proper rfkill support code */
2647 static int
2648 iwm_start_fw(struct iwm_softc *sc,
2649         const struct iwm_fw_sects *fw)
2650 {
2651         int ret;
2652
2653         /* This may fail if AMT took ownership of the device */
2654         if (iwm_prepare_card_hw(sc)) {
2655                 device_printf(sc->sc_dev,
2656                     "%s: Exit HW not ready\n", __func__);
2657                 ret = EIO;
2658                 goto out;
2659         }
2660
2661         IWM_WRITE(sc, IWM_CSR_INT, 0xFFFFFFFF);
2662
2663         iwm_disable_interrupts(sc);
2664
2665         /* make sure rfkill handshake bits are cleared */
2666         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL);
2667         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR,
2668             IWM_CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED);
2669
2670         /* clear (again), then enable host interrupts */
2671         IWM_WRITE(sc, IWM_CSR_INT, 0xFFFFFFFF);
2672
2673         ret = iwm_nic_init(sc);
2674         if (ret) {
2675                 device_printf(sc->sc_dev, "%s: Unable to init nic\n", __func__);
2676                 goto out;
2677         }
2678
2679         /*
2680          * Now, we load the firmware and don't want to be interrupted, even
2681          * by the RF-Kill interrupt (hence mask all the interrupt besides the
2682          * FH_TX interrupt which is needed to load the firmware). If the
2683          * RF-Kill switch is toggled, we will find out after having loaded
2684          * the firmware and return the proper value to the caller.
2685          */
2686         iwm_enable_fw_load_int(sc);
2687
2688         /* really make sure rfkill handshake bits are cleared */
2689         /* maybe we should write a few times more?  just to make sure */
2690         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL);
2691         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL);
2692
2693         /* Load the given image to the HW */
2694         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000)
2695                 ret = iwm_pcie_load_given_ucode_8000(sc, fw);
2696         else
2697                 ret = iwm_pcie_load_given_ucode(sc, fw);
2698
2699         /* XXX re-check RF-Kill state */
2700
2701 out:
2702         return ret;
2703 }
2704
2705 static int
2706 iwm_send_tx_ant_cfg(struct iwm_softc *sc, uint8_t valid_tx_ant)
2707 {
2708         struct iwm_tx_ant_cfg_cmd tx_ant_cmd = {
2709                 .valid = htole32(valid_tx_ant),
2710         };
2711
2712         return iwm_mvm_send_cmd_pdu(sc, IWM_TX_ANT_CONFIGURATION_CMD,
2713             IWM_CMD_SYNC, sizeof(tx_ant_cmd), &tx_ant_cmd);
2714 }
2715
2716 /* iwlwifi: mvm/fw.c */
2717 static int
2718 iwm_send_phy_cfg_cmd(struct iwm_softc *sc)
2719 {
2720         struct iwm_phy_cfg_cmd phy_cfg_cmd;
2721         enum iwm_ucode_type ucode_type = sc->cur_ucode;
2722
2723         /* Set parameters */
2724         phy_cfg_cmd.phy_cfg = htole32(iwm_mvm_get_phy_config(sc));
2725         phy_cfg_cmd.calib_control.event_trigger =
2726             sc->sc_default_calib[ucode_type].event_trigger;
2727         phy_cfg_cmd.calib_control.flow_trigger =
2728             sc->sc_default_calib[ucode_type].flow_trigger;
2729
2730         IWM_DPRINTF(sc, IWM_DEBUG_CMD | IWM_DEBUG_RESET,
2731             "Sending Phy CFG command: 0x%x\n", phy_cfg_cmd.phy_cfg);
2732         return iwm_mvm_send_cmd_pdu(sc, IWM_PHY_CONFIGURATION_CMD, IWM_CMD_SYNC,
2733             sizeof(phy_cfg_cmd), &phy_cfg_cmd);
2734 }
2735
2736 static int
2737 iwm_alive_fn(struct iwm_softc *sc, struct iwm_rx_packet *pkt, void *data)
2738 {
2739         struct iwm_mvm_alive_data *alive_data = data;
2740         struct iwm_mvm_alive_resp_ver1 *palive1;
2741         struct iwm_mvm_alive_resp_ver2 *palive2;
2742         struct iwm_mvm_alive_resp *palive;
2743
2744         if (iwm_rx_packet_payload_len(pkt) == sizeof(*palive1)) {
2745                 palive1 = (void *)pkt->data;
2746
2747                 sc->support_umac_log = FALSE;
2748                 sc->error_event_table =
2749                         le32toh(palive1->error_event_table_ptr);
2750                 sc->log_event_table =
2751                         le32toh(palive1->log_event_table_ptr);
2752                 alive_data->scd_base_addr = le32toh(palive1->scd_base_ptr);
2753
2754                 alive_data->valid = le16toh(palive1->status) ==
2755                                     IWM_ALIVE_STATUS_OK;
2756                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2757                             "Alive VER1 ucode status 0x%04x revision 0x%01X 0x%01X flags 0x%01X\n",
2758                              le16toh(palive1->status), palive1->ver_type,
2759                              palive1->ver_subtype, palive1->flags);
2760         } else if (iwm_rx_packet_payload_len(pkt) == sizeof(*palive2)) {
2761                 palive2 = (void *)pkt->data;
2762                 sc->error_event_table =
2763                         le32toh(palive2->error_event_table_ptr);
2764                 sc->log_event_table =
2765                         le32toh(palive2->log_event_table_ptr);
2766                 alive_data->scd_base_addr = le32toh(palive2->scd_base_ptr);
2767                 sc->umac_error_event_table =
2768                         le32toh(palive2->error_info_addr);
2769
2770                 alive_data->valid = le16toh(palive2->status) ==
2771                                     IWM_ALIVE_STATUS_OK;
2772                 if (sc->umac_error_event_table)
2773                         sc->support_umac_log = TRUE;
2774
2775                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2776                             "Alive VER2 ucode status 0x%04x revision 0x%01X 0x%01X flags 0x%01X\n",
2777                             le16toh(palive2->status), palive2->ver_type,
2778                             palive2->ver_subtype, palive2->flags);
2779
2780                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2781                             "UMAC version: Major - 0x%x, Minor - 0x%x\n",
2782                             palive2->umac_major, palive2->umac_minor);
2783         } else if (iwm_rx_packet_payload_len(pkt) == sizeof(*palive)) {
2784                 palive = (void *)pkt->data;
2785
2786                 sc->error_event_table =
2787                         le32toh(palive->error_event_table_ptr);
2788                 sc->log_event_table =
2789                         le32toh(palive->log_event_table_ptr);
2790                 alive_data->scd_base_addr = le32toh(palive->scd_base_ptr);
2791                 sc->umac_error_event_table =
2792                         le32toh(palive->error_info_addr);
2793
2794                 alive_data->valid = le16toh(palive->status) ==
2795                                     IWM_ALIVE_STATUS_OK;
2796                 if (sc->umac_error_event_table)
2797                         sc->support_umac_log = TRUE;
2798
2799                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2800                             "Alive VER3 ucode status 0x%04x revision 0x%01X 0x%01X flags 0x%01X\n",
2801                             le16toh(palive->status), palive->ver_type,
2802                             palive->ver_subtype, palive->flags);
2803
2804                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2805                             "UMAC version: Major - 0x%x, Minor - 0x%x\n",
2806                             le32toh(palive->umac_major),
2807                             le32toh(palive->umac_minor));
2808         }
2809
2810         return TRUE;
2811 }
2812
2813 static int
2814 iwm_wait_phy_db_entry(struct iwm_softc *sc,
2815         struct iwm_rx_packet *pkt, void *data)
2816 {
2817         struct iwm_phy_db *phy_db = data;
2818
2819         if (pkt->hdr.code != IWM_CALIB_RES_NOTIF_PHY_DB) {
2820                 if(pkt->hdr.code != IWM_INIT_COMPLETE_NOTIF) {
2821                         device_printf(sc->sc_dev, "%s: Unexpected cmd: %d\n",
2822                             __func__, pkt->hdr.code);
2823                 }
2824                 return TRUE;
2825         }
2826
2827         if (iwm_phy_db_set_section(phy_db, pkt)) {
2828                 device_printf(sc->sc_dev,
2829                     "%s: iwm_phy_db_set_section failed\n", __func__);
2830         }
2831
2832         return FALSE;
2833 }
2834
2835 static int
2836 iwm_mvm_load_ucode_wait_alive(struct iwm_softc *sc,
2837         enum iwm_ucode_type ucode_type)
2838 {
2839         struct iwm_notification_wait alive_wait;
2840         struct iwm_mvm_alive_data alive_data;
2841         const struct iwm_fw_sects *fw;
2842         enum iwm_ucode_type old_type = sc->cur_ucode;
2843         int error;
2844         static const uint16_t alive_cmd[] = { IWM_MVM_ALIVE };
2845
2846         if ((error = iwm_read_firmware(sc, ucode_type)) != 0) {
2847                 device_printf(sc->sc_dev, "iwm_read_firmware: failed %d\n",
2848                         error);
2849                 return error;
2850         }
2851         fw = &sc->sc_fw.fw_sects[ucode_type];
2852         sc->cur_ucode = ucode_type;
2853         sc->ucode_loaded = FALSE;
2854
2855         memset(&alive_data, 0, sizeof(alive_data));
2856         iwm_init_notification_wait(sc->sc_notif_wait, &alive_wait,
2857                                    alive_cmd, nitems(alive_cmd),
2858                                    iwm_alive_fn, &alive_data);
2859
2860         error = iwm_start_fw(sc, fw);
2861         if (error) {
2862                 device_printf(sc->sc_dev, "iwm_start_fw: failed %d\n", error);
2863                 sc->cur_ucode = old_type;
2864                 iwm_remove_notification(sc->sc_notif_wait, &alive_wait);
2865                 return error;
2866         }
2867
2868         /*
2869          * Some things may run in the background now, but we
2870          * just wait for the ALIVE notification here.
2871          */
2872         IWM_UNLOCK(sc);
2873         error = iwm_wait_notification(sc->sc_notif_wait, &alive_wait,
2874                                       IWM_MVM_UCODE_ALIVE_TIMEOUT);
2875         IWM_LOCK(sc);
2876         if (error) {
2877                 if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) {
2878                         device_printf(sc->sc_dev,
2879                             "SecBoot CPU1 Status: 0x%x, CPU2 Status: 0x%x\n",
2880                             iwm_read_prph(sc, IWM_SB_CPU_1_STATUS),
2881                             iwm_read_prph(sc, IWM_SB_CPU_2_STATUS));
2882                 }
2883                 sc->cur_ucode = old_type;
2884                 return error;
2885         }
2886
2887         if (!alive_data.valid) {
2888                 device_printf(sc->sc_dev, "%s: Loaded ucode is not valid\n",
2889                     __func__);
2890                 sc->cur_ucode = old_type;
2891                 return EIO;
2892         }
2893
2894         iwm_trans_pcie_fw_alive(sc, alive_data.scd_base_addr);
2895
2896         /*
2897          * configure and operate fw paging mechanism.
2898          * driver configures the paging flow only once, CPU2 paging image
2899          * included in the IWM_UCODE_INIT image.
2900          */
2901         if (fw->paging_mem_size) {
2902                 error = iwm_save_fw_paging(sc, fw);
2903                 if (error) {
2904                         device_printf(sc->sc_dev,
2905                             "%s: failed to save the FW paging image\n",
2906                             __func__);
2907                         return error;
2908                 }
2909
2910                 error = iwm_send_paging_cmd(sc, fw);
2911                 if (error) {
2912                         device_printf(sc->sc_dev,
2913                             "%s: failed to send the paging cmd\n", __func__);
2914                         iwm_free_fw_paging(sc);
2915                         return error;
2916                 }
2917         }
2918
2919         if (!error)
2920                 sc->ucode_loaded = TRUE;
2921         return error;
2922 }
2923
2924 /*
2925  * mvm misc bits
2926  */
2927
2928 /*
2929  * follows iwlwifi/fw.c
2930  */
2931 static int
2932 iwm_run_init_mvm_ucode(struct iwm_softc *sc, int justnvm)
2933 {
2934         struct iwm_notification_wait calib_wait;
2935         static const uint16_t init_complete[] = {
2936                 IWM_INIT_COMPLETE_NOTIF,
2937                 IWM_CALIB_RES_NOTIF_PHY_DB
2938         };
2939         int ret;
2940
2941         /* do not operate with rfkill switch turned on */
2942         if ((sc->sc_flags & IWM_FLAG_RFKILL) && !justnvm) {
2943                 device_printf(sc->sc_dev,
2944                     "radio is disabled by hardware switch\n");
2945                 return EPERM;
2946         }
2947
2948         iwm_init_notification_wait(sc->sc_notif_wait,
2949                                    &calib_wait,
2950                                    init_complete,
2951                                    nitems(init_complete),
2952                                    iwm_wait_phy_db_entry,
2953                                    sc->sc_phy_db);
2954
2955         /* Will also start the device */
2956         ret = iwm_mvm_load_ucode_wait_alive(sc, IWM_UCODE_INIT);
2957         if (ret) {
2958                 device_printf(sc->sc_dev, "Failed to start INIT ucode: %d\n",
2959                     ret);
2960                 goto error;
2961         }
2962
2963         if (justnvm) {
2964                 /* Read nvm */
2965                 ret = iwm_nvm_init(sc);
2966                 if (ret) {
2967                         device_printf(sc->sc_dev, "failed to read nvm\n");
2968                         goto error;
2969                 }
2970                 IEEE80211_ADDR_COPY(sc->sc_ic.ic_macaddr, sc->nvm_data->hw_addr);
2971                 goto error;
2972         }
2973
2974         ret = iwm_send_bt_init_conf(sc);
2975         if (ret) {
2976                 device_printf(sc->sc_dev,
2977                     "failed to send bt coex configuration: %d\n", ret);
2978                 goto error;
2979         }
2980
2981         /* Init Smart FIFO. */
2982         ret = iwm_mvm_sf_config(sc, IWM_SF_INIT_OFF);
2983         if (ret)
2984                 goto error;
2985
2986         /* Send TX valid antennas before triggering calibrations */
2987         ret = iwm_send_tx_ant_cfg(sc, iwm_mvm_get_valid_tx_ant(sc));
2988         if (ret) {
2989                 device_printf(sc->sc_dev,
2990                     "failed to send antennas before calibration: %d\n", ret);
2991                 goto error;
2992         }
2993
2994         /*
2995          * Send phy configurations command to init uCode
2996          * to start the 16.0 uCode init image internal calibrations.
2997          */
2998         ret = iwm_send_phy_cfg_cmd(sc);
2999         if (ret) {
3000                 device_printf(sc->sc_dev,
3001                     "%s: Failed to run INIT calibrations: %d\n",
3002                     __func__, ret);
3003                 goto error;
3004         }
3005
3006         /*
3007          * Nothing to do but wait for the init complete notification
3008          * from the firmware.
3009          */
3010         IWM_UNLOCK(sc);
3011         ret = iwm_wait_notification(sc->sc_notif_wait, &calib_wait,
3012             IWM_MVM_UCODE_CALIB_TIMEOUT);
3013         IWM_LOCK(sc);
3014
3015
3016         goto out;
3017
3018 error:
3019         iwm_remove_notification(sc->sc_notif_wait, &calib_wait);
3020 out:
3021         return ret;
3022 }
3023
3024 /*
3025  * receive side
3026  */
3027
3028 /* (re)stock rx ring, called at init-time and at runtime */
3029 static int
3030 iwm_rx_addbuf(struct iwm_softc *sc, int size, int idx)
3031 {
3032         struct iwm_rx_ring *ring = &sc->rxq;
3033         struct iwm_rx_data *data = &ring->data[idx];
3034         struct mbuf *m;
3035         bus_dmamap_t dmamap = NULL;
3036         bus_dma_segment_t seg;
3037         int nsegs, error;
3038
3039         m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, IWM_RBUF_SIZE);
3040         if (m == NULL)
3041                 return ENOBUFS;
3042
3043         m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
3044         error = bus_dmamap_load_mbuf_sg(ring->data_dmat, ring->spare_map, m,
3045             &seg, &nsegs, BUS_DMA_NOWAIT);
3046         if (error != 0) {
3047                 device_printf(sc->sc_dev,
3048                     "%s: can't map mbuf, error %d\n", __func__, error);
3049                 goto fail;
3050         }
3051
3052         if (data->m != NULL)
3053                 bus_dmamap_unload(ring->data_dmat, data->map);
3054
3055         /* Swap ring->spare_map with data->map */
3056         dmamap = data->map;
3057         data->map = ring->spare_map;
3058         ring->spare_map = dmamap;
3059
3060         bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_PREREAD);
3061         data->m = m;
3062
3063         /* Update RX descriptor. */
3064         KASSERT((seg.ds_addr & 255) == 0, ("seg.ds_addr not aligned"));
3065         ring->desc[idx] = htole32(seg.ds_addr >> 8);
3066         bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map,
3067             BUS_DMASYNC_PREWRITE);
3068
3069         return 0;
3070 fail:
3071         m_freem(m);
3072         return error;
3073 }
3074
3075 /* iwlwifi: mvm/rx.c */
3076 /*
3077  * iwm_mvm_get_signal_strength - use new rx PHY INFO API
3078  * values are reported by the fw as positive values - need to negate
3079  * to obtain their dBM.  Account for missing antennas by replacing 0
3080  * values by -256dBm: practically 0 power and a non-feasible 8 bit value.
3081  */
3082 static int
3083 iwm_mvm_get_signal_strength(struct iwm_softc *sc, struct iwm_rx_phy_info *phy_info)
3084 {
3085         int energy_a, energy_b, energy_c, max_energy;
3086         uint32_t val;
3087
3088         val = le32toh(phy_info->non_cfg_phy[IWM_RX_INFO_ENERGY_ANT_ABC_IDX]);
3089         energy_a = (val & IWM_RX_INFO_ENERGY_ANT_A_MSK) >>
3090             IWM_RX_INFO_ENERGY_ANT_A_POS;
3091         energy_a = energy_a ? -energy_a : -256;
3092         energy_b = (val & IWM_RX_INFO_ENERGY_ANT_B_MSK) >>
3093             IWM_RX_INFO_ENERGY_ANT_B_POS;
3094         energy_b = energy_b ? -energy_b : -256;
3095         energy_c = (val & IWM_RX_INFO_ENERGY_ANT_C_MSK) >>
3096             IWM_RX_INFO_ENERGY_ANT_C_POS;
3097         energy_c = energy_c ? -energy_c : -256;
3098         max_energy = MAX(energy_a, energy_b);
3099         max_energy = MAX(max_energy, energy_c);
3100
3101         IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3102             "energy In A %d B %d C %d , and max %d\n",
3103             energy_a, energy_b, energy_c, max_energy);
3104
3105         return max_energy;
3106 }
3107
3108 static void
3109 iwm_mvm_rx_rx_phy_cmd(struct iwm_softc *sc,
3110         struct iwm_rx_packet *pkt, struct iwm_rx_data *data)
3111 {
3112         struct iwm_rx_phy_info *phy_info = (void *)pkt->data;
3113
3114         IWM_DPRINTF(sc, IWM_DEBUG_RECV, "received PHY stats\n");
3115         bus_dmamap_sync(sc->rxq.data_dmat, data->map, BUS_DMASYNC_POSTREAD);
3116
3117         memcpy(&sc->sc_last_phy_info, phy_info, sizeof(sc->sc_last_phy_info));
3118 }
3119
3120 /*
3121  * Retrieve the average noise (in dBm) among receivers.
3122  */
3123 static int
3124 iwm_get_noise(struct iwm_softc *sc,
3125     const struct iwm_mvm_statistics_rx_non_phy *stats)
3126 {
3127         int i, total, nbant, noise;
3128
3129         total = nbant = noise = 0;
3130         for (i = 0; i < 3; i++) {
3131                 noise = le32toh(stats->beacon_silence_rssi[i]) & 0xff;
3132                 IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: i=%d, noise=%d\n",
3133                     __func__,
3134                     i,
3135                     noise);
3136
3137                 if (noise) {
3138                         total += noise;
3139                         nbant++;
3140                 }
3141         }
3142
3143         IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: nbant=%d, total=%d\n",
3144             __func__, nbant, total);
3145 #if 0
3146         /* There should be at least one antenna but check anyway. */
3147         return (nbant == 0) ? -127 : (total / nbant) - 107;
3148 #else
3149         /* For now, just hard-code it to -96 to be safe */
3150         return (-96);
3151 #endif
3152 }
3153
3154 /*
3155  * iwm_mvm_rx_rx_mpdu - IWM_REPLY_RX_MPDU_CMD handler
3156  *
3157  * Handles the actual data of the Rx packet from the fw
3158  */
3159 static void
3160 iwm_mvm_rx_rx_mpdu(struct iwm_softc *sc, struct mbuf *m)
3161 {
3162         struct ieee80211com *ic = &sc->sc_ic;
3163         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
3164         struct ieee80211_frame *wh;
3165         struct ieee80211_node *ni;
3166         struct ieee80211_rx_stats rxs;
3167         struct iwm_rx_phy_info *phy_info;
3168         struct iwm_rx_mpdu_res_start *rx_res;
3169         struct iwm_rx_packet *pkt = mtod(m, struct iwm_rx_packet *);
3170         uint32_t len;
3171         uint32_t rx_pkt_status;
3172         int rssi;
3173
3174         phy_info = &sc->sc_last_phy_info;
3175         rx_res = (struct iwm_rx_mpdu_res_start *)pkt->data;
3176         wh = (struct ieee80211_frame *)(pkt->data + sizeof(*rx_res));
3177         len = le16toh(rx_res->byte_count);
3178         rx_pkt_status = le32toh(*(uint32_t *)(pkt->data + sizeof(*rx_res) + len));
3179
3180         if (__predict_false(phy_info->cfg_phy_cnt > 20)) {
3181                 device_printf(sc->sc_dev,
3182                     "dsp size out of range [0,20]: %d\n",
3183                     phy_info->cfg_phy_cnt);
3184                 goto fail;
3185         }
3186
3187         if (!(rx_pkt_status & IWM_RX_MPDU_RES_STATUS_CRC_OK) ||
3188             !(rx_pkt_status & IWM_RX_MPDU_RES_STATUS_OVERRUN_OK)) {
3189                 IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3190                     "Bad CRC or FIFO: 0x%08X.\n", rx_pkt_status);
3191                 goto fail;
3192         }
3193
3194         rssi = iwm_mvm_get_signal_strength(sc, phy_info);
3195
3196         /* Map it to relative value */
3197         rssi = rssi - sc->sc_noise;
3198
3199         /* replenish ring for the buffer we're going to feed to the sharks */
3200         if (iwm_rx_addbuf(sc, IWM_RBUF_SIZE, sc->rxq.cur) != 0) {
3201                 device_printf(sc->sc_dev, "%s: unable to add more buffers\n",
3202                     __func__);
3203                 goto fail;
3204         }
3205
3206         m->m_data = pkt->data + sizeof(*rx_res);
3207         m->m_pkthdr.len = m->m_len = len;
3208
3209         IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3210             "%s: rssi=%d, noise=%d\n", __func__, rssi, sc->sc_noise);
3211
3212         ni = ieee80211_find_rxnode(ic, (struct ieee80211_frame_min *)wh);
3213
3214         IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3215             "%s: phy_info: channel=%d, flags=0x%08x\n",
3216             __func__,
3217             le16toh(phy_info->channel),
3218             le16toh(phy_info->phy_flags));
3219
3220         /*
3221          * Populate an RX state struct with the provided information.
3222          */
3223         bzero(&rxs, sizeof(rxs));
3224         rxs.r_flags |= IEEE80211_R_IEEE | IEEE80211_R_FREQ;
3225         rxs.r_flags |= IEEE80211_R_NF | IEEE80211_R_RSSI;
3226         rxs.c_ieee = le16toh(phy_info->channel);
3227         if (le16toh(phy_info->phy_flags & IWM_RX_RES_PHY_FLAGS_BAND_24)) {
3228                 rxs.c_freq = ieee80211_ieee2mhz(rxs.c_ieee, IEEE80211_CHAN_2GHZ);
3229         } else {
3230                 rxs.c_freq = ieee80211_ieee2mhz(rxs.c_ieee, IEEE80211_CHAN_5GHZ);
3231         }
3232
3233         /* rssi is in 1/2db units */
3234         rxs.rssi = rssi * 2;
3235         rxs.nf = sc->sc_noise;
3236
3237         if (ieee80211_radiotap_active_vap(vap)) {
3238                 struct iwm_rx_radiotap_header *tap = &sc->sc_rxtap;
3239
3240                 tap->wr_flags = 0;
3241                 if (phy_info->phy_flags & htole16(IWM_PHY_INFO_FLAG_SHPREAMBLE))
3242                         tap->wr_flags |= IEEE80211_RADIOTAP_F_SHORTPRE;
3243                 tap->wr_chan_freq = htole16(rxs.c_freq);
3244                 /* XXX only if ic->ic_curchan->ic_ieee == rxs.c_ieee */
3245                 tap->wr_chan_flags = htole16(ic->ic_curchan->ic_flags);
3246                 tap->wr_dbm_antsignal = (int8_t)rssi;
3247                 tap->wr_dbm_antnoise = (int8_t)sc->sc_noise;
3248                 tap->wr_tsft = phy_info->system_timestamp;
3249                 switch (phy_info->rate) {
3250                 /* CCK rates. */
3251                 case  10: tap->wr_rate =   2; break;
3252                 case  20: tap->wr_rate =   4; break;
3253                 case  55: tap->wr_rate =  11; break;
3254                 case 110: tap->wr_rate =  22; break;
3255                 /* OFDM rates. */
3256                 case 0xd: tap->wr_rate =  12; break;
3257                 case 0xf: tap->wr_rate =  18; break;
3258                 case 0x5: tap->wr_rate =  24; break;
3259                 case 0x7: tap->wr_rate =  36; break;
3260                 case 0x9: tap->wr_rate =  48; break;
3261                 case 0xb: tap->wr_rate =  72; break;
3262                 case 0x1: tap->wr_rate =  96; break;
3263                 case 0x3: tap->wr_rate = 108; break;
3264                 /* Unknown rate: should not happen. */
3265                 default:  tap->wr_rate =   0;
3266                 }
3267         }
3268
3269         IWM_UNLOCK(sc);
3270         if (ni != NULL) {
3271                 IWM_DPRINTF(sc, IWM_DEBUG_RECV, "input m %p\n", m);
3272                 ieee80211_input_mimo(ni, m, &rxs);
3273                 ieee80211_free_node(ni);
3274         } else {
3275                 IWM_DPRINTF(sc, IWM_DEBUG_RECV, "inputall m %p\n", m);
3276                 ieee80211_input_mimo_all(ic, m, &rxs);
3277         }
3278         IWM_LOCK(sc);
3279
3280         return;
3281
3282 fail:   counter_u64_add(ic->ic_ierrors, 1);
3283 }
3284
3285 static int
3286 iwm_mvm_rx_tx_cmd_single(struct iwm_softc *sc, struct iwm_rx_packet *pkt,
3287         struct iwm_node *in)
3288 {
3289         struct iwm_mvm_tx_resp *tx_resp = (void *)pkt->data;
3290         struct ieee80211_node *ni = &in->in_ni;
3291         struct ieee80211vap *vap = ni->ni_vap;
3292         int status = le16toh(tx_resp->status.status) & IWM_TX_STATUS_MSK;
3293         int failack = tx_resp->failure_frame;
3294
3295         KASSERT(tx_resp->frame_count == 1, ("too many frames"));
3296
3297         /* Update rate control statistics. */
3298         IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "%s: status=0x%04x, seq=%d, fc=%d, btc=%d, frts=%d, ff=%d, irate=%08x, wmt=%d\n",
3299             __func__,
3300             (int) le16toh(tx_resp->status.status),
3301             (int) le16toh(tx_resp->status.sequence),
3302             tx_resp->frame_count,
3303             tx_resp->bt_kill_count,
3304             tx_resp->failure_rts,
3305             tx_resp->failure_frame,
3306             le32toh(tx_resp->initial_rate),
3307             (int) le16toh(tx_resp->wireless_media_time));
3308
3309         if (status != IWM_TX_STATUS_SUCCESS &&
3310             status != IWM_TX_STATUS_DIRECT_DONE) {
3311                 ieee80211_ratectl_tx_complete(vap, ni,
3312                     IEEE80211_RATECTL_TX_FAILURE, &failack, NULL);
3313                 return (1);
3314         } else {
3315                 ieee80211_ratectl_tx_complete(vap, ni,
3316                     IEEE80211_RATECTL_TX_SUCCESS, &failack, NULL);
3317                 return (0);
3318         }
3319 }
3320
3321 static void
3322 iwm_mvm_rx_tx_cmd(struct iwm_softc *sc,
3323         struct iwm_rx_packet *pkt, struct iwm_rx_data *data)
3324 {
3325         struct iwm_cmd_header *cmd_hdr = &pkt->hdr;
3326         int idx = cmd_hdr->idx;
3327         int qid = cmd_hdr->qid;
3328         struct iwm_tx_ring *ring = &sc->txq[qid];
3329         struct iwm_tx_data *txd = &ring->data[idx];
3330         struct iwm_node *in = txd->in;
3331         struct mbuf *m = txd->m;
3332         int status;
3333
3334         KASSERT(txd->done == 0, ("txd not done"));
3335         KASSERT(txd->in != NULL, ("txd without node"));
3336         KASSERT(txd->m != NULL, ("txd without mbuf"));
3337
3338         bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_POSTREAD);
3339
3340         sc->sc_tx_timer = 0;
3341
3342         status = iwm_mvm_rx_tx_cmd_single(sc, pkt, in);
3343
3344         /* Unmap and free mbuf. */
3345         bus_dmamap_sync(ring->data_dmat, txd->map, BUS_DMASYNC_POSTWRITE);
3346         bus_dmamap_unload(ring->data_dmat, txd->map);
3347
3348         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3349             "free txd %p, in %p\n", txd, txd->in);
3350         txd->done = 1;
3351         txd->m = NULL;
3352         txd->in = NULL;
3353
3354         ieee80211_tx_complete(&in->in_ni, m, status);
3355
3356         if (--ring->queued < IWM_TX_RING_LOMARK) {
3357                 sc->qfullmsk &= ~(1 << ring->qid);
3358                 if (sc->qfullmsk == 0) {
3359                         iwm_start(sc);
3360                 }
3361         }
3362 }
3363
3364 /*
3365  * transmit side
3366  */
3367
3368 /*
3369  * Process a "command done" firmware notification.  This is where we wakeup
3370  * processes waiting for a synchronous command completion.
3371  * from if_iwn
3372  */
3373 static void
3374 iwm_cmd_done(struct iwm_softc *sc, struct iwm_rx_packet *pkt)
3375 {
3376         struct iwm_tx_ring *ring = &sc->txq[IWM_MVM_CMD_QUEUE];
3377         struct iwm_tx_data *data;
3378
3379         if (pkt->hdr.qid != IWM_MVM_CMD_QUEUE) {
3380                 return; /* Not a command ack. */
3381         }
3382
3383         /* XXX wide commands? */
3384         IWM_DPRINTF(sc, IWM_DEBUG_CMD,
3385             "cmd notification type 0x%x qid %d idx %d\n",
3386             pkt->hdr.code, pkt->hdr.qid, pkt->hdr.idx);
3387
3388         data = &ring->data[pkt->hdr.idx];
3389
3390         /* If the command was mapped in an mbuf, free it. */
3391         if (data->m != NULL) {
3392                 bus_dmamap_sync(ring->data_dmat, data->map,
3393                     BUS_DMASYNC_POSTWRITE);
3394                 bus_dmamap_unload(ring->data_dmat, data->map);
3395                 m_freem(data->m);
3396                 data->m = NULL;
3397         }
3398         wakeup(&ring->desc[pkt->hdr.idx]);
3399
3400         if (((pkt->hdr.idx + ring->queued) % IWM_TX_RING_COUNT) != ring->cur) {
3401                 device_printf(sc->sc_dev,
3402                     "%s: Some HCMDs skipped?: idx=%d queued=%d cur=%d\n",
3403                     __func__, pkt->hdr.idx, ring->queued, ring->cur);
3404                 /* XXX call iwm_force_nmi() */
3405         }
3406
3407         KASSERT(ring->queued > 0, ("ring->queued is empty?"));
3408         ring->queued--;
3409         if (ring->queued == 0)
3410                 iwm_pcie_clear_cmd_in_flight(sc);
3411 }
3412
3413 #if 0
3414 /*
3415  * necessary only for block ack mode
3416  */
3417 void
3418 iwm_update_sched(struct iwm_softc *sc, int qid, int idx, uint8_t sta_id,
3419         uint16_t len)
3420 {
3421         struct iwm_agn_scd_bc_tbl *scd_bc_tbl;
3422         uint16_t w_val;
3423
3424         scd_bc_tbl = sc->sched_dma.vaddr;
3425
3426         len += 8; /* magic numbers came naturally from paris */
3427         if (sc->sc_capaflags & IWM_UCODE_TLV_FLAGS_DW_BC_TABLE)
3428                 len = roundup(len, 4) / 4;
3429
3430         w_val = htole16(sta_id << 12 | len);
3431
3432         /* Update TX scheduler. */
3433         scd_bc_tbl[qid].tfd_offset[idx] = w_val;
3434         bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map,
3435             BUS_DMASYNC_PREWRITE);
3436
3437         /* I really wonder what this is ?!? */
3438         if (idx < IWM_TFD_QUEUE_SIZE_BC_DUP) {
3439                 scd_bc_tbl[qid].tfd_offset[IWM_TFD_QUEUE_SIZE_MAX + idx] = w_val;
3440                 bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map,
3441                     BUS_DMASYNC_PREWRITE);
3442         }
3443 }
3444 #endif
3445
3446 /*
3447  * Take an 802.11 (non-n) rate, find the relevant rate
3448  * table entry.  return the index into in_ridx[].
3449  *
3450  * The caller then uses that index back into in_ridx
3451  * to figure out the rate index programmed /into/
3452  * the firmware for this given node.
3453  */
3454 static int
3455 iwm_tx_rateidx_lookup(struct iwm_softc *sc, struct iwm_node *in,
3456     uint8_t rate)
3457 {
3458         int i;
3459         uint8_t r;
3460
3461         for (i = 0; i < nitems(in->in_ridx); i++) {
3462                 r = iwm_rates[in->in_ridx[i]].rate;
3463                 if (rate == r)
3464                         return (i);
3465         }
3466
3467         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE,
3468             "%s: couldn't find an entry for rate=%d\n",
3469             __func__,
3470             rate);
3471
3472         /* XXX Return the first */
3473         /* XXX TODO: have it return the /lowest/ */
3474         return (0);
3475 }
3476
3477 static int
3478 iwm_tx_rateidx_global_lookup(struct iwm_softc *sc, uint8_t rate)
3479 {
3480         int i;
3481
3482         for (i = 0; i < nitems(iwm_rates); i++) {
3483                 if (iwm_rates[i].rate == rate)
3484                         return (i);
3485         }
3486         /* XXX error? */
3487         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE,
3488             "%s: couldn't find an entry for rate=%d\n",
3489             __func__,
3490             rate);
3491         return (0);
3492 }
3493
3494 /*
3495  * Fill in the rate related information for a transmit command.
3496  */
3497 static const struct iwm_rate *
3498 iwm_tx_fill_cmd(struct iwm_softc *sc, struct iwm_node *in,
3499         struct mbuf *m, struct iwm_tx_cmd *tx)
3500 {
3501         struct ieee80211_node *ni = &in->in_ni;
3502         struct ieee80211_frame *wh;
3503         const struct ieee80211_txparam *tp = ni->ni_txparms;
3504         const struct iwm_rate *rinfo;
3505         int type;
3506         int ridx, rate_flags;
3507
3508         wh = mtod(m, struct ieee80211_frame *);
3509         type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK;
3510
3511         tx->rts_retry_limit = IWM_RTS_DFAULT_RETRY_LIMIT;
3512         tx->data_retry_limit = IWM_DEFAULT_TX_RETRY;
3513
3514         if (type == IEEE80211_FC0_TYPE_MGT) {
3515                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->mgmtrate);
3516                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3517                     "%s: MGT (%d)\n", __func__, tp->mgmtrate);
3518         } else if (IEEE80211_IS_MULTICAST(wh->i_addr1)) {
3519                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->mcastrate);
3520                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3521                     "%s: MCAST (%d)\n", __func__, tp->mcastrate);
3522         } else if (tp->ucastrate != IEEE80211_FIXED_RATE_NONE) {
3523                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->ucastrate);
3524                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3525                     "%s: FIXED_RATE (%d)\n", __func__, tp->ucastrate);
3526         } else if (m->m_flags & M_EAPOL) {
3527                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->mgmtrate);
3528                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3529                     "%s: EAPOL\n", __func__);
3530         } else if (type == IEEE80211_FC0_TYPE_DATA) {
3531                 int i;
3532
3533                 /* for data frames, use RS table */
3534                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: DATA\n", __func__);
3535                 /* XXX pass pktlen */
3536                 (void) ieee80211_ratectl_rate(ni, NULL, 0);
3537                 i = iwm_tx_rateidx_lookup(sc, in, ni->ni_txrate);
3538                 ridx = in->in_ridx[i];
3539
3540                 /* This is the index into the programmed table */
3541                 tx->initial_rate_index = i;
3542                 tx->tx_flags |= htole32(IWM_TX_CMD_FLG_STA_RATE);
3543
3544                 IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE,
3545                     "%s: start with i=%d, txrate %d\n",
3546                     __func__, i, iwm_rates[ridx].rate);
3547         } else {
3548                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->mgmtrate);
3549                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: DEFAULT (%d)\n",
3550                     __func__, tp->mgmtrate);
3551         }
3552
3553         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE,
3554             "%s: frame type=%d txrate %d\n",
3555                 __func__, type, iwm_rates[ridx].rate);
3556
3557         rinfo = &iwm_rates[ridx];
3558
3559         IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: ridx=%d; rate=%d, CCK=%d\n",
3560             __func__, ridx,
3561             rinfo->rate,
3562             !! (IWM_RIDX_IS_CCK(ridx))
3563             );
3564
3565         /* XXX TODO: hard-coded TX antenna? */
3566         rate_flags = 1 << IWM_RATE_MCS_ANT_POS;
3567         if (IWM_RIDX_IS_CCK(ridx))
3568                 rate_flags |= IWM_RATE_MCS_CCK_MSK;
3569         tx->rate_n_flags = htole32(rate_flags | rinfo->plcp);
3570
3571         return rinfo;
3572 }
3573
3574 #define TB0_SIZE 16
3575 static int
3576 iwm_tx(struct iwm_softc *sc, struct mbuf *m, struct ieee80211_node *ni, int ac)
3577 {
3578         struct ieee80211com *ic = &sc->sc_ic;
3579         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
3580         struct iwm_node *in = IWM_NODE(ni);
3581         struct iwm_tx_ring *ring;
3582         struct iwm_tx_data *data;
3583         struct iwm_tfd *desc;
3584         struct iwm_device_cmd *cmd;
3585         struct iwm_tx_cmd *tx;
3586         struct ieee80211_frame *wh;
3587         struct ieee80211_key *k = NULL;
3588         struct mbuf *m1;
3589         const struct iwm_rate *rinfo;
3590         uint32_t flags;
3591         u_int hdrlen;
3592         bus_dma_segment_t *seg, segs[IWM_MAX_SCATTER];
3593         int nsegs;
3594         uint8_t tid, type;
3595         int i, totlen, error, pad;
3596
3597         wh = mtod(m, struct ieee80211_frame *);
3598         hdrlen = ieee80211_anyhdrsize(wh);
3599         type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK;
3600         tid = 0;
3601         ring = &sc->txq[ac];
3602         desc = &ring->desc[ring->cur];
3603         memset(desc, 0, sizeof(*desc));
3604         data = &ring->data[ring->cur];
3605
3606         /* Fill out iwm_tx_cmd to send to the firmware */
3607         cmd = &ring->cmd[ring->cur];
3608         cmd->hdr.code = IWM_TX_CMD;
3609         cmd->hdr.flags = 0;
3610         cmd->hdr.qid = ring->qid;
3611         cmd->hdr.idx = ring->cur;
3612
3613         tx = (void *)cmd->data;
3614         memset(tx, 0, sizeof(*tx));
3615
3616         rinfo = iwm_tx_fill_cmd(sc, in, m, tx);
3617
3618         /* Encrypt the frame if need be. */
3619         if (wh->i_fc[1] & IEEE80211_FC1_PROTECTED) {
3620                 /* Retrieve key for TX && do software encryption. */
3621                 k = ieee80211_crypto_encap(ni, m);
3622                 if (k == NULL) {
3623                         m_freem(m);
3624                         return (ENOBUFS);
3625                 }
3626                 /* 802.11 header may have moved. */
3627                 wh = mtod(m, struct ieee80211_frame *);
3628         }
3629
3630         if (ieee80211_radiotap_active_vap(vap)) {
3631                 struct iwm_tx_radiotap_header *tap = &sc->sc_txtap;
3632
3633                 tap->wt_flags = 0;
3634                 tap->wt_chan_freq = htole16(ni->ni_chan->ic_freq);
3635                 tap->wt_chan_flags = htole16(ni->ni_chan->ic_flags);
3636                 tap->wt_rate = rinfo->rate;
3637                 if (k != NULL)
3638                         tap->wt_flags |= IEEE80211_RADIOTAP_F_WEP;
3639                 ieee80211_radiotap_tx(vap, m);
3640         }
3641
3642
3643         totlen = m->m_pkthdr.len;
3644
3645         flags = 0;
3646         if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) {
3647                 flags |= IWM_TX_CMD_FLG_ACK;
3648         }
3649
3650         if (type == IEEE80211_FC0_TYPE_DATA
3651             && (totlen + IEEE80211_CRC_LEN > vap->iv_rtsthreshold)
3652             && !IEEE80211_IS_MULTICAST(wh->i_addr1)) {
3653                 flags |= IWM_TX_CMD_FLG_PROT_REQUIRE;
3654         }
3655
3656         if (IEEE80211_IS_MULTICAST(wh->i_addr1) ||
3657             type != IEEE80211_FC0_TYPE_DATA)
3658                 tx->sta_id = sc->sc_aux_sta.sta_id;
3659         else
3660                 tx->sta_id = IWM_STATION_ID;
3661
3662         if (type == IEEE80211_FC0_TYPE_MGT) {
3663                 uint8_t subtype = wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK;
3664
3665                 if (subtype == IEEE80211_FC0_SUBTYPE_ASSOC_REQ ||
3666                     subtype == IEEE80211_FC0_SUBTYPE_REASSOC_REQ) {
3667                         tx->pm_frame_timeout = htole16(IWM_PM_FRAME_ASSOC);
3668                 } else if (subtype == IEEE80211_FC0_SUBTYPE_ACTION) {
3669                         tx->pm_frame_timeout = htole16(IWM_PM_FRAME_NONE);
3670                 } else {
3671                         tx->pm_frame_timeout = htole16(IWM_PM_FRAME_MGMT);
3672                 }
3673         } else {
3674                 tx->pm_frame_timeout = htole16(IWM_PM_FRAME_NONE);
3675         }
3676
3677         if (hdrlen & 3) {
3678                 /* First segment length must be a multiple of 4. */
3679                 flags |= IWM_TX_CMD_FLG_MH_PAD;
3680                 pad = 4 - (hdrlen & 3);
3681         } else
3682                 pad = 0;
3683
3684         tx->driver_txop = 0;
3685         tx->next_frame_len = 0;
3686
3687         tx->len = htole16(totlen);
3688         tx->tid_tspec = tid;
3689         tx->life_time = htole32(IWM_TX_CMD_LIFE_TIME_INFINITE);
3690
3691         /* Set physical address of "scratch area". */
3692         tx->dram_lsb_ptr = htole32(data->scratch_paddr);
3693         tx->dram_msb_ptr = iwm_get_dma_hi_addr(data->scratch_paddr);
3694
3695         /* Copy 802.11 header in TX command. */
3696         memcpy(((uint8_t *)tx) + sizeof(*tx), wh, hdrlen);
3697
3698         flags |= IWM_TX_CMD_FLG_BT_DIS | IWM_TX_CMD_FLG_SEQ_CTL;
3699
3700         tx->sec_ctl = 0;
3701         tx->tx_flags |= htole32(flags);
3702
3703         /* Trim 802.11 header. */
3704         m_adj(m, hdrlen);
3705         error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m,
3706             segs, &nsegs, BUS_DMA_NOWAIT);
3707         if (error != 0) {
3708                 if (error != EFBIG) {
3709                         device_printf(sc->sc_dev, "can't map mbuf (error %d)\n",
3710                             error);
3711                         m_freem(m);
3712                         return error;
3713                 }
3714                 /* Too many DMA segments, linearize mbuf. */
3715                 m1 = m_collapse(m, M_NOWAIT, IWM_MAX_SCATTER - 2);
3716                 if (m1 == NULL) {
3717                         device_printf(sc->sc_dev,
3718                             "%s: could not defrag mbuf\n", __func__);
3719                         m_freem(m);
3720                         return (ENOBUFS);
3721                 }
3722                 m = m1;
3723
3724                 error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m,
3725                     segs, &nsegs, BUS_DMA_NOWAIT);
3726                 if (error != 0) {
3727                         device_printf(sc->sc_dev, "can't map mbuf (error %d)\n",
3728                             error);
3729                         m_freem(m);
3730                         return error;
3731                 }
3732         }
3733         data->m = m;
3734         data->in = in;
3735         data->done = 0;
3736
3737         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3738             "sending txd %p, in %p\n", data, data->in);
3739         KASSERT(data->in != NULL, ("node is NULL"));
3740
3741         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3742             "sending data: qid=%d idx=%d len=%d nsegs=%d txflags=0x%08x rate_n_flags=0x%08x rateidx=%u\n",
3743             ring->qid, ring->cur, totlen, nsegs,
3744             le32toh(tx->tx_flags),
3745             le32toh(tx->rate_n_flags),
3746             tx->initial_rate_index
3747             );
3748
3749         /* Fill TX descriptor. */
3750         desc->num_tbs = 2 + nsegs;
3751
3752         desc->tbs[0].lo = htole32(data->cmd_paddr);
3753         desc->tbs[0].hi_n_len = htole16(iwm_get_dma_hi_addr(data->cmd_paddr)) |
3754             (TB0_SIZE << 4);
3755         desc->tbs[1].lo = htole32(data->cmd_paddr + TB0_SIZE);
3756         desc->tbs[1].hi_n_len = htole16(iwm_get_dma_hi_addr(data->cmd_paddr)) |
3757             ((sizeof(struct iwm_cmd_header) + sizeof(*tx)
3758               + hdrlen + pad - TB0_SIZE) << 4);
3759
3760         /* Other DMA segments are for data payload. */
3761         for (i = 0; i < nsegs; i++) {
3762                 seg = &segs[i];
3763                 desc->tbs[i+2].lo = htole32(seg->ds_addr);
3764                 desc->tbs[i+2].hi_n_len = \
3765                     htole16(iwm_get_dma_hi_addr(seg->ds_addr))
3766                     | ((seg->ds_len) << 4);
3767         }
3768
3769         bus_dmamap_sync(ring->data_dmat, data->map,
3770             BUS_DMASYNC_PREWRITE);
3771         bus_dmamap_sync(ring->cmd_dma.tag, ring->cmd_dma.map,
3772             BUS_DMASYNC_PREWRITE);
3773         bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map,
3774             BUS_DMASYNC_PREWRITE);
3775
3776 #if 0
3777         iwm_update_sched(sc, ring->qid, ring->cur, tx->sta_id, le16toh(tx->len));
3778 #endif
3779
3780         /* Kick TX ring. */
3781         ring->cur = (ring->cur + 1) % IWM_TX_RING_COUNT;
3782         IWM_WRITE(sc, IWM_HBUS_TARG_WRPTR, ring->qid << 8 | ring->cur);
3783
3784         /* Mark TX ring as full if we reach a certain threshold. */
3785         if (++ring->queued > IWM_TX_RING_HIMARK) {
3786                 sc->qfullmsk |= 1 << ring->qid;
3787         }
3788
3789         return 0;
3790 }
3791
3792 static int
3793 iwm_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
3794     const struct ieee80211_bpf_params *params)
3795 {
3796         struct ieee80211com *ic = ni->ni_ic;
3797         struct iwm_softc *sc = ic->ic_softc;
3798         int error = 0;
3799
3800         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3801             "->%s begin\n", __func__);
3802
3803         if ((sc->sc_flags & IWM_FLAG_HW_INITED) == 0) {
3804                 m_freem(m);
3805                 IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3806                     "<-%s not RUNNING\n", __func__);
3807                 return (ENETDOWN);
3808         }
3809
3810         IWM_LOCK(sc);
3811         /* XXX fix this */
3812         if (params == NULL) {
3813                 error = iwm_tx(sc, m, ni, 0);
3814         } else {
3815                 error = iwm_tx(sc, m, ni, 0);
3816         }
3817         sc->sc_tx_timer = 5;
3818         IWM_UNLOCK(sc);
3819
3820         return (error);
3821 }
3822
3823 /*
3824  * mvm/tx.c
3825  */
3826
3827 /*
3828  * Note that there are transports that buffer frames before they reach
3829  * the firmware. This means that after flush_tx_path is called, the
3830  * queue might not be empty. The race-free way to handle this is to:
3831  * 1) set the station as draining
3832  * 2) flush the Tx path
3833  * 3) wait for the transport queues to be empty
3834  */
3835 int
3836 iwm_mvm_flush_tx_path(struct iwm_softc *sc, uint32_t tfd_msk, uint32_t flags)
3837 {
3838         int ret;
3839         struct iwm_tx_path_flush_cmd flush_cmd = {
3840                 .queues_ctl = htole32(tfd_msk),
3841                 .flush_ctl = htole16(IWM_DUMP_TX_FIFO_FLUSH),
3842         };
3843
3844         ret = iwm_mvm_send_cmd_pdu(sc, IWM_TXPATH_FLUSH, flags,
3845             sizeof(flush_cmd), &flush_cmd);
3846         if (ret)
3847                 device_printf(sc->sc_dev,
3848                     "Flushing tx queue failed: %d\n", ret);
3849         return ret;
3850 }
3851
3852 /*
3853  * BEGIN mvm/sta.c
3854  */
3855
3856 static int
3857 iwm_mvm_send_add_sta_cmd_status(struct iwm_softc *sc,
3858         struct iwm_mvm_add_sta_cmd *cmd, int *status)
3859 {
3860         return iwm_mvm_send_cmd_pdu_status(sc, IWM_ADD_STA, sizeof(*cmd),
3861             cmd, status);
3862 }
3863
3864 /* send station add/update command to firmware */
3865 static int
3866 iwm_mvm_sta_send_to_fw(struct iwm_softc *sc, struct iwm_node *in, int update)
3867 {
3868         struct iwm_mvm_add_sta_cmd add_sta_cmd;
3869         int ret;
3870         uint32_t status;
3871
3872         memset(&add_sta_cmd, 0, sizeof(add_sta_cmd));
3873
3874         add_sta_cmd.sta_id = IWM_STATION_ID;
3875         add_sta_cmd.mac_id_n_color
3876             = htole32(IWM_FW_CMD_ID_AND_COLOR(IWM_DEFAULT_MACID,
3877                 IWM_DEFAULT_COLOR));
3878         if (!update) {
3879                 int ac;
3880                 for (ac = 0; ac < WME_NUM_AC; ac++) {
3881                         add_sta_cmd.tfd_queue_msk |=
3882                             htole32(1 << iwm_mvm_ac_to_tx_fifo[ac]);
3883                 }
3884                 IEEE80211_ADDR_COPY(&add_sta_cmd.addr, in->in_ni.ni_bssid);
3885         }
3886         add_sta_cmd.add_modify = update ? 1 : 0;
3887         add_sta_cmd.station_flags_msk
3888             |= htole32(IWM_STA_FLG_FAT_EN_MSK | IWM_STA_FLG_MIMO_EN_MSK);
3889         add_sta_cmd.tid_disable_tx = htole16(0xffff);
3890         if (update)
3891                 add_sta_cmd.modify_mask |= (IWM_STA_MODIFY_TID_DISABLE_TX);
3892
3893         status = IWM_ADD_STA_SUCCESS;
3894         ret = iwm_mvm_send_add_sta_cmd_status(sc, &add_sta_cmd, &status);
3895         if (ret)
3896                 return ret;
3897
3898         switch (status) {
3899         case IWM_ADD_STA_SUCCESS:
3900                 break;
3901         default:
3902                 ret = EIO;
3903                 device_printf(sc->sc_dev, "IWM_ADD_STA failed\n");
3904                 break;
3905         }
3906
3907         return ret;
3908 }
3909
3910 static int
3911 iwm_mvm_add_sta(struct iwm_softc *sc, struct iwm_node *in)
3912 {
3913         return iwm_mvm_sta_send_to_fw(sc, in, 0);
3914 }
3915
3916 static int
3917 iwm_mvm_update_sta(struct iwm_softc *sc, struct iwm_node *in)
3918 {
3919         return iwm_mvm_sta_send_to_fw(sc, in, 1);
3920 }
3921
3922 static int
3923 iwm_mvm_add_int_sta_common(struct iwm_softc *sc, struct iwm_int_sta *sta,
3924         const uint8_t *addr, uint16_t mac_id, uint16_t color)
3925 {
3926         struct iwm_mvm_add_sta_cmd cmd;
3927         int ret;
3928         uint32_t status;
3929
3930         memset(&cmd, 0, sizeof(cmd));
3931         cmd.sta_id = sta->sta_id;
3932         cmd.mac_id_n_color = htole32(IWM_FW_CMD_ID_AND_COLOR(mac_id, color));
3933
3934         cmd.tfd_queue_msk = htole32(sta->tfd_queue_msk);
3935         cmd.tid_disable_tx = htole16(0xffff);
3936
3937         if (addr)
3938                 IEEE80211_ADDR_COPY(cmd.addr, addr);
3939
3940         ret = iwm_mvm_send_add_sta_cmd_status(sc, &cmd, &status);
3941         if (ret)
3942                 return ret;
3943
3944         switch (status) {
3945         case IWM_ADD_STA_SUCCESS:
3946                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
3947                     "%s: Internal station added.\n", __func__);
3948                 return 0;
3949         default:
3950                 device_printf(sc->sc_dev,
3951                     "%s: Add internal station failed, status=0x%x\n",
3952                     __func__, status);
3953                 ret = EIO;
3954                 break;
3955         }
3956         return ret;
3957 }
3958
3959 static int
3960 iwm_mvm_add_aux_sta(struct iwm_softc *sc)
3961 {
3962         int ret;
3963
3964         sc->sc_aux_sta.sta_id = IWM_AUX_STA_ID;
3965         sc->sc_aux_sta.tfd_queue_msk = (1 << IWM_MVM_AUX_QUEUE);
3966
3967         ret = iwm_enable_txq(sc, 0, IWM_MVM_AUX_QUEUE, IWM_MVM_TX_FIFO_MCAST);
3968         if (ret)
3969                 return ret;
3970
3971         ret = iwm_mvm_add_int_sta_common(sc,
3972             &sc->sc_aux_sta, NULL, IWM_MAC_INDEX_AUX, 0);
3973
3974         if (ret)
3975                 memset(&sc->sc_aux_sta, 0, sizeof(sc->sc_aux_sta));
3976         return ret;
3977 }
3978
3979 /*
3980  * END mvm/sta.c
3981  */
3982
3983 /*
3984  * BEGIN mvm/quota.c
3985  */
3986
3987 static int
3988 iwm_mvm_update_quotas(struct iwm_softc *sc, struct iwm_node *in)
3989 {
3990         struct iwm_time_quota_cmd cmd;
3991         int i, idx, ret, num_active_macs, quota, quota_rem;
3992         int colors[IWM_MAX_BINDINGS] = { -1, -1, -1, -1, };
3993         int n_ifs[IWM_MAX_BINDINGS] = {0, };
3994         uint16_t id;
3995
3996         memset(&cmd, 0, sizeof(cmd));
3997
3998         /* currently, PHY ID == binding ID */
3999         if (in) {
4000                 id = in->in_phyctxt->id;
4001                 KASSERT(id < IWM_MAX_BINDINGS, ("invalid id"));
4002                 colors[id] = in->in_phyctxt->color;
4003
4004                 if (1)
4005                         n_ifs[id] = 1;
4006         }
4007
4008         /*
4009          * The FW's scheduling session consists of
4010          * IWM_MVM_MAX_QUOTA fragments. Divide these fragments
4011          * equally between all the bindings that require quota
4012          */
4013         num_active_macs = 0;
4014         for (i = 0; i < IWM_MAX_BINDINGS; i++) {
4015                 cmd.quotas[i].id_and_color = htole32(IWM_FW_CTXT_INVALID);
4016                 num_active_macs += n_ifs[i];
4017         }
4018
4019         quota = 0;
4020         quota_rem = 0;
4021         if (num_active_macs) {
4022                 quota = IWM_MVM_MAX_QUOTA / num_active_macs;
4023                 quota_rem = IWM_MVM_MAX_QUOTA % num_active_macs;
4024         }
4025
4026         for (idx = 0, i = 0; i < IWM_MAX_BINDINGS; i++) {
4027                 if (colors[i] < 0)
4028                         continue;
4029
4030                 cmd.quotas[idx].id_and_color =
4031                         htole32(IWM_FW_CMD_ID_AND_COLOR(i, colors[i]));
4032
4033                 if (n_ifs[i] <= 0) {
4034                         cmd.quotas[idx].quota = htole32(0);
4035                         cmd.quotas[idx].max_duration = htole32(0);
4036                 } else {
4037                         cmd.quotas[idx].quota = htole32(quota * n_ifs[i]);
4038                         cmd.quotas[idx].max_duration = htole32(0);
4039                 }
4040                 idx++;
4041         }
4042
4043         /* Give the remainder of the session to the first binding */
4044         cmd.quotas[0].quota = htole32(le32toh(cmd.quotas[0].quota) + quota_rem);
4045
4046         ret = iwm_mvm_send_cmd_pdu(sc, IWM_TIME_QUOTA_CMD, IWM_CMD_SYNC,
4047             sizeof(cmd), &cmd);
4048         if (ret)
4049                 device_printf(sc->sc_dev,
4050                     "%s: Failed to send quota: %d\n", __func__, ret);
4051         return ret;
4052 }
4053
4054 /*
4055  * END mvm/quota.c
4056  */
4057
4058 /*
4059  * ieee80211 routines
4060  */
4061
4062 /*
4063  * Change to AUTH state in 80211 state machine.  Roughly matches what
4064  * Linux does in bss_info_changed().
4065  */
4066 static int
4067 iwm_auth(struct ieee80211vap *vap, struct iwm_softc *sc)
4068 {
4069         struct ieee80211_node *ni;
4070         struct iwm_node *in;
4071         struct iwm_vap *iv = IWM_VAP(vap);
4072         uint32_t duration;
4073         int error;
4074
4075         /*
4076          * XXX i have a feeling that the vap node is being
4077          * freed from underneath us. Grr.
4078          */
4079         ni = ieee80211_ref_node(vap->iv_bss);
4080         in = IWM_NODE(ni);
4081         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_STATE,
4082             "%s: called; vap=%p, bss ni=%p\n",
4083             __func__,
4084             vap,
4085             ni);
4086
4087         in->in_assoc = 0;
4088
4089         error = iwm_mvm_sf_config(sc, IWM_SF_FULL_ON);
4090         if (error != 0)
4091                 return error;
4092
4093         error = iwm_allow_mcast(vap, sc);
4094         if (error) {
4095                 device_printf(sc->sc_dev,
4096                     "%s: failed to set multicast\n", __func__);
4097                 goto out;
4098         }
4099
4100         /*
4101          * This is where it deviates from what Linux does.
4102          *
4103          * Linux iwlwifi doesn't reset the nic each time, nor does it
4104          * call ctxt_add() here.  Instead, it adds it during vap creation,
4105          * and always does a mac_ctx_changed().
4106          *
4107          * The openbsd port doesn't attempt to do that - it reset things
4108          * at odd states and does the add here.
4109          *
4110          * So, until the state handling is fixed (ie, we never reset
4111          * the NIC except for a firmware failure, which should drag
4112          * the NIC back to IDLE, re-setup and re-add all the mac/phy
4113          * contexts that are required), let's do a dirty hack here.
4114          */
4115         if (iv->is_uploaded) {
4116                 if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) {
4117                         device_printf(sc->sc_dev,
4118                             "%s: failed to update MAC\n", __func__);
4119                         goto out;
4120                 }
4121                 if ((error = iwm_mvm_phy_ctxt_changed(sc, &sc->sc_phyctxt[0],
4122                     in->in_ni.ni_chan, 1, 1)) != 0) {
4123                         device_printf(sc->sc_dev,
4124                             "%s: failed update phy ctxt\n", __func__);
4125                         goto out;
4126                 }
4127                 in->in_phyctxt = &sc->sc_phyctxt[0];
4128
4129                 if ((error = iwm_mvm_binding_update(sc, in)) != 0) {
4130                         device_printf(sc->sc_dev,
4131                             "%s: binding update cmd\n", __func__);
4132                         goto out;
4133                 }
4134                 if ((error = iwm_mvm_update_sta(sc, in)) != 0) {
4135                         device_printf(sc->sc_dev,
4136                             "%s: failed to update sta\n", __func__);
4137                         goto out;
4138                 }
4139         } else {
4140                 if ((error = iwm_mvm_mac_ctxt_add(sc, vap)) != 0) {
4141                         device_printf(sc->sc_dev,
4142                             "%s: failed to add MAC\n", __func__);
4143                         goto out;
4144                 }
4145                 if ((error = iwm_mvm_phy_ctxt_changed(sc, &sc->sc_phyctxt[0],
4146                     in->in_ni.ni_chan, 1, 1)) != 0) {
4147                         device_printf(sc->sc_dev,
4148                             "%s: failed add phy ctxt!\n", __func__);
4149                         error = ETIMEDOUT;
4150                         goto out;
4151                 }
4152                 in->in_phyctxt = &sc->sc_phyctxt[0];
4153
4154                 if ((error = iwm_mvm_binding_add_vif(sc, in)) != 0) {
4155                         device_printf(sc->sc_dev,
4156                             "%s: binding add cmd\n", __func__);
4157                         goto out;
4158                 }
4159                 if ((error = iwm_mvm_add_sta(sc, in)) != 0) {
4160                         device_printf(sc->sc_dev,
4161                             "%s: failed to add sta\n", __func__);
4162                         goto out;
4163                 }
4164         }
4165
4166         /*
4167          * Prevent the FW from wandering off channel during association
4168          * by "protecting" the session with a time event.
4169          */
4170         /* XXX duration is in units of TU, not MS */
4171         duration = IWM_MVM_TE_SESSION_PROTECTION_MAX_TIME_MS;
4172         iwm_mvm_protect_session(sc, in, duration, 500 /* XXX magic number */);
4173         DELAY(100);
4174
4175         error = 0;
4176 out:
4177         ieee80211_free_node(ni);
4178         return (error);
4179 }
4180
4181 static int
4182 iwm_assoc(struct ieee80211vap *vap, struct iwm_softc *sc)
4183 {
4184         struct iwm_node *in = IWM_NODE(vap->iv_bss);
4185         int error;
4186
4187         if ((error = iwm_mvm_update_sta(sc, in)) != 0) {
4188                 device_printf(sc->sc_dev,
4189                     "%s: failed to update STA\n", __func__);
4190                 return error;
4191         }
4192
4193         in->in_assoc = 1;
4194         if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) {
4195                 device_printf(sc->sc_dev,
4196                     "%s: failed to update MAC\n", __func__);
4197                 return error;
4198         }
4199
4200         return 0;
4201 }
4202
4203 static int
4204 iwm_release(struct iwm_softc *sc, struct iwm_node *in)
4205 {
4206         uint32_t tfd_msk;
4207
4208         /*
4209          * Ok, so *technically* the proper set of calls for going
4210          * from RUN back to SCAN is:
4211          *
4212          * iwm_mvm_power_mac_disable(sc, in);
4213          * iwm_mvm_mac_ctxt_changed(sc, in);
4214          * iwm_mvm_rm_sta(sc, in);
4215          * iwm_mvm_update_quotas(sc, NULL);
4216          * iwm_mvm_mac_ctxt_changed(sc, in);
4217          * iwm_mvm_binding_remove_vif(sc, in);
4218          * iwm_mvm_mac_ctxt_remove(sc, in);
4219          *
4220          * However, that freezes the device not matter which permutations
4221          * and modifications are attempted.  Obviously, this driver is missing
4222          * something since it works in the Linux driver, but figuring out what
4223          * is missing is a little more complicated.  Now, since we're going
4224          * back to nothing anyway, we'll just do a complete device reset.
4225          * Up your's, device!
4226          */
4227         /*
4228          * Just using 0xf for the queues mask is fine as long as we only
4229          * get here from RUN state.
4230          */
4231         tfd_msk = 0xf;
4232         mbufq_drain(&sc->sc_snd);
4233         iwm_mvm_flush_tx_path(sc, tfd_msk, IWM_CMD_SYNC);
4234         /*
4235          * We seem to get away with just synchronously sending the
4236          * IWM_TXPATH_FLUSH command.
4237          */
4238 //      iwm_trans_wait_tx_queue_empty(sc, tfd_msk);
4239         iwm_stop_device(sc);
4240         iwm_init_hw(sc);
4241         if (in)
4242                 in->in_assoc = 0;
4243         return 0;
4244
4245 #if 0
4246         int error;
4247
4248         iwm_mvm_power_mac_disable(sc, in);
4249
4250         if ((error = iwm_mvm_mac_ctxt_changed(sc, in)) != 0) {
4251                 device_printf(sc->sc_dev, "mac ctxt change fail 1 %d\n", error);
4252                 return error;
4253         }
4254
4255         if ((error = iwm_mvm_rm_sta(sc, in)) != 0) {
4256                 device_printf(sc->sc_dev, "sta remove fail %d\n", error);
4257                 return error;
4258         }
4259         error = iwm_mvm_rm_sta(sc, in);
4260         in->in_assoc = 0;
4261         iwm_mvm_update_quotas(sc, NULL);
4262         if ((error = iwm_mvm_mac_ctxt_changed(sc, in)) != 0) {
4263                 device_printf(sc->sc_dev, "mac ctxt change fail 2 %d\n", error);
4264                 return error;
4265         }
4266         iwm_mvm_binding_remove_vif(sc, in);
4267
4268         iwm_mvm_mac_ctxt_remove(sc, in);
4269
4270         return error;
4271 #endif
4272 }
4273
4274 static struct ieee80211_node *
4275 iwm_node_alloc(struct ieee80211vap *vap, const uint8_t mac[IEEE80211_ADDR_LEN])
4276 {
4277         return malloc(sizeof (struct iwm_node), M_80211_NODE,
4278             M_NOWAIT | M_ZERO);
4279 }
4280
4281 static void
4282 iwm_setrates(struct iwm_softc *sc, struct iwm_node *in)
4283 {
4284         struct ieee80211_node *ni = &in->in_ni;
4285         struct iwm_lq_cmd *lq = &in->in_lq;
4286         int nrates = ni->ni_rates.rs_nrates;
4287         int i, ridx, tab = 0;
4288 //      int txant = 0;
4289
4290         if (nrates > nitems(lq->rs_table)) {
4291                 device_printf(sc->sc_dev,
4292                     "%s: node supports %d rates, driver handles "
4293                     "only %zu\n", __func__, nrates, nitems(lq->rs_table));
4294                 return;
4295         }
4296         if (nrates == 0) {
4297                 device_printf(sc->sc_dev,
4298                     "%s: node supports 0 rates, odd!\n", __func__);
4299                 return;
4300         }
4301
4302         /*
4303          * XXX .. and most of iwm_node is not initialised explicitly;
4304          * it's all just 0x0 passed to the firmware.
4305          */
4306
4307         /* first figure out which rates we should support */
4308         /* XXX TODO: this isn't 11n aware /at all/ */
4309         memset(&in->in_ridx, -1, sizeof(in->in_ridx));
4310         IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
4311             "%s: nrates=%d\n", __func__, nrates);
4312
4313         /*
4314          * Loop over nrates and populate in_ridx from the highest
4315          * rate to the lowest rate.  Remember, in_ridx[] has
4316          * IEEE80211_RATE_MAXSIZE entries!
4317          */
4318         for (i = 0; i < min(nrates, IEEE80211_RATE_MAXSIZE); i++) {
4319                 int rate = ni->ni_rates.rs_rates[(nrates - 1) - i] & IEEE80211_RATE_VAL;
4320
4321                 /* Map 802.11 rate to HW rate index. */
4322                 for (ridx = 0; ridx <= IWM_RIDX_MAX; ridx++)
4323                         if (iwm_rates[ridx].rate == rate)
4324                                 break;
4325                 if (ridx > IWM_RIDX_MAX) {
4326                         device_printf(sc->sc_dev,
4327                             "%s: WARNING: device rate for %d not found!\n",
4328                             __func__, rate);
4329                 } else {
4330                         IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
4331                             "%s: rate: i: %d, rate=%d, ridx=%d\n",
4332                             __func__,
4333                             i,
4334                             rate,
4335                             ridx);
4336                         in->in_ridx[i] = ridx;
4337                 }
4338         }
4339
4340         /* then construct a lq_cmd based on those */
4341         memset(lq, 0, sizeof(*lq));
4342         lq->sta_id = IWM_STATION_ID;
4343
4344         /* For HT, always enable RTS/CTS to avoid excessive retries. */
4345         if (ni->ni_flags & IEEE80211_NODE_HT)
4346                 lq->flags |= IWM_LQ_FLAG_USE_RTS_MSK;
4347
4348         /*
4349          * are these used? (we don't do SISO or MIMO)
4350          * need to set them to non-zero, though, or we get an error.
4351          */
4352         lq->single_stream_ant_msk = 1;
4353         lq->dual_stream_ant_msk = 1;
4354
4355         /*
4356          * Build the actual rate selection table.
4357          * The lowest bits are the rates.  Additionally,
4358          * CCK needs bit 9 to be set.  The rest of the bits
4359          * we add to the table select the tx antenna
4360          * Note that we add the rates in the highest rate first
4361          * (opposite of ni_rates).
4362          */
4363         /*
4364          * XXX TODO: this should be looping over the min of nrates
4365          * and LQ_MAX_RETRY_NUM.  Sigh.
4366          */
4367         for (i = 0; i < nrates; i++) {
4368                 int nextant;
4369
4370 #if 0
4371                 if (txant == 0)
4372                         txant = iwm_mvm_get_valid_tx_ant(sc);
4373                 nextant = 1<<(ffs(txant)-1);
4374                 txant &= ~nextant;
4375 #else
4376                 nextant = iwm_mvm_get_valid_tx_ant(sc);
4377 #endif
4378                 /*
4379                  * Map the rate id into a rate index into
4380                  * our hardware table containing the
4381                  * configuration to use for this rate.
4382                  */
4383                 ridx = in->in_ridx[i];
4384                 tab = iwm_rates[ridx].plcp;
4385                 tab |= nextant << IWM_RATE_MCS_ANT_POS;
4386                 if (IWM_RIDX_IS_CCK(ridx))
4387                         tab |= IWM_RATE_MCS_CCK_MSK;
4388                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
4389                     "station rate i=%d, rate=%d, hw=%x\n",
4390                     i, iwm_rates[ridx].rate, tab);
4391                 lq->rs_table[i] = htole32(tab);
4392         }
4393         /* then fill the rest with the lowest possible rate */
4394         for (i = nrates; i < nitems(lq->rs_table); i++) {
4395                 KASSERT(tab != 0, ("invalid tab"));
4396                 lq->rs_table[i] = htole32(tab);
4397         }
4398 }
4399
4400 static int
4401 iwm_media_change(struct ifnet *ifp)
4402 {
4403         struct ieee80211vap *vap = ifp->if_softc;
4404         struct ieee80211com *ic = vap->iv_ic;
4405         struct iwm_softc *sc = ic->ic_softc;
4406         int error;
4407
4408         error = ieee80211_media_change(ifp);
4409         if (error != ENETRESET)
4410                 return error;
4411
4412         IWM_LOCK(sc);
4413         if (ic->ic_nrunning > 0) {
4414                 iwm_stop(sc);
4415                 iwm_init(sc);
4416         }
4417         IWM_UNLOCK(sc);
4418         return error;
4419 }
4420
4421
4422 static int
4423 iwm_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
4424 {
4425         struct iwm_vap *ivp = IWM_VAP(vap);
4426         struct ieee80211com *ic = vap->iv_ic;
4427         struct iwm_softc *sc = ic->ic_softc;
4428         struct iwm_node *in;
4429         int error;
4430
4431         IWM_DPRINTF(sc, IWM_DEBUG_STATE,
4432             "switching state %s -> %s\n",
4433             ieee80211_state_name[vap->iv_state],
4434             ieee80211_state_name[nstate]);
4435         IEEE80211_UNLOCK(ic);
4436         IWM_LOCK(sc);
4437
4438         if (vap->iv_state == IEEE80211_S_SCAN && nstate != vap->iv_state)
4439                 iwm_led_blink_stop(sc);
4440
4441         /* disable beacon filtering if we're hopping out of RUN */
4442         if (vap->iv_state == IEEE80211_S_RUN && nstate != vap->iv_state) {
4443                 iwm_mvm_disable_beacon_filter(sc);
4444
4445                 if (((in = IWM_NODE(vap->iv_bss)) != NULL))
4446                         in->in_assoc = 0;
4447
4448                 if (nstate == IEEE80211_S_INIT) {
4449                         IWM_UNLOCK(sc);
4450                         IEEE80211_LOCK(ic);
4451                         error = ivp->iv_newstate(vap, nstate, arg);
4452                         IEEE80211_UNLOCK(ic);
4453                         IWM_LOCK(sc);
4454                         iwm_release(sc, NULL);
4455                         IWM_UNLOCK(sc);
4456                         IEEE80211_LOCK(ic);
4457                         return error;
4458                 }
4459
4460                 /*
4461                  * It's impossible to directly go RUN->SCAN. If we iwm_release()
4462                  * above then the card will be completely reinitialized,
4463                  * so the driver must do everything necessary to bring the card
4464                  * from INIT to SCAN.
4465                  *
4466                  * Additionally, upon receiving deauth frame from AP,
4467                  * OpenBSD 802.11 stack puts the driver in IEEE80211_S_AUTH
4468                  * state. This will also fail with this driver, so bring the FSM
4469                  * from IEEE80211_S_RUN to IEEE80211_S_SCAN in this case as well.
4470                  *
4471                  * XXX TODO: fix this for FreeBSD!
4472                  */
4473                 if (nstate == IEEE80211_S_SCAN ||
4474                     nstate == IEEE80211_S_AUTH ||
4475                     nstate == IEEE80211_S_ASSOC) {
4476                         IWM_DPRINTF(sc, IWM_DEBUG_STATE,
4477                             "Force transition to INIT; MGT=%d\n", arg);
4478                         IWM_UNLOCK(sc);
4479                         IEEE80211_LOCK(ic);
4480                         /* Always pass arg as -1 since we can't Tx right now. */
4481                         /*
4482                          * XXX arg is just ignored anyway when transitioning
4483                          *     to IEEE80211_S_INIT.
4484                          */
4485                         vap->iv_newstate(vap, IEEE80211_S_INIT, -1);
4486                         IWM_DPRINTF(sc, IWM_DEBUG_STATE,
4487                             "Going INIT->SCAN\n");
4488                         nstate = IEEE80211_S_SCAN;
4489                         IEEE80211_UNLOCK(ic);
4490                         IWM_LOCK(sc);
4491                 }
4492         }
4493
4494         switch (nstate) {
4495         case IEEE80211_S_INIT:
4496                 break;
4497
4498         case IEEE80211_S_AUTH:
4499                 if ((error = iwm_auth(vap, sc)) != 0) {
4500                         device_printf(sc->sc_dev,
4501                             "%s: could not move to auth state: %d\n",
4502                             __func__, error);
4503                         break;
4504                 }
4505                 break;
4506
4507         case IEEE80211_S_ASSOC:
4508                 if ((error = iwm_assoc(vap, sc)) != 0) {
4509                         device_printf(sc->sc_dev,
4510                             "%s: failed to associate: %d\n", __func__,
4511                             error);
4512                         break;
4513                 }
4514                 break;
4515
4516         case IEEE80211_S_RUN:
4517         {
4518                 struct iwm_host_cmd cmd = {
4519                         .id = IWM_LQ_CMD,
4520                         .len = { sizeof(in->in_lq), },
4521                         .flags = IWM_CMD_SYNC,
4522                 };
4523
4524                 /* Update the association state, now we have it all */
4525                 /* (eg associd comes in at this point */
4526                 error = iwm_assoc(vap, sc);
4527                 if (error != 0) {
4528                         device_printf(sc->sc_dev,
4529                             "%s: failed to update association state: %d\n",
4530                             __func__,
4531                             error);
4532                         break;
4533                 }
4534
4535                 in = IWM_NODE(vap->iv_bss);
4536                 iwm_mvm_power_mac_update_mode(sc, in);
4537                 iwm_mvm_enable_beacon_filter(sc, in);
4538                 iwm_mvm_update_quotas(sc, in);
4539                 iwm_setrates(sc, in);
4540
4541                 cmd.data[0] = &in->in_lq;
4542                 if ((error = iwm_send_cmd(sc, &cmd)) != 0) {
4543                         device_printf(sc->sc_dev,
4544                             "%s: IWM_LQ_CMD failed\n", __func__);
4545                 }
4546
4547                 iwm_mvm_led_enable(sc);
4548                 break;
4549         }
4550
4551         default:
4552                 break;
4553         }
4554         IWM_UNLOCK(sc);
4555         IEEE80211_LOCK(ic);
4556
4557         return (ivp->iv_newstate(vap, nstate, arg));
4558 }
4559
4560 void
4561 iwm_endscan_cb(void *arg, int pending)
4562 {
4563         struct iwm_softc *sc = arg;
4564         struct ieee80211com *ic = &sc->sc_ic;
4565
4566         IWM_DPRINTF(sc, IWM_DEBUG_SCAN | IWM_DEBUG_TRACE,
4567             "%s: scan ended\n",
4568             __func__);
4569
4570         ieee80211_scan_done(TAILQ_FIRST(&ic->ic_vaps));
4571 }
4572
4573 /*
4574  * Aging and idle timeouts for the different possible scenarios
4575  * in default configuration
4576  */
4577 static const uint32_t
4578 iwm_sf_full_timeout_def[IWM_SF_NUM_SCENARIO][IWM_SF_NUM_TIMEOUT_TYPES] = {
4579         {
4580                 htole32(IWM_SF_SINGLE_UNICAST_AGING_TIMER_DEF),
4581                 htole32(IWM_SF_SINGLE_UNICAST_IDLE_TIMER_DEF)
4582         },
4583         {
4584                 htole32(IWM_SF_AGG_UNICAST_AGING_TIMER_DEF),
4585                 htole32(IWM_SF_AGG_UNICAST_IDLE_TIMER_DEF)
4586         },
4587         {
4588                 htole32(IWM_SF_MCAST_AGING_TIMER_DEF),
4589                 htole32(IWM_SF_MCAST_IDLE_TIMER_DEF)
4590         },
4591         {
4592                 htole32(IWM_SF_BA_AGING_TIMER_DEF),
4593                 htole32(IWM_SF_BA_IDLE_TIMER_DEF)
4594         },
4595         {
4596                 htole32(IWM_SF_TX_RE_AGING_TIMER_DEF),
4597                 htole32(IWM_SF_TX_RE_IDLE_TIMER_DEF)
4598         },
4599 };
4600
4601 /*
4602  * Aging and idle timeouts for the different possible scenarios
4603  * in single BSS MAC configuration.
4604  */
4605 static const uint32_t
4606 iwm_sf_full_timeout[IWM_SF_NUM_SCENARIO][IWM_SF_NUM_TIMEOUT_TYPES] = {
4607         {
4608                 htole32(IWM_SF_SINGLE_UNICAST_AGING_TIMER),
4609                 htole32(IWM_SF_SINGLE_UNICAST_IDLE_TIMER)
4610         },
4611         {
4612                 htole32(IWM_SF_AGG_UNICAST_AGING_TIMER),
4613                 htole32(IWM_SF_AGG_UNICAST_IDLE_TIMER)
4614         },
4615         {
4616                 htole32(IWM_SF_MCAST_AGING_TIMER),
4617                 htole32(IWM_SF_MCAST_IDLE_TIMER)
4618         },
4619         {
4620                 htole32(IWM_SF_BA_AGING_TIMER),
4621                 htole32(IWM_SF_BA_IDLE_TIMER)
4622         },
4623         {
4624                 htole32(IWM_SF_TX_RE_AGING_TIMER),
4625                 htole32(IWM_SF_TX_RE_IDLE_TIMER)
4626         },
4627 };
4628
4629 static void
4630 iwm_mvm_fill_sf_command(struct iwm_softc *sc, struct iwm_sf_cfg_cmd *sf_cmd,
4631     struct ieee80211_node *ni)
4632 {
4633         int i, j, watermark;
4634
4635         sf_cmd->watermark[IWM_SF_LONG_DELAY_ON] = htole32(IWM_SF_W_MARK_SCAN);
4636
4637         /*
4638          * If we are in association flow - check antenna configuration
4639          * capabilities of the AP station, and choose the watermark accordingly.
4640          */
4641         if (ni) {
4642                 if (ni->ni_flags & IEEE80211_NODE_HT) {
4643 #ifdef notyet
4644                         if (ni->ni_rxmcs[2] != 0)
4645                                 watermark = IWM_SF_W_MARK_MIMO3;
4646                         else if (ni->ni_rxmcs[1] != 0)
4647                                 watermark = IWM_SF_W_MARK_MIMO2;
4648                         else
4649 #endif
4650                                 watermark = IWM_SF_W_MARK_SISO;
4651                 } else {
4652                         watermark = IWM_SF_W_MARK_LEGACY;
4653                 }
4654         /* default watermark value for unassociated mode. */
4655         } else {
4656                 watermark = IWM_SF_W_MARK_MIMO2;
4657         }
4658         sf_cmd->watermark[IWM_SF_FULL_ON] = htole32(watermark);
4659
4660         for (i = 0; i < IWM_SF_NUM_SCENARIO; i++) {
4661                 for (j = 0; j < IWM_SF_NUM_TIMEOUT_TYPES; j++) {
4662                         sf_cmd->long_delay_timeouts[i][j] =
4663                                         htole32(IWM_SF_LONG_DELAY_AGING_TIMER);
4664                 }
4665         }
4666
4667         if (ni) {
4668                 memcpy(sf_cmd->full_on_timeouts, iwm_sf_full_timeout,
4669                        sizeof(iwm_sf_full_timeout));
4670         } else {
4671                 memcpy(sf_cmd->full_on_timeouts, iwm_sf_full_timeout_def,
4672                        sizeof(iwm_sf_full_timeout_def));
4673         }
4674 }
4675
4676 static int
4677 iwm_mvm_sf_config(struct iwm_softc *sc, enum iwm_sf_state new_state)
4678 {
4679         struct ieee80211com *ic = &sc->sc_ic;
4680         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
4681         struct iwm_sf_cfg_cmd sf_cmd = {
4682                 .state = htole32(IWM_SF_FULL_ON),
4683         };
4684         int ret = 0;
4685
4686         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000)
4687                 sf_cmd.state |= htole32(IWM_SF_CFG_DUMMY_NOTIF_OFF);
4688
4689         switch (new_state) {
4690         case IWM_SF_UNINIT:
4691         case IWM_SF_INIT_OFF:
4692                 iwm_mvm_fill_sf_command(sc, &sf_cmd, NULL);
4693                 break;
4694         case IWM_SF_FULL_ON:
4695                 iwm_mvm_fill_sf_command(sc, &sf_cmd, vap->iv_bss);
4696                 break;
4697         default:
4698                 IWM_DPRINTF(sc, IWM_DEBUG_PWRSAVE,
4699                     "Invalid state: %d. not sending Smart Fifo cmd\n",
4700                           new_state);
4701                 return EINVAL;
4702         }
4703
4704         ret = iwm_mvm_send_cmd_pdu(sc, IWM_REPLY_SF_CFG_CMD, IWM_CMD_ASYNC,
4705                                    sizeof(sf_cmd), &sf_cmd);
4706         return ret;
4707 }
4708
4709 static int
4710 iwm_send_bt_init_conf(struct iwm_softc *sc)
4711 {
4712         struct iwm_bt_coex_cmd bt_cmd;
4713
4714         bt_cmd.mode = htole32(IWM_BT_COEX_WIFI);
4715         bt_cmd.enabled_modules = htole32(IWM_BT_COEX_HIGH_BAND_RET);
4716
4717         return iwm_mvm_send_cmd_pdu(sc, IWM_BT_CONFIG, 0, sizeof(bt_cmd),
4718             &bt_cmd);
4719 }
4720
4721 static int
4722 iwm_send_update_mcc_cmd(struct iwm_softc *sc, const char *alpha2)
4723 {
4724         struct iwm_mcc_update_cmd mcc_cmd;
4725         struct iwm_host_cmd hcmd = {
4726                 .id = IWM_MCC_UPDATE_CMD,
4727                 .flags = (IWM_CMD_SYNC | IWM_CMD_WANT_SKB),
4728                 .data = { &mcc_cmd },
4729         };
4730         int ret;
4731 #ifdef IWM_DEBUG
4732         struct iwm_rx_packet *pkt;
4733         struct iwm_mcc_update_resp_v1 *mcc_resp_v1 = NULL;
4734         struct iwm_mcc_update_resp *mcc_resp;
4735         int n_channels;
4736         uint16_t mcc;
4737 #endif
4738         int resp_v2 = fw_has_capa(&sc->ucode_capa,
4739             IWM_UCODE_TLV_CAPA_LAR_SUPPORT_V2);
4740
4741         memset(&mcc_cmd, 0, sizeof(mcc_cmd));
4742         mcc_cmd.mcc = htole16(alpha2[0] << 8 | alpha2[1]);
4743         if (fw_has_api(&sc->ucode_capa, IWM_UCODE_TLV_API_WIFI_MCC_UPDATE) ||
4744             fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_LAR_MULTI_MCC))
4745                 mcc_cmd.source_id = IWM_MCC_SOURCE_GET_CURRENT;
4746         else
4747                 mcc_cmd.source_id = IWM_MCC_SOURCE_OLD_FW;
4748
4749         if (resp_v2)
4750                 hcmd.len[0] = sizeof(struct iwm_mcc_update_cmd);
4751         else
4752                 hcmd.len[0] = sizeof(struct iwm_mcc_update_cmd_v1);
4753
4754         IWM_DPRINTF(sc, IWM_DEBUG_NODE,
4755             "send MCC update to FW with '%c%c' src = %d\n",
4756             alpha2[0], alpha2[1], mcc_cmd.source_id);
4757
4758         ret = iwm_send_cmd(sc, &hcmd);
4759         if (ret)
4760                 return ret;
4761
4762 #ifdef IWM_DEBUG
4763         pkt = hcmd.resp_pkt;
4764
4765         /* Extract MCC response */
4766         if (resp_v2) {
4767                 mcc_resp = (void *)pkt->data;
4768                 mcc = mcc_resp->mcc;
4769                 n_channels =  le32toh(mcc_resp->n_channels);
4770         } else {
4771                 mcc_resp_v1 = (void *)pkt->data;
4772                 mcc = mcc_resp_v1->mcc;
4773                 n_channels =  le32toh(mcc_resp_v1->n_channels);
4774         }
4775
4776         /* W/A for a FW/NVM issue - returns 0x00 for the world domain */
4777         if (mcc == 0)
4778                 mcc = 0x3030;  /* "00" - world */
4779
4780         IWM_DPRINTF(sc, IWM_DEBUG_NODE,
4781             "regulatory domain '%c%c' (%d channels available)\n",
4782             mcc >> 8, mcc & 0xff, n_channels);
4783 #endif
4784         iwm_free_resp(sc, &hcmd);
4785
4786         return 0;
4787 }
4788
4789 static void
4790 iwm_mvm_tt_tx_backoff(struct iwm_softc *sc, uint32_t backoff)
4791 {
4792         struct iwm_host_cmd cmd = {
4793                 .id = IWM_REPLY_THERMAL_MNG_BACKOFF,
4794                 .len = { sizeof(uint32_t), },
4795                 .data = { &backoff, },
4796         };
4797
4798         if (iwm_send_cmd(sc, &cmd) != 0) {
4799                 device_printf(sc->sc_dev,
4800                     "failed to change thermal tx backoff\n");
4801         }
4802 }
4803
4804 static int
4805 iwm_init_hw(struct iwm_softc *sc)
4806 {
4807         struct ieee80211com *ic = &sc->sc_ic;
4808         int error, i, ac;
4809
4810         if ((error = iwm_start_hw(sc)) != 0) {
4811                 printf("iwm_start_hw: failed %d\n", error);
4812                 return error;
4813         }
4814
4815         if ((error = iwm_run_init_mvm_ucode(sc, 0)) != 0) {
4816                 printf("iwm_run_init_mvm_ucode: failed %d\n", error);
4817                 return error;
4818         }
4819
4820         /*
4821          * should stop and start HW since that INIT
4822          * image just loaded
4823          */
4824         iwm_stop_device(sc);
4825         if ((error = iwm_start_hw(sc)) != 0) {
4826                 device_printf(sc->sc_dev, "could not initialize hardware\n");
4827                 return error;
4828         }
4829
4830         /* omstart, this time with the regular firmware */
4831         error = iwm_mvm_load_ucode_wait_alive(sc, IWM_UCODE_REGULAR);
4832         if (error) {
4833                 device_printf(sc->sc_dev, "could not load firmware\n");
4834                 goto error;
4835         }
4836
4837         if ((error = iwm_send_bt_init_conf(sc)) != 0) {
4838                 device_printf(sc->sc_dev, "bt init conf failed\n");
4839                 goto error;
4840         }
4841
4842         error = iwm_send_tx_ant_cfg(sc, iwm_mvm_get_valid_tx_ant(sc));
4843         if (error != 0) {
4844                 device_printf(sc->sc_dev, "antenna config failed\n");
4845                 goto error;
4846         }
4847
4848         /* Send phy db control command and then phy db calibration */
4849         if ((error = iwm_send_phy_db_data(sc->sc_phy_db)) != 0)
4850                 goto error;
4851
4852         if ((error = iwm_send_phy_cfg_cmd(sc)) != 0) {
4853                 device_printf(sc->sc_dev, "phy_cfg_cmd failed\n");
4854                 goto error;
4855         }
4856
4857         /* Add auxiliary station for scanning */
4858         if ((error = iwm_mvm_add_aux_sta(sc)) != 0) {
4859                 device_printf(sc->sc_dev, "add_aux_sta failed\n");
4860                 goto error;
4861         }
4862
4863         for (i = 0; i < IWM_NUM_PHY_CTX; i++) {
4864                 /*
4865                  * The channel used here isn't relevant as it's
4866                  * going to be overwritten in the other flows.
4867                  * For now use the first channel we have.
4868                  */
4869                 if ((error = iwm_mvm_phy_ctxt_add(sc,
4870                     &sc->sc_phyctxt[i], &ic->ic_channels[1], 1, 1)) != 0)
4871                         goto error;
4872         }
4873
4874         /* Initialize tx backoffs to the minimum. */
4875         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
4876                 iwm_mvm_tt_tx_backoff(sc, 0);
4877
4878         error = iwm_mvm_power_update_device(sc);
4879         if (error)
4880                 goto error;
4881
4882         if (fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_LAR_SUPPORT)) {
4883                 if ((error = iwm_send_update_mcc_cmd(sc, "ZZ")) != 0)
4884                         goto error;
4885         }
4886
4887         if (fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_UMAC_SCAN)) {
4888                 if ((error = iwm_mvm_config_umac_scan(sc)) != 0)
4889                         goto error;
4890         }
4891
4892         /* Enable Tx queues. */
4893         for (ac = 0; ac < WME_NUM_AC; ac++) {
4894                 error = iwm_enable_txq(sc, IWM_STATION_ID, ac,
4895                     iwm_mvm_ac_to_tx_fifo[ac]);
4896                 if (error)
4897                         goto error;
4898         }
4899
4900         if ((error = iwm_mvm_disable_beacon_filter(sc)) != 0) {
4901                 device_printf(sc->sc_dev, "failed to disable beacon filter\n");
4902                 goto error;
4903         }
4904
4905         return 0;
4906
4907  error:
4908         iwm_stop_device(sc);
4909         return error;
4910 }
4911
4912 /* Allow multicast from our BSSID. */
4913 static int
4914 iwm_allow_mcast(struct ieee80211vap *vap, struct iwm_softc *sc)
4915 {
4916         struct ieee80211_node *ni = vap->iv_bss;
4917         struct iwm_mcast_filter_cmd *cmd;
4918         size_t size;
4919         int error;
4920
4921         size = roundup(sizeof(*cmd), 4);
4922         cmd = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
4923         if (cmd == NULL)
4924                 return ENOMEM;
4925         cmd->filter_own = 1;
4926         cmd->port_id = 0;
4927         cmd->count = 0;
4928         cmd->pass_all = 1;
4929         IEEE80211_ADDR_COPY(cmd->bssid, ni->ni_bssid);
4930
4931         error = iwm_mvm_send_cmd_pdu(sc, IWM_MCAST_FILTER_CMD,
4932             IWM_CMD_SYNC, size, cmd);
4933         free(cmd, M_DEVBUF);
4934
4935         return (error);
4936 }
4937
4938 /*
4939  * ifnet interfaces
4940  */
4941
4942 static void
4943 iwm_init(struct iwm_softc *sc)
4944 {
4945         int error;
4946
4947         if (sc->sc_flags & IWM_FLAG_HW_INITED) {
4948                 return;
4949         }
4950         sc->sc_generation++;
4951         sc->sc_flags &= ~IWM_FLAG_STOPPED;
4952
4953         if ((error = iwm_init_hw(sc)) != 0) {
4954                 printf("iwm_init_hw failed %d\n", error);
4955                 iwm_stop(sc);
4956                 return;
4957         }
4958
4959         /*
4960          * Ok, firmware loaded and we are jogging
4961          */
4962         sc->sc_flags |= IWM_FLAG_HW_INITED;
4963         callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog, sc);
4964 }
4965
4966 static int
4967 iwm_transmit(struct ieee80211com *ic, struct mbuf *m)
4968 {
4969         struct iwm_softc *sc;
4970         int error;
4971
4972         sc = ic->ic_softc;
4973
4974         IWM_LOCK(sc);
4975         if ((sc->sc_flags & IWM_FLAG_HW_INITED) == 0) {
4976                 IWM_UNLOCK(sc);
4977                 return (ENXIO);
4978         }
4979         error = mbufq_enqueue(&sc->sc_snd, m);
4980         if (error) {
4981                 IWM_UNLOCK(sc);
4982                 return (error);
4983         }
4984         iwm_start(sc);
4985         IWM_UNLOCK(sc);
4986         return (0);
4987 }
4988
4989 /*
4990  * Dequeue packets from sendq and call send.
4991  */
4992 static void
4993 iwm_start(struct iwm_softc *sc)
4994 {
4995         struct ieee80211_node *ni;
4996         struct mbuf *m;
4997         int ac = 0;
4998
4999         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TRACE, "->%s\n", __func__);
5000         while (sc->qfullmsk == 0 &&
5001                 (m = mbufq_dequeue(&sc->sc_snd)) != NULL) {
5002                 ni = (struct ieee80211_node *)m->m_pkthdr.rcvif;
5003                 if (iwm_tx(sc, m, ni, ac) != 0) {
5004                         if_inc_counter(ni->ni_vap->iv_ifp,
5005                             IFCOUNTER_OERRORS, 1);
5006                         ieee80211_free_node(ni);
5007                         continue;
5008                 }
5009                 sc->sc_tx_timer = 15;
5010         }
5011         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TRACE, "<-%s\n", __func__);
5012 }
5013
5014 static void
5015 iwm_stop(struct iwm_softc *sc)
5016 {
5017
5018         sc->sc_flags &= ~IWM_FLAG_HW_INITED;
5019         sc->sc_flags |= IWM_FLAG_STOPPED;
5020         sc->sc_generation++;
5021         iwm_led_blink_stop(sc);
5022         sc->sc_tx_timer = 0;
5023         iwm_stop_device(sc);
5024         sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
5025 }
5026
5027 static void
5028 iwm_watchdog(void *arg)
5029 {
5030         struct iwm_softc *sc = arg;
5031         struct ieee80211com *ic = &sc->sc_ic;
5032
5033         if (sc->sc_tx_timer > 0) {
5034                 if (--sc->sc_tx_timer == 0) {
5035                         device_printf(sc->sc_dev, "device timeout\n");
5036 #ifdef IWM_DEBUG
5037                         iwm_nic_error(sc);
5038 #endif
5039                         ieee80211_restart_all(ic);
5040                         counter_u64_add(sc->sc_ic.ic_oerrors, 1);
5041                         return;
5042                 }
5043         }
5044         callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog, sc);
5045 }
5046
5047 static void
5048 iwm_parent(struct ieee80211com *ic)
5049 {
5050         struct iwm_softc *sc = ic->ic_softc;
5051         int startall = 0;
5052
5053         IWM_LOCK(sc);
5054         if (ic->ic_nrunning > 0) {
5055                 if (!(sc->sc_flags & IWM_FLAG_HW_INITED)) {
5056                         iwm_init(sc);
5057                         startall = 1;
5058                 }
5059         } else if (sc->sc_flags & IWM_FLAG_HW_INITED)
5060                 iwm_stop(sc);
5061         IWM_UNLOCK(sc);
5062         if (startall)
5063                 ieee80211_start_all(ic);
5064 }
5065
5066 /*
5067  * The interrupt side of things
5068  */
5069
5070 /*
5071  * error dumping routines are from iwlwifi/mvm/utils.c
5072  */
5073
5074 /*
5075  * Note: This structure is read from the device with IO accesses,
5076  * and the reading already does the endian conversion. As it is
5077  * read with uint32_t-sized accesses, any members with a different size
5078  * need to be ordered correctly though!
5079  */
5080 struct iwm_error_event_table {
5081         uint32_t valid;         /* (nonzero) valid, (0) log is empty */
5082         uint32_t error_id;              /* type of error */
5083         uint32_t trm_hw_status0;        /* TRM HW status */
5084         uint32_t trm_hw_status1;        /* TRM HW status */
5085         uint32_t blink2;                /* branch link */
5086         uint32_t ilink1;                /* interrupt link */
5087         uint32_t ilink2;                /* interrupt link */
5088         uint32_t data1;         /* error-specific data */
5089         uint32_t data2;         /* error-specific data */
5090         uint32_t data3;         /* error-specific data */
5091         uint32_t bcon_time;             /* beacon timer */
5092         uint32_t tsf_low;               /* network timestamp function timer */
5093         uint32_t tsf_hi;                /* network timestamp function timer */
5094         uint32_t gp1;           /* GP1 timer register */
5095         uint32_t gp2;           /* GP2 timer register */
5096         uint32_t fw_rev_type;   /* firmware revision type */
5097         uint32_t major;         /* uCode version major */
5098         uint32_t minor;         /* uCode version minor */
5099         uint32_t hw_ver;                /* HW Silicon version */
5100         uint32_t brd_ver;               /* HW board version */
5101         uint32_t log_pc;                /* log program counter */
5102         uint32_t frame_ptr;             /* frame pointer */
5103         uint32_t stack_ptr;             /* stack pointer */
5104         uint32_t hcmd;          /* last host command header */
5105         uint32_t isr0;          /* isr status register LMPM_NIC_ISR0:
5106                                  * rxtx_flag */
5107         uint32_t isr1;          /* isr status register LMPM_NIC_ISR1:
5108                                  * host_flag */
5109         uint32_t isr2;          /* isr status register LMPM_NIC_ISR2:
5110                                  * enc_flag */
5111         uint32_t isr3;          /* isr status register LMPM_NIC_ISR3:
5112                                  * time_flag */
5113         uint32_t isr4;          /* isr status register LMPM_NIC_ISR4:
5114                                  * wico interrupt */
5115         uint32_t last_cmd_id;   /* last HCMD id handled by the firmware */
5116         uint32_t wait_event;            /* wait event() caller address */
5117         uint32_t l2p_control;   /* L2pControlField */
5118         uint32_t l2p_duration;  /* L2pDurationField */
5119         uint32_t l2p_mhvalid;   /* L2pMhValidBits */
5120         uint32_t l2p_addr_match;        /* L2pAddrMatchStat */
5121         uint32_t lmpm_pmg_sel;  /* indicate which clocks are turned on
5122                                  * (LMPM_PMG_SEL) */
5123         uint32_t u_timestamp;   /* indicate when the date and time of the
5124                                  * compilation */
5125         uint32_t flow_handler;  /* FH read/write pointers, RX credit */
5126 } __packed /* LOG_ERROR_TABLE_API_S_VER_3 */;
5127
5128 /*
5129  * UMAC error struct - relevant starting from family 8000 chip.
5130  * Note: This structure is read from the device with IO accesses,
5131  * and the reading already does the endian conversion. As it is
5132  * read with u32-sized accesses, any members with a different size
5133  * need to be ordered correctly though!
5134  */
5135 struct iwm_umac_error_event_table {
5136         uint32_t valid;         /* (nonzero) valid, (0) log is empty */
5137         uint32_t error_id;      /* type of error */
5138         uint32_t blink1;        /* branch link */
5139         uint32_t blink2;        /* branch link */
5140         uint32_t ilink1;        /* interrupt link */
5141         uint32_t ilink2;        /* interrupt link */
5142         uint32_t data1;         /* error-specific data */
5143         uint32_t data2;         /* error-specific data */
5144         uint32_t data3;         /* error-specific data */
5145         uint32_t umac_major;
5146         uint32_t umac_minor;
5147         uint32_t frame_pointer; /* core register 27*/
5148         uint32_t stack_pointer; /* core register 28 */
5149         uint32_t cmd_header;    /* latest host cmd sent to UMAC */
5150         uint32_t nic_isr_pref;  /* ISR status register */
5151 } __packed;
5152
5153 #define ERROR_START_OFFSET  (1 * sizeof(uint32_t))
5154 #define ERROR_ELEM_SIZE     (7 * sizeof(uint32_t))
5155
5156 #ifdef IWM_DEBUG
5157 struct {
5158         const char *name;
5159         uint8_t num;
5160 } advanced_lookup[] = {
5161         { "NMI_INTERRUPT_WDG", 0x34 },
5162         { "SYSASSERT", 0x35 },
5163         { "UCODE_VERSION_MISMATCH", 0x37 },
5164         { "BAD_COMMAND", 0x38 },
5165         { "NMI_INTERRUPT_DATA_ACTION_PT", 0x3C },
5166         { "FATAL_ERROR", 0x3D },
5167         { "NMI_TRM_HW_ERR", 0x46 },
5168         { "NMI_INTERRUPT_TRM", 0x4C },
5169         { "NMI_INTERRUPT_BREAK_POINT", 0x54 },
5170         { "NMI_INTERRUPT_WDG_RXF_FULL", 0x5C },
5171         { "NMI_INTERRUPT_WDG_NO_RBD_RXF_FULL", 0x64 },
5172         { "NMI_INTERRUPT_HOST", 0x66 },
5173         { "NMI_INTERRUPT_ACTION_PT", 0x7C },
5174         { "NMI_INTERRUPT_UNKNOWN", 0x84 },
5175         { "NMI_INTERRUPT_INST_ACTION_PT", 0x86 },
5176         { "ADVANCED_SYSASSERT", 0 },
5177 };
5178
5179 static const char *
5180 iwm_desc_lookup(uint32_t num)
5181 {
5182         int i;
5183
5184         for (i = 0; i < nitems(advanced_lookup) - 1; i++)
5185                 if (advanced_lookup[i].num == num)
5186                         return advanced_lookup[i].name;
5187
5188         /* No entry matches 'num', so it is the last: ADVANCED_SYSASSERT */
5189         return advanced_lookup[i].name;
5190 }
5191
5192 static void
5193 iwm_nic_umac_error(struct iwm_softc *sc)
5194 {
5195         struct iwm_umac_error_event_table table;
5196         uint32_t base;
5197
5198         base = sc->umac_error_event_table;
5199
5200         if (base < 0x800000) {
5201                 device_printf(sc->sc_dev, "Invalid error log pointer 0x%08x\n",
5202                     base);
5203                 return;
5204         }
5205
5206         if (iwm_read_mem(sc, base, &table, sizeof(table)/sizeof(uint32_t))) {
5207                 device_printf(sc->sc_dev, "reading errlog failed\n");
5208                 return;
5209         }
5210
5211         if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) {
5212                 device_printf(sc->sc_dev, "Start UMAC Error Log Dump:\n");
5213                 device_printf(sc->sc_dev, "Status: 0x%x, count: %d\n",
5214                     sc->sc_flags, table.valid);
5215         }
5216
5217         device_printf(sc->sc_dev, "0x%08X | %s\n", table.error_id,
5218                 iwm_desc_lookup(table.error_id));
5219         device_printf(sc->sc_dev, "0x%08X | umac branchlink1\n", table.blink1);
5220         device_printf(sc->sc_dev, "0x%08X | umac branchlink2\n", table.blink2);
5221         device_printf(sc->sc_dev, "0x%08X | umac interruptlink1\n",
5222             table.ilink1);
5223         device_printf(sc->sc_dev, "0x%08X | umac interruptlink2\n",
5224             table.ilink2);
5225         device_printf(sc->sc_dev, "0x%08X | umac data1\n", table.data1);
5226         device_printf(sc->sc_dev, "0x%08X | umac data2\n", table.data2);
5227         device_printf(sc->sc_dev, "0x%08X | umac data3\n", table.data3);
5228         device_printf(sc->sc_dev, "0x%08X | umac major\n", table.umac_major);
5229         device_printf(sc->sc_dev, "0x%08X | umac minor\n", table.umac_minor);
5230         device_printf(sc->sc_dev, "0x%08X | frame pointer\n",
5231             table.frame_pointer);
5232         device_printf(sc->sc_dev, "0x%08X | stack pointer\n",
5233             table.stack_pointer);
5234         device_printf(sc->sc_dev, "0x%08X | last host cmd\n", table.cmd_header);
5235         device_printf(sc->sc_dev, "0x%08X | isr status reg\n",
5236             table.nic_isr_pref);
5237 }
5238
5239 /*
5240  * Support for dumping the error log seemed like a good idea ...
5241  * but it's mostly hex junk and the only sensible thing is the
5242  * hw/ucode revision (which we know anyway).  Since it's here,
5243  * I'll just leave it in, just in case e.g. the Intel guys want to
5244  * help us decipher some "ADVANCED_SYSASSERT" later.
5245  */
5246 static void
5247 iwm_nic_error(struct iwm_softc *sc)
5248 {
5249         struct iwm_error_event_table table;
5250         uint32_t base;
5251
5252         device_printf(sc->sc_dev, "dumping device error log\n");
5253         base = sc->error_event_table;
5254         if (base < 0x800000) {
5255                 device_printf(sc->sc_dev,
5256                     "Invalid error log pointer 0x%08x\n", base);
5257                 return;
5258         }
5259
5260         if (iwm_read_mem(sc, base, &table, sizeof(table)/sizeof(uint32_t))) {
5261                 device_printf(sc->sc_dev, "reading errlog failed\n");
5262                 return;
5263         }
5264
5265         if (!table.valid) {
5266                 device_printf(sc->sc_dev, "errlog not found, skipping\n");
5267                 return;
5268         }
5269
5270         if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) {
5271                 device_printf(sc->sc_dev, "Start Error Log Dump:\n");
5272                 device_printf(sc->sc_dev, "Status: 0x%x, count: %d\n",
5273                     sc->sc_flags, table.valid);
5274         }
5275
5276         device_printf(sc->sc_dev, "0x%08X | %-28s\n", table.error_id,
5277             iwm_desc_lookup(table.error_id));
5278         device_printf(sc->sc_dev, "%08X | trm_hw_status0\n",
5279             table.trm_hw_status0);
5280         device_printf(sc->sc_dev, "%08X | trm_hw_status1\n",
5281             table.trm_hw_status1);
5282         device_printf(sc->sc_dev, "%08X | branchlink2\n", table.blink2);
5283         device_printf(sc->sc_dev, "%08X | interruptlink1\n", table.ilink1);
5284         device_printf(sc->sc_dev, "%08X | interruptlink2\n", table.ilink2);
5285         device_printf(sc->sc_dev, "%08X | data1\n", table.data1);
5286         device_printf(sc->sc_dev, "%08X | data2\n", table.data2);
5287         device_printf(sc->sc_dev, "%08X | data3\n", table.data3);
5288         device_printf(sc->sc_dev, "%08X | beacon time\n", table.bcon_time);
5289         device_printf(sc->sc_dev, "%08X | tsf low\n", table.tsf_low);
5290         device_printf(sc->sc_dev, "%08X | tsf hi\n", table.tsf_hi);
5291         device_printf(sc->sc_dev, "%08X | time gp1\n", table.gp1);
5292         device_printf(sc->sc_dev, "%08X | time gp2\n", table.gp2);
5293         device_printf(sc->sc_dev, "%08X | uCode revision type\n",
5294             table.fw_rev_type);
5295         device_printf(sc->sc_dev, "%08X | uCode version major\n", table.major);
5296         device_printf(sc->sc_dev, "%08X | uCode version minor\n", table.minor);
5297         device_printf(sc->sc_dev, "%08X | hw version\n", table.hw_ver);
5298         device_printf(sc->sc_dev, "%08X | board version\n", table.brd_ver);
5299         device_printf(sc->sc_dev, "%08X | hcmd\n", table.hcmd);
5300         device_printf(sc->sc_dev, "%08X | isr0\n", table.isr0);
5301         device_printf(sc->sc_dev, "%08X | isr1\n", table.isr1);
5302         device_printf(sc->sc_dev, "%08X | isr2\n", table.isr2);
5303         device_printf(sc->sc_dev, "%08X | isr3\n", table.isr3);
5304         device_printf(sc->sc_dev, "%08X | isr4\n", table.isr4);
5305         device_printf(sc->sc_dev, "%08X | last cmd Id\n", table.last_cmd_id);
5306         device_printf(sc->sc_dev, "%08X | wait_event\n", table.wait_event);
5307         device_printf(sc->sc_dev, "%08X | l2p_control\n", table.l2p_control);
5308         device_printf(sc->sc_dev, "%08X | l2p_duration\n", table.l2p_duration);
5309         device_printf(sc->sc_dev, "%08X | l2p_mhvalid\n", table.l2p_mhvalid);
5310         device_printf(sc->sc_dev, "%08X | l2p_addr_match\n", table.l2p_addr_match);
5311         device_printf(sc->sc_dev, "%08X | lmpm_pmg_sel\n", table.lmpm_pmg_sel);
5312         device_printf(sc->sc_dev, "%08X | timestamp\n", table.u_timestamp);
5313         device_printf(sc->sc_dev, "%08X | flow_handler\n", table.flow_handler);
5314
5315         if (sc->umac_error_event_table)
5316                 iwm_nic_umac_error(sc);
5317 }
5318 #endif
5319
5320 #define ADVANCE_RXQ(sc) (sc->rxq.cur = (sc->rxq.cur + 1) % IWM_RX_RING_COUNT);
5321
5322 /*
5323  * Process an IWM_CSR_INT_BIT_FH_RX or IWM_CSR_INT_BIT_SW_RX interrupt.
5324  * Basic structure from if_iwn
5325  */
5326 static void
5327 iwm_notif_intr(struct iwm_softc *sc)
5328 {
5329         struct ieee80211com *ic = &sc->sc_ic;
5330         uint16_t hw;
5331
5332         bus_dmamap_sync(sc->rxq.stat_dma.tag, sc->rxq.stat_dma.map,
5333             BUS_DMASYNC_POSTREAD);
5334
5335         hw = le16toh(sc->rxq.stat->closed_rb_num) & 0xfff;
5336
5337         /*
5338          * Process responses
5339          */
5340         while (sc->rxq.cur != hw) {
5341                 struct iwm_rx_ring *ring = &sc->rxq;
5342                 struct iwm_rx_data *data = &ring->data[ring->cur];
5343                 struct iwm_rx_packet *pkt;
5344                 struct iwm_cmd_response *cresp;
5345                 int qid, idx, code;
5346
5347                 bus_dmamap_sync(ring->data_dmat, data->map,
5348                     BUS_DMASYNC_POSTREAD);
5349                 pkt = mtod(data->m, struct iwm_rx_packet *);
5350
5351                 qid = pkt->hdr.qid & ~0x80;
5352                 idx = pkt->hdr.idx;
5353
5354                 code = IWM_WIDE_ID(pkt->hdr.flags, pkt->hdr.code);
5355                 IWM_DPRINTF(sc, IWM_DEBUG_INTR,
5356                     "rx packet qid=%d idx=%d type=%x %d %d\n",
5357                     pkt->hdr.qid & ~0x80, pkt->hdr.idx, code, ring->cur, hw);
5358
5359                 /*
5360                  * randomly get these from the firmware, no idea why.
5361                  * they at least seem harmless, so just ignore them for now
5362                  */
5363                 if (__predict_false((pkt->hdr.code == 0 && qid == 0 && idx == 0)
5364                     || pkt->len_n_flags == htole32(0x55550000))) {
5365                         ADVANCE_RXQ(sc);
5366                         continue;
5367                 }
5368
5369                 iwm_notification_wait_notify(sc->sc_notif_wait, code, pkt);
5370
5371                 switch (code) {
5372                 case IWM_REPLY_RX_PHY_CMD:
5373                         iwm_mvm_rx_rx_phy_cmd(sc, pkt, data);
5374                         break;
5375
5376                 case IWM_REPLY_RX_MPDU_CMD:
5377                         iwm_mvm_rx_rx_mpdu(sc, data->m);
5378                         break;
5379
5380                 case IWM_TX_CMD:
5381                         iwm_mvm_rx_tx_cmd(sc, pkt, data);
5382                         break;
5383
5384                 case IWM_MISSED_BEACONS_NOTIFICATION: {
5385                         struct iwm_missed_beacons_notif *resp;
5386                         int missed;
5387
5388                         /* XXX look at mac_id to determine interface ID */
5389                         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
5390
5391                         resp = (void *)pkt->data;
5392                         missed = le32toh(resp->consec_missed_beacons);
5393
5394                         IWM_DPRINTF(sc, IWM_DEBUG_BEACON | IWM_DEBUG_STATE,
5395                             "%s: MISSED_BEACON: mac_id=%d, "
5396                             "consec_since_last_rx=%d, consec=%d, num_expect=%d "
5397                             "num_rx=%d\n",
5398                             __func__,
5399                             le32toh(resp->mac_id),
5400                             le32toh(resp->consec_missed_beacons_since_last_rx),
5401                             le32toh(resp->consec_missed_beacons),
5402                             le32toh(resp->num_expected_beacons),
5403                             le32toh(resp->num_recvd_beacons));
5404
5405                         /* Be paranoid */
5406                         if (vap == NULL)
5407                                 break;
5408
5409                         /* XXX no net80211 locking? */
5410                         if (vap->iv_state == IEEE80211_S_RUN &&
5411                             (ic->ic_flags & IEEE80211_F_SCAN) == 0) {
5412                                 if (missed > vap->iv_bmissthreshold) {
5413                                         /* XXX bad locking; turn into task */
5414                                         IWM_UNLOCK(sc);
5415                                         ieee80211_beacon_miss(ic);
5416                                         IWM_LOCK(sc);
5417                                 }
5418                         }
5419
5420                         break;
5421                 }
5422
5423                 case IWM_MFUART_LOAD_NOTIFICATION:
5424                         break;
5425
5426                 case IWM_MVM_ALIVE:
5427                         break;
5428
5429                 case IWM_CALIB_RES_NOTIF_PHY_DB:
5430                         break;
5431
5432                 case IWM_STATISTICS_NOTIFICATION: {
5433                         struct iwm_notif_statistics *stats;
5434                         stats = (void *)pkt->data;
5435                         memcpy(&sc->sc_stats, stats, sizeof(sc->sc_stats));
5436                         sc->sc_noise = iwm_get_noise(sc, &stats->rx.general);
5437                         break;
5438                 }
5439
5440                 case IWM_NVM_ACCESS_CMD:
5441                 case IWM_MCC_UPDATE_CMD:
5442                         if (sc->sc_wantresp == ((qid << 16) | idx)) {
5443                                 memcpy(sc->sc_cmd_resp,
5444                                     pkt, sizeof(sc->sc_cmd_resp));
5445                         }
5446                         break;
5447
5448                 case IWM_MCC_CHUB_UPDATE_CMD: {
5449                         struct iwm_mcc_chub_notif *notif;
5450                         notif = (void *)pkt->data;
5451
5452                         sc->sc_fw_mcc[0] = (notif->mcc & 0xff00) >> 8;
5453                         sc->sc_fw_mcc[1] = notif->mcc & 0xff;
5454                         sc->sc_fw_mcc[2] = '\0';
5455                         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
5456                             "fw source %d sent CC '%s'\n",
5457                             notif->source_id, sc->sc_fw_mcc);
5458                         break;
5459                 }
5460
5461                 case IWM_DTS_MEASUREMENT_NOTIFICATION:
5462                 case IWM_WIDE_ID(IWM_PHY_OPS_GROUP,
5463                                  IWM_DTS_MEASUREMENT_NOTIF_WIDE): {
5464                         struct iwm_dts_measurement_notif_v1 *notif;
5465
5466                         if (iwm_rx_packet_payload_len(pkt) < sizeof(*notif)) {
5467                                 device_printf(sc->sc_dev,
5468                                     "Invalid DTS_MEASUREMENT_NOTIFICATION\n");
5469                                 break;
5470                         }
5471                         notif = (void *)pkt->data;
5472                         IWM_DPRINTF(sc, IWM_DEBUG_TEMP,
5473                             "IWM_DTS_MEASUREMENT_NOTIFICATION - %d\n",
5474                             notif->temp);
5475                         break;
5476                 }
5477
5478                 case IWM_PHY_CONFIGURATION_CMD:
5479                 case IWM_TX_ANT_CONFIGURATION_CMD:
5480                 case IWM_ADD_STA:
5481                 case IWM_MAC_CONTEXT_CMD:
5482                 case IWM_REPLY_SF_CFG_CMD:
5483                 case IWM_POWER_TABLE_CMD:
5484                 case IWM_PHY_CONTEXT_CMD:
5485                 case IWM_BINDING_CONTEXT_CMD:
5486                 case IWM_TIME_EVENT_CMD:
5487                 case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_CFG_CMD):
5488                 case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_REQ_UMAC):
5489                 case IWM_SCAN_ABORT_UMAC:
5490                 case IWM_SCAN_OFFLOAD_REQUEST_CMD:
5491                 case IWM_SCAN_OFFLOAD_ABORT_CMD:
5492                 case IWM_REPLY_BEACON_FILTERING_CMD:
5493                 case IWM_MAC_PM_POWER_TABLE:
5494                 case IWM_TIME_QUOTA_CMD:
5495                 case IWM_REMOVE_STA:
5496                 case IWM_TXPATH_FLUSH:
5497                 case IWM_LQ_CMD:
5498                 case IWM_FW_PAGING_BLOCK_CMD:
5499                 case IWM_BT_CONFIG:
5500                 case IWM_REPLY_THERMAL_MNG_BACKOFF:
5501                         cresp = (void *)pkt->data;
5502                         if (sc->sc_wantresp == ((qid << 16) | idx)) {
5503                                 memcpy(sc->sc_cmd_resp,
5504                                     pkt, sizeof(*pkt)+sizeof(*cresp));
5505                         }
5506                         break;
5507
5508                 /* ignore */
5509                 case 0x6c: /* IWM_PHY_DB_CMD, no idea why it's not in fw-api.h */
5510                         break;
5511
5512                 case IWM_INIT_COMPLETE_NOTIF:
5513                         break;
5514
5515                 case IWM_SCAN_OFFLOAD_COMPLETE: {
5516                         struct iwm_periodic_scan_complete *notif;
5517                         notif = (void *)pkt->data;
5518                         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
5519                                 sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
5520                                 ieee80211_runtask(ic, &sc->sc_es_task);
5521                         }
5522                         break;
5523                 }
5524
5525                 case IWM_SCAN_ITERATION_COMPLETE: {
5526                         struct iwm_lmac_scan_complete_notif *notif;
5527                         notif = (void *)pkt->data;
5528                         ieee80211_runtask(&sc->sc_ic, &sc->sc_es_task);
5529                         break;
5530                 }
5531
5532                 case IWM_SCAN_COMPLETE_UMAC: {
5533                         struct iwm_umac_scan_complete *notif;
5534                         notif = (void *)pkt->data;
5535
5536                         IWM_DPRINTF(sc, IWM_DEBUG_SCAN,
5537                             "UMAC scan complete, status=0x%x\n",
5538                             notif->status);
5539                         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
5540                                 sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
5541                                 ieee80211_runtask(ic, &sc->sc_es_task);
5542                         }
5543                         break;
5544                 }
5545
5546                 case IWM_SCAN_ITERATION_COMPLETE_UMAC: {
5547                         struct iwm_umac_scan_iter_complete_notif *notif;
5548                         notif = (void *)pkt->data;
5549
5550                         IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "UMAC scan iteration "
5551                             "complete, status=0x%x, %d channels scanned\n",
5552                             notif->status, notif->scanned_channels);
5553                         ieee80211_runtask(&sc->sc_ic, &sc->sc_es_task);
5554                         break;
5555                 }
5556
5557                 case IWM_REPLY_ERROR: {
5558                         struct iwm_error_resp *resp;
5559                         resp = (void *)pkt->data;
5560
5561                         device_printf(sc->sc_dev,
5562                             "firmware error 0x%x, cmd 0x%x\n",
5563                             le32toh(resp->error_type),
5564                             resp->cmd_id);
5565                         break;
5566                 }
5567
5568                 case IWM_TIME_EVENT_NOTIFICATION: {
5569                         struct iwm_time_event_notif *notif;
5570                         notif = (void *)pkt->data;
5571
5572                         IWM_DPRINTF(sc, IWM_DEBUG_INTR,
5573                             "TE notif status = 0x%x action = 0x%x\n",
5574                             notif->status, notif->action);
5575                         break;
5576                 }
5577
5578                 case IWM_MCAST_FILTER_CMD:
5579                         break;
5580
5581                 case IWM_SCD_QUEUE_CFG: {
5582                         struct iwm_scd_txq_cfg_rsp *rsp;
5583                         rsp = (void *)pkt->data;
5584
5585                         IWM_DPRINTF(sc, IWM_DEBUG_CMD,
5586                             "queue cfg token=0x%x sta_id=%d "
5587                             "tid=%d scd_queue=%d\n",
5588                             rsp->token, rsp->sta_id, rsp->tid,
5589                             rsp->scd_queue);
5590                         break;
5591                 }
5592
5593                 default:
5594                         device_printf(sc->sc_dev,
5595                             "frame %d/%d %x UNHANDLED (this should "
5596                             "not happen)\n", qid, idx,
5597                             pkt->len_n_flags);
5598                         break;
5599                 }
5600
5601                 /*
5602                  * Why test bit 0x80?  The Linux driver:
5603                  *
5604                  * There is one exception:  uCode sets bit 15 when it
5605                  * originates the response/notification, i.e. when the
5606                  * response/notification is not a direct response to a
5607                  * command sent by the driver.  For example, uCode issues
5608                  * IWM_REPLY_RX when it sends a received frame to the driver;
5609                  * it is not a direct response to any driver command.
5610                  *
5611                  * Ok, so since when is 7 == 15?  Well, the Linux driver
5612                  * uses a slightly different format for pkt->hdr, and "qid"
5613                  * is actually the upper byte of a two-byte field.
5614                  */
5615                 if (!(pkt->hdr.qid & (1 << 7))) {
5616                         iwm_cmd_done(sc, pkt);
5617                 }
5618
5619                 ADVANCE_RXQ(sc);
5620         }
5621
5622         /*
5623          * Tell the firmware what we have processed.
5624          * Seems like the hardware gets upset unless we align
5625          * the write by 8??
5626          */
5627         hw = (hw == 0) ? IWM_RX_RING_COUNT - 1 : hw - 1;
5628         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_WPTR, hw & ~7);
5629 }
5630
5631 static void
5632 iwm_intr(void *arg)
5633 {
5634         struct iwm_softc *sc = arg;
5635         int handled = 0;
5636         int r1, r2, rv = 0;
5637         int isperiodic = 0;
5638
5639         IWM_LOCK(sc);
5640         IWM_WRITE(sc, IWM_CSR_INT_MASK, 0);
5641
5642         if (sc->sc_flags & IWM_FLAG_USE_ICT) {
5643                 uint32_t *ict = sc->ict_dma.vaddr;
5644                 int tmp;
5645
5646                 tmp = htole32(ict[sc->ict_cur]);
5647                 if (!tmp)
5648                         goto out_ena;
5649
5650                 /*
5651                  * ok, there was something.  keep plowing until we have all.
5652                  */
5653                 r1 = r2 = 0;
5654                 while (tmp) {
5655                         r1 |= tmp;
5656                         ict[sc->ict_cur] = 0;
5657                         sc->ict_cur = (sc->ict_cur+1) % IWM_ICT_COUNT;
5658                         tmp = htole32(ict[sc->ict_cur]);
5659                 }
5660
5661                 /* this is where the fun begins.  don't ask */
5662                 if (r1 == 0xffffffff)
5663                         r1 = 0;
5664
5665                 /* i am not expected to understand this */
5666                 if (r1 & 0xc0000)
5667                         r1 |= 0x8000;
5668                 r1 = (0xff & r1) | ((0xff00 & r1) << 16);
5669         } else {
5670                 r1 = IWM_READ(sc, IWM_CSR_INT);
5671                 /* "hardware gone" (where, fishing?) */
5672                 if (r1 == 0xffffffff || (r1 & 0xfffffff0) == 0xa5a5a5a0)
5673                         goto out;
5674                 r2 = IWM_READ(sc, IWM_CSR_FH_INT_STATUS);
5675         }
5676         if (r1 == 0 && r2 == 0) {
5677                 goto out_ena;
5678         }
5679
5680         IWM_WRITE(sc, IWM_CSR_INT, r1 | ~sc->sc_intmask);
5681
5682         /* Safely ignore these bits for debug checks below */
5683         r1 &= ~(IWM_CSR_INT_BIT_ALIVE | IWM_CSR_INT_BIT_SCD);
5684
5685         if (r1 & IWM_CSR_INT_BIT_SW_ERR) {
5686                 int i;
5687                 struct ieee80211com *ic = &sc->sc_ic;
5688                 struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
5689
5690 #ifdef IWM_DEBUG
5691                 iwm_nic_error(sc);
5692 #endif
5693                 /* Dump driver status (TX and RX rings) while we're here. */
5694                 device_printf(sc->sc_dev, "driver status:\n");
5695                 for (i = 0; i < IWM_MVM_MAX_QUEUES; i++) {
5696                         struct iwm_tx_ring *ring = &sc->txq[i];
5697                         device_printf(sc->sc_dev,
5698                             "  tx ring %2d: qid=%-2d cur=%-3d "
5699                             "queued=%-3d\n",
5700                             i, ring->qid, ring->cur, ring->queued);
5701                 }
5702                 device_printf(sc->sc_dev,
5703                     "  rx ring: cur=%d\n", sc->rxq.cur);
5704                 device_printf(sc->sc_dev,
5705                     "  802.11 state %d\n", (vap == NULL) ? -1 : vap->iv_state);
5706
5707                 /* Don't stop the device; just do a VAP restart */
5708                 IWM_UNLOCK(sc);
5709
5710                 if (vap == NULL) {
5711                         printf("%s: null vap\n", __func__);
5712                         return;
5713                 }
5714
5715                 device_printf(sc->sc_dev, "%s: controller panicked, iv_state = %d; "
5716                     "restarting\n", __func__, vap->iv_state);
5717
5718                 /* XXX TODO: turn this into a callout/taskqueue */
5719                 ieee80211_restart_all(ic);
5720                 return;
5721         }
5722
5723         if (r1 & IWM_CSR_INT_BIT_HW_ERR) {
5724                 handled |= IWM_CSR_INT_BIT_HW_ERR;
5725                 device_printf(sc->sc_dev, "hardware error, stopping device\n");
5726                 iwm_stop(sc);
5727                 rv = 1;
5728                 goto out;
5729         }
5730
5731         /* firmware chunk loaded */
5732         if (r1 & IWM_CSR_INT_BIT_FH_TX) {
5733                 IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, IWM_CSR_FH_INT_TX_MASK);
5734                 handled |= IWM_CSR_INT_BIT_FH_TX;
5735                 sc->sc_fw_chunk_done = 1;
5736                 wakeup(&sc->sc_fw);
5737         }
5738
5739         if (r1 & IWM_CSR_INT_BIT_RF_KILL) {
5740                 handled |= IWM_CSR_INT_BIT_RF_KILL;
5741                 if (iwm_check_rfkill(sc)) {
5742                         device_printf(sc->sc_dev,
5743                             "%s: rfkill switch, disabling interface\n",
5744                             __func__);
5745                         iwm_stop(sc);
5746                 }
5747         }
5748
5749         /*
5750          * The Linux driver uses periodic interrupts to avoid races.
5751          * We cargo-cult like it's going out of fashion.
5752          */
5753         if (r1 & IWM_CSR_INT_BIT_RX_PERIODIC) {
5754                 handled |= IWM_CSR_INT_BIT_RX_PERIODIC;
5755                 IWM_WRITE(sc, IWM_CSR_INT, IWM_CSR_INT_BIT_RX_PERIODIC);
5756                 if ((r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX)) == 0)
5757                         IWM_WRITE_1(sc,
5758                             IWM_CSR_INT_PERIODIC_REG, IWM_CSR_INT_PERIODIC_DIS);
5759                 isperiodic = 1;
5760         }
5761
5762         if ((r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX)) || isperiodic) {
5763                 handled |= (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX);
5764                 IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, IWM_CSR_FH_INT_RX_MASK);
5765
5766                 iwm_notif_intr(sc);
5767
5768                 /* enable periodic interrupt, see above */
5769                 if (r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX) && !isperiodic)
5770                         IWM_WRITE_1(sc, IWM_CSR_INT_PERIODIC_REG,
5771                             IWM_CSR_INT_PERIODIC_ENA);
5772         }
5773
5774         if (__predict_false(r1 & ~handled))
5775                 IWM_DPRINTF(sc, IWM_DEBUG_INTR,
5776                     "%s: unhandled interrupts: %x\n", __func__, r1);
5777         rv = 1;
5778
5779  out_ena:
5780         iwm_restore_interrupts(sc);
5781  out:
5782         IWM_UNLOCK(sc);
5783         return;
5784 }
5785
5786 /*
5787  * Autoconf glue-sniffing
5788  */
5789 #define PCI_VENDOR_INTEL                0x8086
5790 #define PCI_PRODUCT_INTEL_WL_3160_1     0x08b3
5791 #define PCI_PRODUCT_INTEL_WL_3160_2     0x08b4
5792 #define PCI_PRODUCT_INTEL_WL_3165_1     0x3165
5793 #define PCI_PRODUCT_INTEL_WL_3165_2     0x3166
5794 #define PCI_PRODUCT_INTEL_WL_7260_1     0x08b1
5795 #define PCI_PRODUCT_INTEL_WL_7260_2     0x08b2
5796 #define PCI_PRODUCT_INTEL_WL_7265_1     0x095a
5797 #define PCI_PRODUCT_INTEL_WL_7265_2     0x095b
5798 #define PCI_PRODUCT_INTEL_WL_8260_1     0x24f3
5799 #define PCI_PRODUCT_INTEL_WL_8260_2     0x24f4
5800
5801 static const struct iwm_devices {
5802         uint16_t                device;
5803         const struct iwm_cfg    *cfg;
5804 } iwm_devices[] = {
5805         { PCI_PRODUCT_INTEL_WL_3160_1, &iwm3160_cfg },
5806         { PCI_PRODUCT_INTEL_WL_3160_2, &iwm3160_cfg },
5807         { PCI_PRODUCT_INTEL_WL_3165_1, &iwm3165_cfg },
5808         { PCI_PRODUCT_INTEL_WL_3165_2, &iwm3165_cfg },
5809         { PCI_PRODUCT_INTEL_WL_7260_1, &iwm7260_cfg },
5810         { PCI_PRODUCT_INTEL_WL_7260_2, &iwm7260_cfg },
5811         { PCI_PRODUCT_INTEL_WL_7265_1, &iwm7265_cfg },
5812         { PCI_PRODUCT_INTEL_WL_7265_2, &iwm7265_cfg },
5813         { PCI_PRODUCT_INTEL_WL_8260_1, &iwm8260_cfg },
5814         { PCI_PRODUCT_INTEL_WL_8260_2, &iwm8260_cfg },
5815 };
5816
5817 static int
5818 iwm_probe(device_t dev)
5819 {
5820         int i;
5821
5822         for (i = 0; i < nitems(iwm_devices); i++) {
5823                 if (pci_get_vendor(dev) == PCI_VENDOR_INTEL &&
5824                     pci_get_device(dev) == iwm_devices[i].device) {
5825                         device_set_desc(dev, iwm_devices[i].cfg->name);
5826                         return (BUS_PROBE_DEFAULT);
5827                 }
5828         }
5829
5830         return (ENXIO);
5831 }
5832
5833 static int
5834 iwm_dev_check(device_t dev)
5835 {
5836         struct iwm_softc *sc;
5837         uint16_t devid;
5838         int i;
5839
5840         sc = device_get_softc(dev);
5841
5842         devid = pci_get_device(dev);
5843         for (i = 0; i < nitems(iwm_devices); i++) {
5844                 if (iwm_devices[i].device == devid) {
5845                         sc->cfg = iwm_devices[i].cfg;
5846                         return (0);
5847                 }
5848         }
5849         device_printf(dev, "unknown adapter type\n");
5850         return ENXIO;
5851 }
5852
5853 /* PCI registers */
5854 #define PCI_CFG_RETRY_TIMEOUT   0x041
5855
5856 static int
5857 iwm_pci_attach(device_t dev)
5858 {
5859         struct iwm_softc *sc;
5860         int count, error, rid;
5861         uint16_t reg;
5862
5863         sc = device_get_softc(dev);
5864
5865         /* We disable the RETRY_TIMEOUT register (0x41) to keep
5866          * PCI Tx retries from interfering with C3 CPU state */
5867         pci_write_config(dev, PCI_CFG_RETRY_TIMEOUT, 0x00, 1);
5868
5869         /* Enable bus-mastering and hardware bug workaround. */
5870         pci_enable_busmaster(dev);
5871         reg = pci_read_config(dev, PCIR_STATUS, sizeof(reg));
5872         /* if !MSI */
5873         if (reg & PCIM_STATUS_INTxSTATE) {
5874                 reg &= ~PCIM_STATUS_INTxSTATE;
5875         }
5876         pci_write_config(dev, PCIR_STATUS, reg, sizeof(reg));
5877
5878         rid = PCIR_BAR(0);
5879         sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
5880             RF_ACTIVE);
5881         if (sc->sc_mem == NULL) {
5882                 device_printf(sc->sc_dev, "can't map mem space\n");
5883                 return (ENXIO);
5884         }
5885         sc->sc_st = rman_get_bustag(sc->sc_mem);
5886         sc->sc_sh = rman_get_bushandle(sc->sc_mem);
5887
5888         /* Install interrupt handler. */
5889         count = 1;
5890         rid = 0;
5891         if (pci_alloc_msi(dev, &count) == 0)
5892                 rid = 1;
5893         sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE |
5894             (rid != 0 ? 0 : RF_SHAREABLE));
5895         if (sc->sc_irq == NULL) {
5896                 device_printf(dev, "can't map interrupt\n");
5897                         return (ENXIO);
5898         }
5899         error = bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
5900             NULL, iwm_intr, sc, &sc->sc_ih);
5901         if (sc->sc_ih == NULL) {
5902                 device_printf(dev, "can't establish interrupt");
5903                         return (ENXIO);
5904         }
5905         sc->sc_dmat = bus_get_dma_tag(sc->sc_dev);
5906
5907         return (0);
5908 }
5909
5910 static void
5911 iwm_pci_detach(device_t dev)
5912 {
5913         struct iwm_softc *sc = device_get_softc(dev);
5914
5915         if (sc->sc_irq != NULL) {
5916                 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
5917                 bus_release_resource(dev, SYS_RES_IRQ,
5918                     rman_get_rid(sc->sc_irq), sc->sc_irq);
5919                 pci_release_msi(dev);
5920         }
5921         if (sc->sc_mem != NULL)
5922                 bus_release_resource(dev, SYS_RES_MEMORY,
5923                     rman_get_rid(sc->sc_mem), sc->sc_mem);
5924 }
5925
5926
5927
5928 static int
5929 iwm_attach(device_t dev)
5930 {
5931         struct iwm_softc *sc = device_get_softc(dev);
5932         struct ieee80211com *ic = &sc->sc_ic;
5933         int error;
5934         int txq_i, i;
5935
5936         sc->sc_dev = dev;
5937         sc->sc_attached = 1;
5938         IWM_LOCK_INIT(sc);
5939         mbufq_init(&sc->sc_snd, ifqmaxlen);
5940         callout_init_mtx(&sc->sc_watchdog_to, &sc->sc_mtx, 0);
5941         callout_init_mtx(&sc->sc_led_blink_to, &sc->sc_mtx, 0);
5942         TASK_INIT(&sc->sc_es_task, 0, iwm_endscan_cb, sc);
5943
5944         sc->sc_notif_wait = iwm_notification_wait_init(sc);
5945         if (sc->sc_notif_wait == NULL) {
5946                 device_printf(dev, "failed to init notification wait struct\n");
5947                 goto fail;
5948         }
5949
5950         /* Init phy db */
5951         sc->sc_phy_db = iwm_phy_db_init(sc);
5952         if (!sc->sc_phy_db) {
5953                 device_printf(dev, "Cannot init phy_db\n");
5954                 goto fail;
5955         }
5956
5957         /* PCI attach */
5958         error = iwm_pci_attach(dev);
5959         if (error != 0)
5960                 goto fail;
5961
5962         sc->sc_wantresp = -1;
5963
5964         /* Check device type */
5965         error = iwm_dev_check(dev);
5966         if (error != 0)
5967                 goto fail;
5968
5969         sc->sc_hw_rev = IWM_READ(sc, IWM_CSR_HW_REV);
5970         /*
5971          * In the 8000 HW family the format of the 4 bytes of CSR_HW_REV have
5972          * changed, and now the revision step also includes bit 0-1 (no more
5973          * "dash" value). To keep hw_rev backwards compatible - we'll store it
5974          * in the old format.
5975          */
5976         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000)
5977                 sc->sc_hw_rev = (sc->sc_hw_rev & 0xfff0) |
5978                                 (IWM_CSR_HW_REV_STEP(sc->sc_hw_rev << 2) << 2);
5979
5980         if (iwm_prepare_card_hw(sc) != 0) {
5981                 device_printf(dev, "could not initialize hardware\n");
5982                 goto fail;
5983         }
5984
5985         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) {
5986                 int ret;
5987                 uint32_t hw_step;
5988
5989                 /*
5990                  * In order to recognize C step the driver should read the
5991                  * chip version id located at the AUX bus MISC address.
5992                  */
5993                 IWM_SETBITS(sc, IWM_CSR_GP_CNTRL,
5994                             IWM_CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
5995                 DELAY(2);
5996
5997                 ret = iwm_poll_bit(sc, IWM_CSR_GP_CNTRL,
5998                                    IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
5999                                    IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
6000                                    25000);
6001                 if (!ret) {
6002                         device_printf(sc->sc_dev,
6003                             "Failed to wake up the nic\n");
6004                         goto fail;
6005                 }
6006
6007                 if (iwm_nic_lock(sc)) {
6008                         hw_step = iwm_read_prph(sc, IWM_WFPM_CTRL_REG);
6009                         hw_step |= IWM_ENABLE_WFPM;
6010                         iwm_write_prph(sc, IWM_WFPM_CTRL_REG, hw_step);
6011                         hw_step = iwm_read_prph(sc, IWM_AUX_MISC_REG);
6012                         hw_step = (hw_step >> IWM_HW_STEP_LOCATION_BITS) & 0xF;
6013                         if (hw_step == 0x3)
6014                                 sc->sc_hw_rev = (sc->sc_hw_rev & 0xFFFFFFF3) |
6015                                                 (IWM_SILICON_C_STEP << 2);
6016                         iwm_nic_unlock(sc);
6017                 } else {
6018                         device_printf(sc->sc_dev, "Failed to lock the nic\n");
6019                         goto fail;
6020                 }
6021         }
6022
6023         /* special-case 7265D, it has the same PCI IDs. */
6024         if (sc->cfg == &iwm7265_cfg &&
6025             (sc->sc_hw_rev & IWM_CSR_HW_REV_TYPE_MSK) == IWM_CSR_HW_REV_TYPE_7265D) {
6026                 sc->cfg = &iwm7265d_cfg;
6027         }
6028
6029         /* Allocate DMA memory for firmware transfers. */
6030         if ((error = iwm_alloc_fwmem(sc)) != 0) {
6031                 device_printf(dev, "could not allocate memory for firmware\n");
6032                 goto fail;
6033         }
6034
6035         /* Allocate "Keep Warm" page. */
6036         if ((error = iwm_alloc_kw(sc)) != 0) {
6037                 device_printf(dev, "could not allocate keep warm page\n");
6038                 goto fail;
6039         }
6040
6041         /* We use ICT interrupts */
6042         if ((error = iwm_alloc_ict(sc)) != 0) {
6043                 device_printf(dev, "could not allocate ICT table\n");
6044                 goto fail;
6045         }
6046
6047         /* Allocate TX scheduler "rings". */
6048         if ((error = iwm_alloc_sched(sc)) != 0) {
6049                 device_printf(dev, "could not allocate TX scheduler rings\n");
6050                 goto fail;
6051         }
6052
6053         /* Allocate TX rings */
6054         for (txq_i = 0; txq_i < nitems(sc->txq); txq_i++) {
6055                 if ((error = iwm_alloc_tx_ring(sc,
6056                     &sc->txq[txq_i], txq_i)) != 0) {
6057                         device_printf(dev,
6058                             "could not allocate TX ring %d\n",
6059                             txq_i);
6060                         goto fail;
6061                 }
6062         }
6063
6064         /* Allocate RX ring. */
6065         if ((error = iwm_alloc_rx_ring(sc, &sc->rxq)) != 0) {
6066                 device_printf(dev, "could not allocate RX ring\n");
6067                 goto fail;
6068         }
6069
6070         /* Clear pending interrupts. */
6071         IWM_WRITE(sc, IWM_CSR_INT, 0xffffffff);
6072
6073         ic->ic_softc = sc;
6074         ic->ic_name = device_get_nameunit(sc->sc_dev);
6075         ic->ic_phytype = IEEE80211_T_OFDM;      /* not only, but not used */
6076         ic->ic_opmode = IEEE80211_M_STA;        /* default to BSS mode */
6077
6078         /* Set device capabilities. */
6079         ic->ic_caps =
6080             IEEE80211_C_STA |
6081             IEEE80211_C_WPA |           /* WPA/RSN */
6082             IEEE80211_C_WME |
6083             IEEE80211_C_SHSLOT |        /* short slot time supported */
6084             IEEE80211_C_SHPREAMBLE      /* short preamble supported */
6085 //          IEEE80211_C_BGSCAN          /* capable of bg scanning */
6086             ;
6087         for (i = 0; i < nitems(sc->sc_phyctxt); i++) {
6088                 sc->sc_phyctxt[i].id = i;
6089                 sc->sc_phyctxt[i].color = 0;
6090                 sc->sc_phyctxt[i].ref = 0;
6091                 sc->sc_phyctxt[i].channel = NULL;
6092         }
6093
6094         /* Default noise floor */
6095         sc->sc_noise = -96;
6096
6097         /* Max RSSI */
6098         sc->sc_max_rssi = IWM_MAX_DBM - IWM_MIN_DBM;
6099
6100         sc->sc_preinit_hook.ich_func = iwm_preinit;
6101         sc->sc_preinit_hook.ich_arg = sc;
6102         if (config_intrhook_establish(&sc->sc_preinit_hook) != 0) {
6103                 device_printf(dev, "config_intrhook_establish failed\n");
6104                 goto fail;
6105         }
6106
6107 #ifdef IWM_DEBUG
6108         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
6109             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug",
6110             CTLFLAG_RW, &sc->sc_debug, 0, "control debugging");
6111 #endif
6112
6113         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6114             "<-%s\n", __func__);
6115
6116         return 0;
6117
6118         /* Free allocated memory if something failed during attachment. */
6119 fail:
6120         iwm_detach_local(sc, 0);
6121
6122         return ENXIO;
6123 }
6124
6125 static int
6126 iwm_is_valid_ether_addr(uint8_t *addr)
6127 {
6128         char zero_addr[IEEE80211_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 };
6129
6130         if ((addr[0] & 1) || IEEE80211_ADDR_EQ(zero_addr, addr))
6131                 return (FALSE);
6132
6133         return (TRUE);
6134 }
6135
6136 static int
6137 iwm_update_edca(struct ieee80211com *ic)
6138 {
6139         struct iwm_softc *sc = ic->ic_softc;
6140
6141         device_printf(sc->sc_dev, "%s: called\n", __func__);
6142         return (0);
6143 }
6144
6145 static void
6146 iwm_preinit(void *arg)
6147 {
6148         struct iwm_softc *sc = arg;
6149         device_t dev = sc->sc_dev;
6150         struct ieee80211com *ic = &sc->sc_ic;
6151         int error;
6152
6153         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6154             "->%s\n", __func__);
6155
6156         IWM_LOCK(sc);
6157         if ((error = iwm_start_hw(sc)) != 0) {
6158                 device_printf(dev, "could not initialize hardware\n");
6159                 IWM_UNLOCK(sc);
6160                 goto fail;
6161         }
6162
6163         error = iwm_run_init_mvm_ucode(sc, 1);
6164         iwm_stop_device(sc);
6165         if (error) {
6166                 IWM_UNLOCK(sc);
6167                 goto fail;
6168         }
6169         device_printf(dev,
6170             "hw rev 0x%x, fw ver %s, address %s\n",
6171             sc->sc_hw_rev & IWM_CSR_HW_REV_TYPE_MSK,
6172             sc->sc_fwver, ether_sprintf(sc->nvm_data->hw_addr));
6173
6174         /* not all hardware can do 5GHz band */
6175         if (!sc->nvm_data->sku_cap_band_52GHz_enable)
6176                 memset(&ic->ic_sup_rates[IEEE80211_MODE_11A], 0,
6177                     sizeof(ic->ic_sup_rates[IEEE80211_MODE_11A]));
6178         IWM_UNLOCK(sc);
6179
6180         iwm_init_channel_map(ic, IEEE80211_CHAN_MAX, &ic->ic_nchans,
6181             ic->ic_channels);
6182
6183         /*
6184          * At this point we've committed - if we fail to do setup,
6185          * we now also have to tear down the net80211 state.
6186          */
6187         ieee80211_ifattach(ic);
6188         ic->ic_vap_create = iwm_vap_create;
6189         ic->ic_vap_delete = iwm_vap_delete;
6190         ic->ic_raw_xmit = iwm_raw_xmit;
6191         ic->ic_node_alloc = iwm_node_alloc;
6192         ic->ic_scan_start = iwm_scan_start;
6193         ic->ic_scan_end = iwm_scan_end;
6194         ic->ic_update_mcast = iwm_update_mcast;
6195         ic->ic_getradiocaps = iwm_init_channel_map;
6196         ic->ic_set_channel = iwm_set_channel;
6197         ic->ic_scan_curchan = iwm_scan_curchan;
6198         ic->ic_scan_mindwell = iwm_scan_mindwell;
6199         ic->ic_wme.wme_update = iwm_update_edca;
6200         ic->ic_parent = iwm_parent;
6201         ic->ic_transmit = iwm_transmit;
6202         iwm_radiotap_attach(sc);
6203         if (bootverbose)
6204                 ieee80211_announce(ic);
6205
6206         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6207             "<-%s\n", __func__);
6208         config_intrhook_disestablish(&sc->sc_preinit_hook);
6209
6210         return;
6211 fail:
6212         config_intrhook_disestablish(&sc->sc_preinit_hook);
6213         iwm_detach_local(sc, 0);
6214 }
6215
6216 /*
6217  * Attach the interface to 802.11 radiotap.
6218  */
6219 static void
6220 iwm_radiotap_attach(struct iwm_softc *sc)
6221 {
6222         struct ieee80211com *ic = &sc->sc_ic;
6223
6224         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6225             "->%s begin\n", __func__);
6226         ieee80211_radiotap_attach(ic,
6227             &sc->sc_txtap.wt_ihdr, sizeof(sc->sc_txtap),
6228                 IWM_TX_RADIOTAP_PRESENT,
6229             &sc->sc_rxtap.wr_ihdr, sizeof(sc->sc_rxtap),
6230                 IWM_RX_RADIOTAP_PRESENT);
6231         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6232             "->%s end\n", __func__);
6233 }
6234
6235 static struct ieee80211vap *
6236 iwm_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], int unit,
6237     enum ieee80211_opmode opmode, int flags,
6238     const uint8_t bssid[IEEE80211_ADDR_LEN],
6239     const uint8_t mac[IEEE80211_ADDR_LEN])
6240 {
6241         struct iwm_vap *ivp;
6242         struct ieee80211vap *vap;
6243
6244         if (!TAILQ_EMPTY(&ic->ic_vaps))         /* only one at a time */
6245                 return NULL;
6246         ivp = malloc(sizeof(struct iwm_vap), M_80211_VAP, M_WAITOK | M_ZERO);
6247         vap = &ivp->iv_vap;
6248         ieee80211_vap_setup(ic, vap, name, unit, opmode, flags, bssid);
6249         vap->iv_bmissthreshold = 10;            /* override default */
6250         /* Override with driver methods. */
6251         ivp->iv_newstate = vap->iv_newstate;
6252         vap->iv_newstate = iwm_newstate;
6253
6254         ieee80211_ratectl_init(vap);
6255         /* Complete setup. */
6256         ieee80211_vap_attach(vap, iwm_media_change, ieee80211_media_status,
6257             mac);
6258         ic->ic_opmode = opmode;
6259
6260         return vap;
6261 }
6262
6263 static void
6264 iwm_vap_delete(struct ieee80211vap *vap)
6265 {
6266         struct iwm_vap *ivp = IWM_VAP(vap);
6267
6268         ieee80211_ratectl_deinit(vap);
6269         ieee80211_vap_detach(vap);
6270         free(ivp, M_80211_VAP);
6271 }
6272
6273 static void
6274 iwm_scan_start(struct ieee80211com *ic)
6275 {
6276         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
6277         struct iwm_softc *sc = ic->ic_softc;
6278         int error;
6279
6280         IWM_LOCK(sc);
6281         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
6282                 /* This should not be possible */
6283                 device_printf(sc->sc_dev,
6284                     "%s: Previous scan not completed yet\n", __func__);
6285         }
6286         if (fw_has_capa(&sc->ucode_capa, IWM_UCODE_TLV_CAPA_UMAC_SCAN))
6287                 error = iwm_mvm_umac_scan(sc);
6288         else
6289                 error = iwm_mvm_lmac_scan(sc);
6290         if (error != 0) {
6291                 device_printf(sc->sc_dev, "could not initiate scan\n");
6292                 IWM_UNLOCK(sc);
6293                 ieee80211_cancel_scan(vap);
6294         } else {
6295                 sc->sc_flags |= IWM_FLAG_SCAN_RUNNING;
6296                 iwm_led_blink_start(sc);
6297                 IWM_UNLOCK(sc);
6298         }
6299 }
6300
6301 static void
6302 iwm_scan_end(struct ieee80211com *ic)
6303 {
6304         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
6305         struct iwm_softc *sc = ic->ic_softc;
6306
6307         IWM_LOCK(sc);
6308         iwm_led_blink_stop(sc);
6309         if (vap->iv_state == IEEE80211_S_RUN)
6310                 iwm_mvm_led_enable(sc);
6311         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
6312                 /*
6313                  * Removing IWM_FLAG_SCAN_RUNNING now, is fine because
6314                  * both iwm_scan_end and iwm_scan_start run in the ic->ic_tq
6315                  * taskqueue.
6316                  */
6317                 sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
6318                 iwm_mvm_scan_stop_wait(sc);
6319         }
6320         IWM_UNLOCK(sc);
6321
6322         /*
6323          * Make sure we don't race, if sc_es_task is still enqueued here.
6324          * This is to make sure that it won't call ieee80211_scan_done
6325          * when we have already started the next scan.
6326          */
6327         taskqueue_cancel(ic->ic_tq, &sc->sc_es_task, NULL);
6328 }
6329
6330 static void
6331 iwm_update_mcast(struct ieee80211com *ic)
6332 {
6333 }
6334
6335 static void
6336 iwm_set_channel(struct ieee80211com *ic)
6337 {
6338 }
6339
6340 static void
6341 iwm_scan_curchan(struct ieee80211_scan_state *ss, unsigned long maxdwell)
6342 {
6343 }
6344
6345 static void
6346 iwm_scan_mindwell(struct ieee80211_scan_state *ss)
6347 {
6348         return;
6349 }
6350
6351 void
6352 iwm_init_task(void *arg1)
6353 {
6354         struct iwm_softc *sc = arg1;
6355
6356         IWM_LOCK(sc);
6357         while (sc->sc_flags & IWM_FLAG_BUSY)
6358                 msleep(&sc->sc_flags, &sc->sc_mtx, 0, "iwmpwr", 0);
6359         sc->sc_flags |= IWM_FLAG_BUSY;
6360         iwm_stop(sc);
6361         if (sc->sc_ic.ic_nrunning > 0)
6362                 iwm_init(sc);
6363         sc->sc_flags &= ~IWM_FLAG_BUSY;
6364         wakeup(&sc->sc_flags);
6365         IWM_UNLOCK(sc);
6366 }
6367
6368 static int
6369 iwm_resume(device_t dev)
6370 {
6371         struct iwm_softc *sc = device_get_softc(dev);
6372         int do_reinit = 0;
6373
6374         /*
6375          * We disable the RETRY_TIMEOUT register (0x41) to keep
6376          * PCI Tx retries from interfering with C3 CPU state.
6377          */
6378         pci_write_config(dev, PCI_CFG_RETRY_TIMEOUT, 0x00, 1);
6379         iwm_init_task(device_get_softc(dev));
6380
6381         IWM_LOCK(sc);
6382         if (sc->sc_flags & IWM_FLAG_SCANNING) {
6383                 sc->sc_flags &= ~IWM_FLAG_SCANNING;
6384                 do_reinit = 1;
6385         }
6386         IWM_UNLOCK(sc);
6387
6388         if (do_reinit)
6389                 ieee80211_resume_all(&sc->sc_ic);
6390
6391         return 0;
6392 }
6393
6394 static int
6395 iwm_suspend(device_t dev)
6396 {
6397         int do_stop = 0;
6398         struct iwm_softc *sc = device_get_softc(dev);
6399
6400         do_stop = !! (sc->sc_ic.ic_nrunning > 0);
6401
6402         ieee80211_suspend_all(&sc->sc_ic);
6403
6404         if (do_stop) {
6405                 IWM_LOCK(sc);
6406                 iwm_stop(sc);
6407                 sc->sc_flags |= IWM_FLAG_SCANNING;
6408                 IWM_UNLOCK(sc);
6409         }
6410
6411         return (0);
6412 }
6413
6414 static int
6415 iwm_detach_local(struct iwm_softc *sc, int do_net80211)
6416 {
6417         struct iwm_fw_info *fw = &sc->sc_fw;
6418         device_t dev = sc->sc_dev;
6419         int i;
6420
6421         if (!sc->sc_attached)
6422                 return 0;
6423         sc->sc_attached = 0;
6424
6425         if (do_net80211)
6426                 ieee80211_draintask(&sc->sc_ic, &sc->sc_es_task);
6427
6428         callout_drain(&sc->sc_led_blink_to);
6429         callout_drain(&sc->sc_watchdog_to);
6430         iwm_stop_device(sc);
6431         if (do_net80211) {
6432                 ieee80211_ifdetach(&sc->sc_ic);
6433         }
6434
6435         iwm_phy_db_free(sc->sc_phy_db);
6436         sc->sc_phy_db = NULL;
6437
6438         iwm_free_nvm_data(sc->nvm_data);
6439
6440         /* Free descriptor rings */
6441         iwm_free_rx_ring(sc, &sc->rxq);
6442         for (i = 0; i < nitems(sc->txq); i++)
6443                 iwm_free_tx_ring(sc, &sc->txq[i]);
6444
6445         /* Free firmware */
6446         if (fw->fw_fp != NULL)
6447                 iwm_fw_info_free(fw);
6448
6449         /* Free scheduler */
6450         iwm_dma_contig_free(&sc->sched_dma);
6451         iwm_dma_contig_free(&sc->ict_dma);
6452         iwm_dma_contig_free(&sc->kw_dma);
6453         iwm_dma_contig_free(&sc->fw_dma);
6454
6455         iwm_free_fw_paging(sc);
6456
6457         /* Finished with the hardware - detach things */
6458         iwm_pci_detach(dev);
6459
6460         if (sc->sc_notif_wait != NULL) {
6461                 iwm_notification_wait_free(sc->sc_notif_wait);
6462                 sc->sc_notif_wait = NULL;
6463         }
6464
6465         mbufq_drain(&sc->sc_snd);
6466         IWM_LOCK_DESTROY(sc);
6467
6468         return (0);
6469 }
6470
6471 static int
6472 iwm_detach(device_t dev)
6473 {
6474         struct iwm_softc *sc = device_get_softc(dev);
6475
6476         return (iwm_detach_local(sc, 1));
6477 }
6478
6479 static device_method_t iwm_pci_methods[] = {
6480         /* Device interface */
6481         DEVMETHOD(device_probe,         iwm_probe),
6482         DEVMETHOD(device_attach,        iwm_attach),
6483         DEVMETHOD(device_detach,        iwm_detach),
6484         DEVMETHOD(device_suspend,       iwm_suspend),
6485         DEVMETHOD(device_resume,        iwm_resume),
6486
6487         DEVMETHOD_END
6488 };
6489
6490 static driver_t iwm_pci_driver = {
6491         "iwm",
6492         iwm_pci_methods,
6493         sizeof (struct iwm_softc)
6494 };
6495
6496 static devclass_t iwm_devclass;
6497
6498 DRIVER_MODULE(iwm, pci, iwm_pci_driver, iwm_devclass, NULL, NULL);
6499 MODULE_DEPEND(iwm, firmware, 1, 1, 1);
6500 MODULE_DEPEND(iwm, pci, 1, 1, 1);
6501 MODULE_DEPEND(iwm, wlan, 1, 1, 1);