sys/dev/iwm/if_iwm.c

   1 /*      $OpenBSD: if_iwm.c,v 1.167 2017/04/04 00:40:52 claudio Exp $    */
   2
   3 /*
   4  * Copyright (c) 2014 genua mbh <info@genua.de>
   5  * Copyright (c) 2014 Fixup Software Ltd.
   6  *
   7  * Permission to use, copy, modify, and distribute this software for any
   8  * purpose with or without fee is hereby granted, provided that the above
   9  * copyright notice and this permission notice appear in all copies.
  10  *
  11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
  14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
  15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
  16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
  17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
  18  */
  19
  20 /*-
  21  * Based on BSD-licensed source modules in the Linux iwlwifi driver,
  22  * which were used as the reference documentation for this implementation.
  23  *
  24  * Driver version we are currently based off of is
  25  * Linux 3.14.3 (tag id a2df521e42b1d9a23f620ac79dbfe8655a8391dd)
  26  *
  27  ***********************************************************************
  28  *
  29  * This file is provided under a dual BSD/GPLv2 license.  When using or
  30  * redistributing this file, you may do so under either license.
  31  *
  32  * GPL LICENSE SUMMARY
  33  *
  34  * Copyright(c) 2007 - 2013 Intel Corporation. All rights reserved.
  35  *
  36  * This program is free software; you can redistribute it and/or modify
  37  * it under the terms of version 2 of the GNU General Public License as
  38  * published by the Free Software Foundation.
  39  *
  40  * This program is distributed in the hope that it will be useful, but
  41  * WITHOUT ANY WARRANTY; without even the implied warranty of
  42  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  43  * General Public License for more details.
  44  *
  45  * You should have received a copy of the GNU General Public License
  46  * along with this program; if not, write to the Free Software
  47  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110,
  48  * USA
  49  *
  50  * The full GNU General Public License is included in this distribution
  51  * in the file called COPYING.
  52  *
  53  * Contact Information:
  54  *  Intel Linux Wireless <ilw@linux.intel.com>
  55  * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
  56  *
  57  *
  58  * BSD LICENSE
  59  *
  60  * Copyright(c) 2005 - 2013 Intel Corporation. All rights reserved.
  61  * All rights reserved.
  62  *
  63  * Redistribution and use in source and binary forms, with or without
  64  * modification, are permitted provided that the following conditions
  65  * are met:
  66  *
  67  *  * Redistributions of source code must retain the above copyright
  68  *    notice, this list of conditions and the following disclaimer.
  69  *  * Redistributions in binary form must reproduce the above copyright
  70  *    notice, this list of conditions and the following disclaimer in
  71  *    the documentation and/or other materials provided with the
  72  *    distribution.
  73  *  * Neither the name Intel Corporation nor the names of its
  74  *    contributors may be used to endorse or promote products derived
  75  *    from this software without specific prior written permission.
  76  *
  77  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  78  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  79  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  80  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  81  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  82  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  83  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  84  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  85  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  86  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  87  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  88  */
  89
  90 /*-
  91  * Copyright (c) 2007-2010 Damien Bergamini <damien.bergamini@free.fr>
  92  *
  93  * Permission to use, copy, modify, and distribute this software for any
  94  * purpose with or without fee is hereby granted, provided that the above
  95  * copyright notice and this permission notice appear in all copies.
  96  *
  97  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
  98  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
  99  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 100  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 101  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 102  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 103  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 104  */
 105 #include <sys/cdefs.h>
 106 __FBSDID("$FreeBSD$");
 107
 108 #include "opt_wlan.h"
 109 #include "opt_iwm.h"
 110
 111 #include <sys/param.h>
 112 #include <sys/bus.h>
 113 #include <sys/conf.h>
 114 #include <sys/endian.h>
 115 #include <sys/firmware.h>
 116 #include <sys/kernel.h>
 117 #include <sys/malloc.h>
 118 #include <sys/mbuf.h>
 119 #include <sys/mutex.h>
 120 #include <sys/module.h>
 121 #include <sys/proc.h>
 122 #include <sys/rman.h>
 123 #include <sys/socket.h>
 124 #include <sys/sockio.h>
 125 #include <sys/sysctl.h>
 126 #include <sys/linker.h>
 127
 128 #include <machine/bus.h>
 129 #include <machine/endian.h>
 130 #include <machine/resource.h>
 131
 132 #include <dev/pci/pcivar.h>
 133 #include <dev/pci/pcireg.h>
 134
 135 #include <net/bpf.h>
 136
 137 #include <net/if.h>
 138 #include <net/if_var.h>
 139 #include <net/if_arp.h>
 140 #include <net/if_dl.h>
 141 #include <net/if_media.h>
 142 #include <net/if_types.h>
 143
 144 #include <netinet/in.h>
 145 #include <netinet/in_systm.h>
 146 #include <netinet/if_ether.h>
 147 #include <netinet/ip.h>
 148
 149 #include <net80211/ieee80211_var.h>
 150 #include <net80211/ieee80211_regdomain.h>
 151 #include <net80211/ieee80211_ratectl.h>
 152 #include <net80211/ieee80211_radiotap.h>
 153
 154 #include <dev/iwm/if_iwmreg.h>
 155 #include <dev/iwm/if_iwmvar.h>
 156 #include <dev/iwm/if_iwm_config.h>
 157 #include <dev/iwm/if_iwm_debug.h>
 158 #include <dev/iwm/if_iwm_notif_wait.h>
 159 #include <dev/iwm/if_iwm_util.h>
 160 #include <dev/iwm/if_iwm_binding.h>
 161 #include <dev/iwm/if_iwm_phy_db.h>
 162 #include <dev/iwm/if_iwm_mac_ctxt.h>
 163 #include <dev/iwm/if_iwm_phy_ctxt.h>
 164 #include <dev/iwm/if_iwm_time_event.h>
 165 #include <dev/iwm/if_iwm_power.h>
 166 #include <dev/iwm/if_iwm_scan.h>
 167 #include <dev/iwm/if_iwm_sf.h>
 168 #include <dev/iwm/if_iwm_sta.h>
 169
 170 #include <dev/iwm/if_iwm_pcie_trans.h>
 171 #include <dev/iwm/if_iwm_led.h>
 172 #include <dev/iwm/if_iwm_fw.h>
 173
 174 /* From DragonflyBSD */
 175 #define mtodoff(m, t, off)      ((t)((m)->m_data + (off)))
 176
 177 const uint8_t iwm_nvm_channels[] = {
 178         /* 2.4 GHz */
 179         1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
 180         /* 5 GHz */
 181         36, 40, 44, 48, 52, 56, 60, 64,
 182         100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144,
 183         149, 153, 157, 161, 165
 184 };
 185 _Static_assert(nitems(iwm_nvm_channels) <= IWM_NUM_CHANNELS,
 186     "IWM_NUM_CHANNELS is too small");
 187
 188 const uint8_t iwm_nvm_channels_8000[] = {
 189         /* 2.4 GHz */
 190         1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14,
 191         /* 5 GHz */
 192         36, 40, 44, 48, 52, 56, 60, 64, 68, 72, 76, 80, 84, 88, 92,
 193         96, 100, 104, 108, 112, 116, 120, 124, 128, 132, 136, 140, 144,
 194         149, 153, 157, 161, 165, 169, 173, 177, 181
 195 };
 196 _Static_assert(nitems(iwm_nvm_channels_8000) <= IWM_NUM_CHANNELS_8000,
 197     "IWM_NUM_CHANNELS_8000 is too small");
 198
 199 #define IWM_NUM_2GHZ_CHANNELS   14
 200 #define IWM_N_HW_ADDR_MASK      0xF
 201
 202 /*
 203  * XXX For now, there's simply a fixed set of rate table entries
 204  * that are populated.
 205  */
 206 const struct iwm_rate {
 207         uint8_t rate;
 208         uint8_t plcp;
 209 } iwm_rates[] = {
 210         {   2,  IWM_RATE_1M_PLCP  },
 211         {   4,  IWM_RATE_2M_PLCP  },
 212         {  11,  IWM_RATE_5M_PLCP  },
 213         {  22,  IWM_RATE_11M_PLCP },
 214         {  12,  IWM_RATE_6M_PLCP  },
 215         {  18,  IWM_RATE_9M_PLCP  },
 216         {  24,  IWM_RATE_12M_PLCP },
 217         {  36,  IWM_RATE_18M_PLCP },
 218         {  48,  IWM_RATE_24M_PLCP },
 219         {  72,  IWM_RATE_36M_PLCP },
 220         {  96,  IWM_RATE_48M_PLCP },
 221         { 108,  IWM_RATE_54M_PLCP },
 222 };
 223 #define IWM_RIDX_CCK    0
 224 #define IWM_RIDX_OFDM   4
 225 #define IWM_RIDX_MAX    (nitems(iwm_rates)-1)
 226 #define IWM_RIDX_IS_CCK(_i_) ((_i_) < IWM_RIDX_OFDM)
 227 #define IWM_RIDX_IS_OFDM(_i_) ((_i_) >= IWM_RIDX_OFDM)
 228
 229 struct iwm_nvm_section {
 230         uint16_t length;
 231         uint8_t *data;
 232 };
 233
 234 #define IWM_MVM_UCODE_ALIVE_TIMEOUT     hz
 235 #define IWM_MVM_UCODE_CALIB_TIMEOUT     (2*hz)
 236
 237 struct iwm_mvm_alive_data {
 238         int valid;
 239         uint32_t scd_base_addr;
 240 };
 241
 242 static int      iwm_store_cscheme(struct iwm_softc *, const uint8_t *, size_t);
 243 static int      iwm_firmware_store_section(struct iwm_softc *,
 244                                            enum iwm_ucode_type,
 245                                            const uint8_t *, size_t);
 246 static int      iwm_set_default_calib(struct iwm_softc *, const void *);
 247 static void     iwm_fw_info_free(struct iwm_fw_info *);
 248 static int      iwm_read_firmware(struct iwm_softc *);
 249 static int      iwm_alloc_fwmem(struct iwm_softc *);
 250 static int      iwm_alloc_sched(struct iwm_softc *);
 251 static int      iwm_alloc_kw(struct iwm_softc *);
 252 static int      iwm_alloc_ict(struct iwm_softc *);
 253 static int      iwm_alloc_rx_ring(struct iwm_softc *, struct iwm_rx_ring *);
 254 static void     iwm_reset_rx_ring(struct iwm_softc *, struct iwm_rx_ring *);
 255 static void     iwm_free_rx_ring(struct iwm_softc *, struct iwm_rx_ring *);
 256 static int      iwm_alloc_tx_ring(struct iwm_softc *, struct iwm_tx_ring *,
 257                                   int);
 258 static void     iwm_reset_tx_ring(struct iwm_softc *, struct iwm_tx_ring *);
 259 static void     iwm_free_tx_ring(struct iwm_softc *, struct iwm_tx_ring *);
 260 static void     iwm_enable_interrupts(struct iwm_softc *);
 261 static void     iwm_restore_interrupts(struct iwm_softc *);
 262 static void     iwm_disable_interrupts(struct iwm_softc *);
 263 static void     iwm_ict_reset(struct iwm_softc *);
 264 static int      iwm_allow_mcast(struct ieee80211vap *, struct iwm_softc *);
 265 static void     iwm_stop_device(struct iwm_softc *);
 266 static void     iwm_mvm_nic_config(struct iwm_softc *);
 267 static int      iwm_nic_rx_init(struct iwm_softc *);
 268 static int      iwm_nic_tx_init(struct iwm_softc *);
 269 static int      iwm_nic_init(struct iwm_softc *);
 270 static int      iwm_trans_pcie_fw_alive(struct iwm_softc *, uint32_t);
 271 static int      iwm_nvm_read_chunk(struct iwm_softc *, uint16_t, uint16_t,
 272                                    uint16_t, uint8_t *, uint16_t *);
 273 static int      iwm_nvm_read_section(struct iwm_softc *, uint16_t, uint8_t *,
 274                                      uint16_t *, uint32_t);
 275 static uint32_t iwm_eeprom_channel_flags(uint16_t);
 276 static void     iwm_add_channel_band(struct iwm_softc *,
 277                     struct ieee80211_channel[], int, int *, int, size_t,
 278                     const uint8_t[]);
 279 static void     iwm_init_channel_map(struct ieee80211com *, int, int *,
 280                     struct ieee80211_channel[]);
 281 static struct iwm_nvm_data *
 282         iwm_parse_nvm_data(struct iwm_softc *, const uint16_t *,
 283                            const uint16_t *, const uint16_t *,
 284                            const uint16_t *, const uint16_t *,
 285                            const uint16_t *);
 286 static void     iwm_free_nvm_data(struct iwm_nvm_data *);
 287 static void     iwm_set_hw_address_family_8000(struct iwm_softc *,
 288                                                struct iwm_nvm_data *,
 289                                                const uint16_t *,
 290                                                const uint16_t *);
 291 static int      iwm_get_sku(const struct iwm_softc *, const uint16_t *,
 292                             const uint16_t *);
 293 static int      iwm_get_nvm_version(const struct iwm_softc *, const uint16_t *);
 294 static int      iwm_get_radio_cfg(const struct iwm_softc *, const uint16_t *,
 295                                   const uint16_t *);
 296 static int      iwm_get_n_hw_addrs(const struct iwm_softc *,
 297                                    const uint16_t *);
 298 static void     iwm_set_radio_cfg(const struct iwm_softc *,
 299                                   struct iwm_nvm_data *, uint32_t);
 300 static struct iwm_nvm_data *
 301         iwm_parse_nvm_sections(struct iwm_softc *, struct iwm_nvm_section *);
 302 static int      iwm_nvm_init(struct iwm_softc *);
 303 static int      iwm_pcie_load_section(struct iwm_softc *, uint8_t,
 304                                       const struct iwm_fw_desc *);
 305 static int      iwm_pcie_load_firmware_chunk(struct iwm_softc *, uint32_t,
 306                                              bus_addr_t, uint32_t);
 307 static int      iwm_pcie_load_cpu_sections_8000(struct iwm_softc *sc,
 308                                                 const struct iwm_fw_img *,
 309                                                 int, int *);
 310 static int      iwm_pcie_load_cpu_sections(struct iwm_softc *,
 311                                            const struct iwm_fw_img *,
 312                                            int, int *);
 313 static int      iwm_pcie_load_given_ucode_8000(struct iwm_softc *,
 314                                                const struct iwm_fw_img *);
 315 static int      iwm_pcie_load_given_ucode(struct iwm_softc *,
 316                                           const struct iwm_fw_img *);
 317 static int      iwm_start_fw(struct iwm_softc *, const struct iwm_fw_img *);
 318 static int      iwm_send_tx_ant_cfg(struct iwm_softc *, uint8_t);
 319 static int      iwm_send_phy_cfg_cmd(struct iwm_softc *);
 320 static int      iwm_mvm_load_ucode_wait_alive(struct iwm_softc *,
 321                                               enum iwm_ucode_type);
 322 static int      iwm_run_init_mvm_ucode(struct iwm_softc *, int);
 323 static int      iwm_mvm_config_ltr(struct iwm_softc *sc);
 324 static int      iwm_rx_addbuf(struct iwm_softc *, int, int);
 325 static int      iwm_mvm_get_signal_strength(struct iwm_softc *,
 326                                             struct iwm_rx_phy_info *);
 327 static void     iwm_mvm_rx_rx_phy_cmd(struct iwm_softc *,
 328                                       struct iwm_rx_packet *);
 329 static int      iwm_get_noise(struct iwm_softc *,
 330                     const struct iwm_mvm_statistics_rx_non_phy *);
 331 static void     iwm_mvm_handle_rx_statistics(struct iwm_softc *,
 332                     struct iwm_rx_packet *);
 333 static boolean_t iwm_mvm_rx_rx_mpdu(struct iwm_softc *, struct mbuf *,
 334                                     uint32_t, boolean_t);
 335 static int      iwm_mvm_rx_tx_cmd_single(struct iwm_softc *,
 336                                          struct iwm_rx_packet *,
 337                                          struct iwm_node *);
 338 static void     iwm_mvm_rx_tx_cmd(struct iwm_softc *, struct iwm_rx_packet *);
 339 static void     iwm_cmd_done(struct iwm_softc *, struct iwm_rx_packet *);
 340 #if 0
 341 static void     iwm_update_sched(struct iwm_softc *, int, int, uint8_t,
 342                                  uint16_t);
 343 #endif
 344 static const struct iwm_rate *
 345         iwm_tx_fill_cmd(struct iwm_softc *, struct iwm_node *,
 346                         struct mbuf *, struct iwm_tx_cmd *);
 347 static int      iwm_tx(struct iwm_softc *, struct mbuf *,
 348                        struct ieee80211_node *, int);
 349 static int      iwm_raw_xmit(struct ieee80211_node *, struct mbuf *,
 350                              const struct ieee80211_bpf_params *);
 351 static int      iwm_mvm_update_quotas(struct iwm_softc *, struct iwm_vap *);
 352 static int      iwm_auth(struct ieee80211vap *, struct iwm_softc *);
 353 static struct ieee80211_node *
 354                 iwm_node_alloc(struct ieee80211vap *,
 355                                const uint8_t[IEEE80211_ADDR_LEN]);
 356 static uint8_t  iwm_rate_from_ucode_rate(uint32_t);
 357 static int      iwm_rate2ridx(struct iwm_softc *, uint8_t);
 358 static void     iwm_setrates(struct iwm_softc *, struct iwm_node *, int);
 359 static int      iwm_media_change(struct ifnet *);
 360 static int      iwm_newstate(struct ieee80211vap *, enum ieee80211_state, int);
 361 static void     iwm_endscan_cb(void *, int);
 362 static int      iwm_send_bt_init_conf(struct iwm_softc *);
 363 static boolean_t iwm_mvm_is_lar_supported(struct iwm_softc *);
 364 static boolean_t iwm_mvm_is_wifi_mcc_supported(struct iwm_softc *);
 365 static int      iwm_send_update_mcc_cmd(struct iwm_softc *, const char *);
 366 static void     iwm_mvm_tt_tx_backoff(struct iwm_softc *, uint32_t);
 367 static int      iwm_init_hw(struct iwm_softc *);
 368 static void     iwm_init(struct iwm_softc *);
 369 static void     iwm_start(struct iwm_softc *);
 370 static void     iwm_stop(struct iwm_softc *);
 371 static void     iwm_watchdog(void *);
 372 static void     iwm_parent(struct ieee80211com *);
 373 #ifdef IWM_DEBUG
 374 static const char *
 375                 iwm_desc_lookup(uint32_t);
 376 static void     iwm_nic_error(struct iwm_softc *);
 377 static void     iwm_nic_umac_error(struct iwm_softc *);
 378 #endif
 379 static void     iwm_handle_rxb(struct iwm_softc *, struct mbuf *);
 380 static void     iwm_notif_intr(struct iwm_softc *);
 381 static void     iwm_intr(void *);
 382 static int      iwm_attach(device_t);
 383 static int      iwm_is_valid_ether_addr(uint8_t *);
 384 static void     iwm_preinit(void *);
 385 static int      iwm_detach_local(struct iwm_softc *sc, int);
 386 static void     iwm_init_task(void *);
 387 static void     iwm_radiotap_attach(struct iwm_softc *);
 388 static struct ieee80211vap *
 389                 iwm_vap_create(struct ieee80211com *,
 390                                const char [IFNAMSIZ], int,
 391                                enum ieee80211_opmode, int,
 392                                const uint8_t [IEEE80211_ADDR_LEN],
 393                                const uint8_t [IEEE80211_ADDR_LEN]);
 394 static void     iwm_vap_delete(struct ieee80211vap *);
 395 static void     iwm_xmit_queue_drain(struct iwm_softc *);
 396 static void     iwm_scan_start(struct ieee80211com *);
 397 static void     iwm_scan_end(struct ieee80211com *);
 398 static void     iwm_update_mcast(struct ieee80211com *);
 399 static void     iwm_set_channel(struct ieee80211com *);
 400 static void     iwm_scan_curchan(struct ieee80211_scan_state *, unsigned long);
 401 static void     iwm_scan_mindwell(struct ieee80211_scan_state *);
 402 static int      iwm_detach(device_t);
 403
 404 static int      iwm_lar_disable = 0;
 405 TUNABLE_INT("hw.iwm.lar.disable", &iwm_lar_disable);
 406
 407 /*
 408  * Firmware parser.
 409  */
 410
 411 static int
 412 iwm_store_cscheme(struct iwm_softc *sc, const uint8_t *data, size_t dlen)
 413 {
 414         const struct iwm_fw_cscheme_list *l = (const void *)data;
 415
 416         if (dlen < sizeof(*l) ||
 417             dlen < sizeof(l->size) + l->size * sizeof(*l->cs))
 418                 return EINVAL;
 419
 420         /* we don't actually store anything for now, always use s/w crypto */
 421
 422         return 0;
 423 }
 424
 425 static int
 426 iwm_firmware_store_section(struct iwm_softc *sc,
 427     enum iwm_ucode_type type, const uint8_t *data, size_t dlen)
 428 {
 429         struct iwm_fw_img *fws;
 430         struct iwm_fw_desc *fwone;
 431
 432         if (type >= IWM_UCODE_TYPE_MAX)
 433                 return EINVAL;
 434         if (dlen < sizeof(uint32_t))
 435                 return EINVAL;
 436
 437         fws = &sc->sc_fw.img[type];
 438         if (fws->fw_count >= IWM_UCODE_SECTION_MAX)
 439                 return EINVAL;
 440
 441         fwone = &fws->sec[fws->fw_count];
 442
 443         /* first 32bit are device load offset */
 444         memcpy(&fwone->offset, data, sizeof(uint32_t));
 445
 446         /* rest is data */
 447         fwone->data = data + sizeof(uint32_t);
 448         fwone->len = dlen - sizeof(uint32_t);
 449
 450         fws->fw_count++;
 451
 452         return 0;
 453 }
 454
 455 #define IWM_DEFAULT_SCAN_CHANNELS 40
 456
 457 /* iwlwifi: iwl-drv.c */
 458 struct iwm_tlv_calib_data {
 459         uint32_t ucode_type;
 460         struct iwm_tlv_calib_ctrl calib;
 461 } __packed;
 462
 463 static int
 464 iwm_set_default_calib(struct iwm_softc *sc, const void *data)
 465 {
 466         const struct iwm_tlv_calib_data *def_calib = data;
 467         uint32_t ucode_type = le32toh(def_calib->ucode_type);
 468
 469         if (ucode_type >= IWM_UCODE_TYPE_MAX) {
 470                 device_printf(sc->sc_dev,
 471                     "Wrong ucode_type %u for default "
 472                     "calibration.\n", ucode_type);
 473                 return EINVAL;
 474         }
 475
 476         sc->sc_default_calib[ucode_type].flow_trigger =
 477             def_calib->calib.flow_trigger;
 478         sc->sc_default_calib[ucode_type].event_trigger =
 479             def_calib->calib.event_trigger;
 480
 481         return 0;
 482 }
 483
 484 static int
 485 iwm_set_ucode_api_flags(struct iwm_softc *sc, const uint8_t *data,
 486                         struct iwm_ucode_capabilities *capa)
 487 {
 488         const struct iwm_ucode_api *ucode_api = (const void *)data;
 489         uint32_t api_index = le32toh(ucode_api->api_index);
 490         uint32_t api_flags = le32toh(ucode_api->api_flags);
 491         int i;
 492
 493         if (api_index >= howmany(IWM_NUM_UCODE_TLV_API, 32)) {
 494                 device_printf(sc->sc_dev,
 495                     "api flags index %d larger than supported by driver\n",
 496                     api_index);
 497                 /* don't return an error so we can load FW that has more bits */
 498                 return 0;
 499         }
 500
 501         for (i = 0; i < 32; i++) {
 502                 if (api_flags & (1U << i))
 503                         setbit(capa->enabled_api, i + 32 * api_index);
 504         }
 505
 506         return 0;
 507 }
 508
 509 static int
 510 iwm_set_ucode_capabilities(struct iwm_softc *sc, const uint8_t *data,
 511                            struct iwm_ucode_capabilities *capa)
 512 {
 513         const struct iwm_ucode_capa *ucode_capa = (const void *)data;
 514         uint32_t api_index = le32toh(ucode_capa->api_index);
 515         uint32_t api_flags = le32toh(ucode_capa->api_capa);
 516         int i;
 517
 518         if (api_index >= howmany(IWM_NUM_UCODE_TLV_CAPA, 32)) {
 519                 device_printf(sc->sc_dev,
 520                     "capa flags index %d larger than supported by driver\n",
 521                     api_index);
 522                 /* don't return an error so we can load FW that has more bits */
 523                 return 0;
 524         }
 525
 526         for (i = 0; i < 32; i++) {
 527                 if (api_flags & (1U << i))
 528                         setbit(capa->enabled_capa, i + 32 * api_index);
 529         }
 530
 531         return 0;
 532 }
 533
 534 static void
 535 iwm_fw_info_free(struct iwm_fw_info *fw)
 536 {
 537         firmware_put(fw->fw_fp, FIRMWARE_UNLOAD);
 538         fw->fw_fp = NULL;
 539         memset(fw->img, 0, sizeof(fw->img));
 540 }
 541
 542 static int
 543 iwm_read_firmware(struct iwm_softc *sc)
 544 {
 545         struct iwm_fw_info *fw = &sc->sc_fw;
 546         const struct iwm_tlv_ucode_header *uhdr;
 547         const struct iwm_ucode_tlv *tlv;
 548         struct iwm_ucode_capabilities *capa = &sc->sc_fw.ucode_capa;
 549         enum iwm_ucode_tlv_type tlv_type;
 550         const struct firmware *fwp;
 551         const uint8_t *data;
 552         uint32_t tlv_len;
 553         uint32_t usniffer_img;
 554         const uint8_t *tlv_data;
 555         uint32_t paging_mem_size;
 556         int num_of_cpus;
 557         int error = 0;
 558         size_t len;
 559
 560         /*
 561          * Load firmware into driver memory.
 562          * fw_fp will be set.
 563          */
 564         fwp = firmware_get(sc->cfg->fw_name);
 565         if (fwp == NULL) {
 566                 device_printf(sc->sc_dev,
 567                     "could not read firmware %s (error %d)\n",
 568                     sc->cfg->fw_name, error);
 569                 goto out;
 570         }
 571         fw->fw_fp = fwp;
 572
 573         /* (Re-)Initialize default values. */
 574         capa->flags = 0;
 575         capa->max_probe_length = IWM_DEFAULT_MAX_PROBE_LENGTH;
 576         capa->n_scan_channels = IWM_DEFAULT_SCAN_CHANNELS;
 577         memset(capa->enabled_capa, 0, sizeof(capa->enabled_capa));
 578         memset(capa->enabled_api, 0, sizeof(capa->enabled_api));
 579         memset(sc->sc_fw_mcc, 0, sizeof(sc->sc_fw_mcc));
 580
 581         /*
 582          * Parse firmware contents
 583          */
 584
 585         uhdr = (const void *)fw->fw_fp->data;
 586         if (*(const uint32_t *)fw->fw_fp->data != 0
 587             || le32toh(uhdr->magic) != IWM_TLV_UCODE_MAGIC) {
 588                 device_printf(sc->sc_dev, "invalid firmware %s\n",
 589                     sc->cfg->fw_name);
 590                 error = EINVAL;
 591                 goto out;
 592         }
 593
 594         snprintf(sc->sc_fwver, sizeof(sc->sc_fwver), "%u.%u (API ver %u)",
 595             IWM_UCODE_MAJOR(le32toh(uhdr->ver)),
 596             IWM_UCODE_MINOR(le32toh(uhdr->ver)),
 597             IWM_UCODE_API(le32toh(uhdr->ver)));
 598         data = uhdr->data;
 599         len = fw->fw_fp->datasize - sizeof(*uhdr);
 600
 601         while (len >= sizeof(*tlv)) {
 602                 len -= sizeof(*tlv);
 603                 tlv = (const void *)data;
 604
 605                 tlv_len = le32toh(tlv->length);
 606                 tlv_type = le32toh(tlv->type);
 607                 tlv_data = tlv->data;
 608
 609                 if (len < tlv_len) {
 610                         device_printf(sc->sc_dev,
 611                             "firmware too short: %zu bytes\n",
 612                             len);
 613                         error = EINVAL;
 614                         goto parse_out;
 615                 }
 616                 len -= roundup2(tlv_len, 4);
 617                 data += sizeof(*tlv) + roundup2(tlv_len, 4);
 618
 619                 switch ((int)tlv_type) {
 620                 case IWM_UCODE_TLV_PROBE_MAX_LEN:
 621                         if (tlv_len != sizeof(uint32_t)) {
 622                                 device_printf(sc->sc_dev,
 623                                     "%s: PROBE_MAX_LEN (%u) != sizeof(uint32_t)\n",
 624                                     __func__, tlv_len);
 625                                 error = EINVAL;
 626                                 goto parse_out;
 627                         }
 628                         capa->max_probe_length =
 629                             le32_to_cpup((const uint32_t *)tlv_data);
 630                         /* limit it to something sensible */
 631                         if (capa->max_probe_length >
 632                             IWM_SCAN_OFFLOAD_PROBE_REQ_SIZE) {
 633                                 IWM_DPRINTF(sc, IWM_DEBUG_FIRMWARE_TLV,
 634                                     "%s: IWM_UCODE_TLV_PROBE_MAX_LEN "
 635                                     "ridiculous\n", __func__);
 636                                 error = EINVAL;
 637                                 goto parse_out;
 638                         }
 639                         break;
 640                 case IWM_UCODE_TLV_PAN:
 641                         if (tlv_len) {
 642                                 device_printf(sc->sc_dev,
 643                                     "%s: IWM_UCODE_TLV_PAN: tlv_len (%u) > 0\n",
 644                                     __func__, tlv_len);
 645                                 error = EINVAL;
 646                                 goto parse_out;
 647                         }
 648                         capa->flags |= IWM_UCODE_TLV_FLAGS_PAN;
 649                         break;
 650                 case IWM_UCODE_TLV_FLAGS:
 651                         if (tlv_len < sizeof(uint32_t)) {
 652                                 device_printf(sc->sc_dev,
 653                                     "%s: IWM_UCODE_TLV_FLAGS: tlv_len (%u) < sizeof(uint32_t)\n",
 654                                     __func__, tlv_len);
 655                                 error = EINVAL;
 656                                 goto parse_out;
 657                         }
 658                         if (tlv_len % sizeof(uint32_t)) {
 659                                 device_printf(sc->sc_dev,
 660                                     "%s: IWM_UCODE_TLV_FLAGS: tlv_len (%u) %% sizeof(uint32_t)\n",
 661                                     __func__, tlv_len);
 662                                 error = EINVAL;
 663                                 goto parse_out;
 664                         }
 665                         /*
 666                          * Apparently there can be many flags, but Linux driver
 667                          * parses only the first one, and so do we.
 668                          *
 669                          * XXX: why does this override IWM_UCODE_TLV_PAN?
 670                          * Intentional or a bug?  Observations from
 671                          * current firmware file:
 672                          *  1) TLV_PAN is parsed first
 673                          *  2) TLV_FLAGS contains TLV_FLAGS_PAN
 674                          * ==> this resets TLV_PAN to itself... hnnnk
 675                          */
 676                         capa->flags = le32_to_cpup((const uint32_t *)tlv_data);
 677                         break;
 678                 case IWM_UCODE_TLV_CSCHEME:
 679                         if ((error = iwm_store_cscheme(sc,
 680                             tlv_data, tlv_len)) != 0) {
 681                                 device_printf(sc->sc_dev,
 682                                     "%s: iwm_store_cscheme(): returned %d\n",
 683                                     __func__, error);
 684                                 goto parse_out;
 685                         }
 686                         break;
 687                 case IWM_UCODE_TLV_NUM_OF_CPU:
 688                         if (tlv_len != sizeof(uint32_t)) {
 689                                 device_printf(sc->sc_dev,
 690                                     "%s: IWM_UCODE_TLV_NUM_OF_CPU: tlv_len (%u) != sizeof(uint32_t)\n",
 691                                     __func__, tlv_len);
 692                                 error = EINVAL;
 693                                 goto parse_out;
 694                         }
 695                         num_of_cpus = le32_to_cpup((const uint32_t *)tlv_data);
 696                         if (num_of_cpus == 2) {
 697                                 fw->img[IWM_UCODE_REGULAR].is_dual_cpus =
 698                                         TRUE;
 699                                 fw->img[IWM_UCODE_INIT].is_dual_cpus =
 700                                         TRUE;
 701                                 fw->img[IWM_UCODE_WOWLAN].is_dual_cpus =
 702                                         TRUE;
 703                         } else if ((num_of_cpus > 2) || (num_of_cpus < 1)) {
 704                                 device_printf(sc->sc_dev,
 705                                     "%s: Driver supports only 1 or 2 CPUs\n",
 706                                     __func__);
 707                                 error = EINVAL;
 708                                 goto parse_out;
 709                         }
 710                         break;
 711                 case IWM_UCODE_TLV_SEC_RT:
 712                         if ((error = iwm_firmware_store_section(sc,
 713                             IWM_UCODE_REGULAR, tlv_data, tlv_len)) != 0) {
 714                                 device_printf(sc->sc_dev,
 715                                     "%s: IWM_UCODE_REGULAR: iwm_firmware_store_section() failed; %d\n",
 716                                     __func__, error);
 717                                 goto parse_out;
 718                         }
 719                         break;
 720                 case IWM_UCODE_TLV_SEC_INIT:
 721                         if ((error = iwm_firmware_store_section(sc,
 722                             IWM_UCODE_INIT, tlv_data, tlv_len)) != 0) {
 723                                 device_printf(sc->sc_dev,
 724                                     "%s: IWM_UCODE_INIT: iwm_firmware_store_section() failed; %d\n",
 725                                     __func__, error);
 726                                 goto parse_out;
 727                         }
 728                         break;
 729                 case IWM_UCODE_TLV_SEC_WOWLAN:
 730                         if ((error = iwm_firmware_store_section(sc,
 731                             IWM_UCODE_WOWLAN, tlv_data, tlv_len)) != 0) {
 732                                 device_printf(sc->sc_dev,
 733                                     "%s: IWM_UCODE_WOWLAN: iwm_firmware_store_section() failed; %d\n",
 734                                     __func__, error);
 735                                 goto parse_out;
 736                         }
 737                         break;
 738                 case IWM_UCODE_TLV_DEF_CALIB:
 739                         if (tlv_len != sizeof(struct iwm_tlv_calib_data)) {
 740                                 device_printf(sc->sc_dev,
 741                                     "%s: IWM_UCODE_TLV_DEV_CALIB: tlv_len (%u) < sizeof(iwm_tlv_calib_data) (%zu)\n",
 742                                     __func__, tlv_len,
 743                                     sizeof(struct iwm_tlv_calib_data));
 744                                 error = EINVAL;
 745                                 goto parse_out;
 746                         }
 747                         if ((error = iwm_set_default_calib(sc, tlv_data)) != 0) {
 748                                 device_printf(sc->sc_dev,
 749                                     "%s: iwm_set_default_calib() failed: %d\n",
 750                                     __func__, error);
 751                                 goto parse_out;
 752                         }
 753                         break;
 754                 case IWM_UCODE_TLV_PHY_SKU:
 755                         if (tlv_len != sizeof(uint32_t)) {
 756                                 error = EINVAL;
 757                                 device_printf(sc->sc_dev,
 758                                     "%s: IWM_UCODE_TLV_PHY_SKU: tlv_len (%u) < sizeof(uint32_t)\n",
 759                                     __func__, tlv_len);
 760                                 goto parse_out;
 761                         }
 762                         sc->sc_fw.phy_config =
 763                             le32_to_cpup((const uint32_t *)tlv_data);
 764                         sc->sc_fw.valid_tx_ant = (sc->sc_fw.phy_config &
 765                                                   IWM_FW_PHY_CFG_TX_CHAIN) >>
 766                                                   IWM_FW_PHY_CFG_TX_CHAIN_POS;
 767                         sc->sc_fw.valid_rx_ant = (sc->sc_fw.phy_config &
 768                                                   IWM_FW_PHY_CFG_RX_CHAIN) >>
 769                                                   IWM_FW_PHY_CFG_RX_CHAIN_POS;
 770                         break;
 771
 772                 case IWM_UCODE_TLV_API_CHANGES_SET: {
 773                         if (tlv_len != sizeof(struct iwm_ucode_api)) {
 774                                 error = EINVAL;
 775                                 goto parse_out;
 776                         }
 777                         if (iwm_set_ucode_api_flags(sc, tlv_data, capa)) {
 778                                 error = EINVAL;
 779                                 goto parse_out;
 780                         }
 781                         break;
 782                 }
 783
 784                 case IWM_UCODE_TLV_ENABLED_CAPABILITIES: {
 785                         if (tlv_len != sizeof(struct iwm_ucode_capa)) {
 786                                 error = EINVAL;
 787                                 goto parse_out;
 788                         }
 789                         if (iwm_set_ucode_capabilities(sc, tlv_data, capa)) {
 790                                 error = EINVAL;
 791                                 goto parse_out;
 792                         }
 793                         break;
 794                 }
 795
 796                 case 48: /* undocumented TLV */
 797                 case IWM_UCODE_TLV_SDIO_ADMA_ADDR:
 798                 case IWM_UCODE_TLV_FW_GSCAN_CAPA:
 799                         /* ignore, not used by current driver */
 800                         break;
 801
 802                 case IWM_UCODE_TLV_SEC_RT_USNIFFER:
 803                         if ((error = iwm_firmware_store_section(sc,
 804                             IWM_UCODE_REGULAR_USNIFFER, tlv_data,
 805                             tlv_len)) != 0)
 806                                 goto parse_out;
 807                         break;
 808
 809                 case IWM_UCODE_TLV_PAGING:
 810                         if (tlv_len != sizeof(uint32_t)) {
 811                                 error = EINVAL;
 812                                 goto parse_out;
 813                         }
 814                         paging_mem_size = le32_to_cpup((const uint32_t *)tlv_data);
 815
 816                         IWM_DPRINTF(sc, IWM_DEBUG_FIRMWARE_TLV,
 817                             "%s: Paging: paging enabled (size = %u bytes)\n",
 818                             __func__, paging_mem_size);
 819                         if (paging_mem_size > IWM_MAX_PAGING_IMAGE_SIZE) {
 820                                 device_printf(sc->sc_dev,
 821                                         "%s: Paging: driver supports up to %u bytes for paging image\n",
 822                                         __func__, IWM_MAX_PAGING_IMAGE_SIZE);
 823                                 error = EINVAL;
 824                                 goto out;
 825                         }
 826                         if (paging_mem_size & (IWM_FW_PAGING_SIZE - 1)) {
 827                                 device_printf(sc->sc_dev,
 828                                     "%s: Paging: image isn't multiple %u\n",
 829                                     __func__, IWM_FW_PAGING_SIZE);
 830                                 error = EINVAL;
 831                                 goto out;
 832                         }
 833
 834                         sc->sc_fw.img[IWM_UCODE_REGULAR].paging_mem_size =
 835                             paging_mem_size;
 836                         usniffer_img = IWM_UCODE_REGULAR_USNIFFER;
 837                         sc->sc_fw.img[usniffer_img].paging_mem_size =
 838                             paging_mem_size;
 839                         break;
 840
 841                 case IWM_UCODE_TLV_N_SCAN_CHANNELS:
 842                         if (tlv_len != sizeof(uint32_t)) {
 843                                 error = EINVAL;
 844                                 goto parse_out;
 845                         }
 846                         capa->n_scan_channels =
 847                             le32_to_cpup((const uint32_t *)tlv_data);
 848                         break;
 849
 850                 case IWM_UCODE_TLV_FW_VERSION:
 851                         if (tlv_len != sizeof(uint32_t) * 3) {
 852                                 error = EINVAL;
 853                                 goto parse_out;
 854                         }
 855                         snprintf(sc->sc_fwver, sizeof(sc->sc_fwver),
 856                             "%d.%d.%d",
 857                             le32toh(((const uint32_t *)tlv_data)[0]),
 858                             le32toh(((const uint32_t *)tlv_data)[1]),
 859                             le32toh(((const uint32_t *)tlv_data)[2]));
 860                         break;
 861
 862                 case IWM_UCODE_TLV_FW_MEM_SEG:
 863                         break;
 864
 865                 default:
 866                         device_printf(sc->sc_dev,
 867                             "%s: unknown firmware section %d, abort\n",
 868                             __func__, tlv_type);
 869                         error = EINVAL;
 870                         goto parse_out;
 871                 }
 872         }
 873
 874         KASSERT(error == 0, ("unhandled error"));
 875
 876  parse_out:
 877         if (error) {
 878                 device_printf(sc->sc_dev, "firmware parse error %d, "
 879                     "section type %d\n", error, tlv_type);
 880         }
 881
 882  out:
 883         if (error) {
 884                 if (fw->fw_fp != NULL)
 885                         iwm_fw_info_free(fw);
 886         }
 887
 888         return error;
 889 }
 890
 891 /*
 892  * DMA resource routines
 893  */
 894
 895 /* fwmem is used to load firmware onto the card */
 896 static int
 897 iwm_alloc_fwmem(struct iwm_softc *sc)
 898 {
 899         /* Must be aligned on a 16-byte boundary. */
 900         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->fw_dma,
 901             IWM_FH_MEM_TB_MAX_LENGTH, 16);
 902 }
 903
 904 /* tx scheduler rings.  not used? */
 905 static int
 906 iwm_alloc_sched(struct iwm_softc *sc)
 907 {
 908         /* TX scheduler rings must be aligned on a 1KB boundary. */
 909         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->sched_dma,
 910             nitems(sc->txq) * sizeof(struct iwm_agn_scd_bc_tbl), 1024);
 911 }
 912
 913 /* keep-warm page is used internally by the card.  see iwl-fh.h for more info */
 914 static int
 915 iwm_alloc_kw(struct iwm_softc *sc)
 916 {
 917         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->kw_dma, 4096, 4096);
 918 }
 919
 920 /* interrupt cause table */
 921 static int
 922 iwm_alloc_ict(struct iwm_softc *sc)
 923 {
 924         return iwm_dma_contig_alloc(sc->sc_dmat, &sc->ict_dma,
 925             IWM_ICT_SIZE, 1<<IWM_ICT_PADDR_SHIFT);
 926 }
 927
 928 static int
 929 iwm_alloc_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring)
 930 {
 931         bus_size_t size;
 932         int i, error;
 933
 934         ring->cur = 0;
 935
 936         /* Allocate RX descriptors (256-byte aligned). */
 937         size = IWM_RX_RING_COUNT * sizeof(uint32_t);
 938         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->desc_dma, size, 256);
 939         if (error != 0) {
 940                 device_printf(sc->sc_dev,
 941                     "could not allocate RX ring DMA memory\n");
 942                 goto fail;
 943         }
 944         ring->desc = ring->desc_dma.vaddr;
 945
 946         /* Allocate RX status area (16-byte aligned). */
 947         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->stat_dma,
 948             sizeof(*ring->stat), 16);
 949         if (error != 0) {
 950                 device_printf(sc->sc_dev,
 951                     "could not allocate RX status DMA memory\n");
 952                 goto fail;
 953         }
 954         ring->stat = ring->stat_dma.vaddr;
 955
 956         /* Create RX buffer DMA tag. */
 957         error = bus_dma_tag_create(sc->sc_dmat, 1, 0,
 958             BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL,
 959             IWM_RBUF_SIZE, 1, IWM_RBUF_SIZE, 0, NULL, NULL, &ring->data_dmat);
 960         if (error != 0) {
 961                 device_printf(sc->sc_dev,
 962                     "%s: could not create RX buf DMA tag, error %d\n",
 963                     __func__, error);
 964                 goto fail;
 965         }
 966
 967         /* Allocate spare bus_dmamap_t for iwm_rx_addbuf() */
 968         error = bus_dmamap_create(ring->data_dmat, 0, &ring->spare_map);
 969         if (error != 0) {
 970                 device_printf(sc->sc_dev,
 971                     "%s: could not create RX buf DMA map, error %d\n",
 972                     __func__, error);
 973                 goto fail;
 974         }
 975         /*
 976          * Allocate and map RX buffers.
 977          */
 978         for (i = 0; i < IWM_RX_RING_COUNT; i++) {
 979                 struct iwm_rx_data *data = &ring->data[i];
 980                 error = bus_dmamap_create(ring->data_dmat, 0, &data->map);
 981                 if (error != 0) {
 982                         device_printf(sc->sc_dev,
 983                             "%s: could not create RX buf DMA map, error %d\n",
 984                             __func__, error);
 985                         goto fail;
 986                 }
 987                 data->m = NULL;
 988
 989                 if ((error = iwm_rx_addbuf(sc, IWM_RBUF_SIZE, i)) != 0) {
 990                         goto fail;
 991                 }
 992         }
 993         return 0;
 994
 995 fail:   iwm_free_rx_ring(sc, ring);
 996         return error;
 997 }
 998
 999 static void
1000 iwm_reset_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring)
1001 {
1002         /* Reset the ring state */
1003         ring->cur = 0;
1004
1005         /*
1006          * The hw rx ring index in shared memory must also be cleared,
1007          * otherwise the discrepancy can cause reprocessing chaos.
1008          */
1009         if (sc->rxq.stat)
1010                 memset(sc->rxq.stat, 0, sizeof(*sc->rxq.stat));
1011 }
1012
1013 static void
1014 iwm_free_rx_ring(struct iwm_softc *sc, struct iwm_rx_ring *ring)
1015 {
1016         int i;
1017
1018         iwm_dma_contig_free(&ring->desc_dma);
1019         iwm_dma_contig_free(&ring->stat_dma);
1020
1021         for (i = 0; i < IWM_RX_RING_COUNT; i++) {
1022                 struct iwm_rx_data *data = &ring->data[i];
1023
1024                 if (data->m != NULL) {
1025                         bus_dmamap_sync(ring->data_dmat, data->map,
1026                             BUS_DMASYNC_POSTREAD);
1027                         bus_dmamap_unload(ring->data_dmat, data->map);
1028                         m_freem(data->m);
1029                         data->m = NULL;
1030                 }
1031                 if (data->map != NULL) {
1032                         bus_dmamap_destroy(ring->data_dmat, data->map);
1033                         data->map = NULL;
1034                 }
1035         }
1036         if (ring->spare_map != NULL) {
1037                 bus_dmamap_destroy(ring->data_dmat, ring->spare_map);
1038                 ring->spare_map = NULL;
1039         }
1040         if (ring->data_dmat != NULL) {
1041                 bus_dma_tag_destroy(ring->data_dmat);
1042                 ring->data_dmat = NULL;
1043         }
1044 }
1045
1046 static int
1047 iwm_alloc_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring, int qid)
1048 {
1049         bus_addr_t paddr;
1050         bus_size_t size;
1051         size_t maxsize;
1052         int nsegments;
1053         int i, error;
1054
1055         ring->qid = qid;
1056         ring->queued = 0;
1057         ring->cur = 0;
1058
1059         /* Allocate TX descriptors (256-byte aligned). */
1060         size = IWM_TX_RING_COUNT * sizeof (struct iwm_tfd);
1061         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->desc_dma, size, 256);
1062         if (error != 0) {
1063                 device_printf(sc->sc_dev,
1064                     "could not allocate TX ring DMA memory\n");
1065                 goto fail;
1066         }
1067         ring->desc = ring->desc_dma.vaddr;
1068
1069         /*
1070          * We only use rings 0 through 9 (4 EDCA + cmd) so there is no need
1071          * to allocate commands space for other rings.
1072          */
1073         if (qid > IWM_MVM_CMD_QUEUE)
1074                 return 0;
1075
1076         size = IWM_TX_RING_COUNT * sizeof(struct iwm_device_cmd);
1077         error = iwm_dma_contig_alloc(sc->sc_dmat, &ring->cmd_dma, size, 4);
1078         if (error != 0) {
1079                 device_printf(sc->sc_dev,
1080                     "could not allocate TX cmd DMA memory\n");
1081                 goto fail;
1082         }
1083         ring->cmd = ring->cmd_dma.vaddr;
1084
1085         /* FW commands may require more mapped space than packets. */
1086         if (qid == IWM_MVM_CMD_QUEUE) {
1087                 maxsize = IWM_RBUF_SIZE;
1088                 nsegments = 1;
1089         } else {
1090                 maxsize = MCLBYTES;
1091                 nsegments = IWM_MAX_SCATTER - 2;
1092         }
1093
1094         error = bus_dma_tag_create(sc->sc_dmat, 1, 0,
1095             BUS_SPACE_MAXADDR_32BIT, BUS_SPACE_MAXADDR, NULL, NULL, maxsize,
1096             nsegments, maxsize, 0, NULL, NULL, &ring->data_dmat);
1097         if (error != 0) {
1098                 device_printf(sc->sc_dev, "could not create TX buf DMA tag\n");
1099                 goto fail;
1100         }
1101
1102         paddr = ring->cmd_dma.paddr;
1103         for (i = 0; i < IWM_TX_RING_COUNT; i++) {
1104                 struct iwm_tx_data *data = &ring->data[i];
1105
1106                 data->cmd_paddr = paddr;
1107                 data->scratch_paddr = paddr + sizeof(struct iwm_cmd_header)
1108                     + offsetof(struct iwm_tx_cmd, scratch);
1109                 paddr += sizeof(struct iwm_device_cmd);
1110
1111                 error = bus_dmamap_create(ring->data_dmat, 0, &data->map);
1112                 if (error != 0) {
1113                         device_printf(sc->sc_dev,
1114                             "could not create TX buf DMA map\n");
1115                         goto fail;
1116                 }
1117         }
1118         KASSERT(paddr == ring->cmd_dma.paddr + size,
1119             ("invalid physical address"));
1120         return 0;
1121
1122 fail:   iwm_free_tx_ring(sc, ring);
1123         return error;
1124 }
1125
1126 static void
1127 iwm_reset_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring)
1128 {
1129         int i;
1130
1131         for (i = 0; i < IWM_TX_RING_COUNT; i++) {
1132                 struct iwm_tx_data *data = &ring->data[i];
1133
1134                 if (data->m != NULL) {
1135                         bus_dmamap_sync(ring->data_dmat, data->map,
1136                             BUS_DMASYNC_POSTWRITE);
1137                         bus_dmamap_unload(ring->data_dmat, data->map);
1138                         m_freem(data->m);
1139                         data->m = NULL;
1140                 }
1141         }
1142         /* Clear TX descriptors. */
1143         memset(ring->desc, 0, ring->desc_dma.size);
1144         bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map,
1145             BUS_DMASYNC_PREWRITE);
1146         sc->qfullmsk &= ~(1 << ring->qid);
1147         ring->queued = 0;
1148         ring->cur = 0;
1149
1150         if (ring->qid == IWM_MVM_CMD_QUEUE && sc->cmd_hold_nic_awake)
1151                 iwm_pcie_clear_cmd_in_flight(sc);
1152 }
1153
1154 static void
1155 iwm_free_tx_ring(struct iwm_softc *sc, struct iwm_tx_ring *ring)
1156 {
1157         int i;
1158
1159         iwm_dma_contig_free(&ring->desc_dma);
1160         iwm_dma_contig_free(&ring->cmd_dma);
1161
1162         for (i = 0; i < IWM_TX_RING_COUNT; i++) {
1163                 struct iwm_tx_data *data = &ring->data[i];
1164
1165                 if (data->m != NULL) {
1166                         bus_dmamap_sync(ring->data_dmat, data->map,
1167                             BUS_DMASYNC_POSTWRITE);
1168                         bus_dmamap_unload(ring->data_dmat, data->map);
1169                         m_freem(data->m);
1170                         data->m = NULL;
1171                 }
1172                 if (data->map != NULL) {
1173                         bus_dmamap_destroy(ring->data_dmat, data->map);
1174                         data->map = NULL;
1175                 }
1176         }
1177         if (ring->data_dmat != NULL) {
1178                 bus_dma_tag_destroy(ring->data_dmat);
1179                 ring->data_dmat = NULL;
1180         }
1181 }
1182
1183 /*
1184  * High-level hardware frobbing routines
1185  */
1186
1187 static void
1188 iwm_enable_interrupts(struct iwm_softc *sc)
1189 {
1190         sc->sc_intmask = IWM_CSR_INI_SET_MASK;
1191         IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask);
1192 }
1193
1194 static void
1195 iwm_restore_interrupts(struct iwm_softc *sc)
1196 {
1197         IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask);
1198 }
1199
1200 static void
1201 iwm_disable_interrupts(struct iwm_softc *sc)
1202 {
1203         /* disable interrupts */
1204         IWM_WRITE(sc, IWM_CSR_INT_MASK, 0);
1205
1206         /* acknowledge all interrupts */
1207         IWM_WRITE(sc, IWM_CSR_INT, ~0);
1208         IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, ~0);
1209 }
1210
1211 static void
1212 iwm_ict_reset(struct iwm_softc *sc)
1213 {
1214         iwm_disable_interrupts(sc);
1215
1216         /* Reset ICT table. */
1217         memset(sc->ict_dma.vaddr, 0, IWM_ICT_SIZE);
1218         sc->ict_cur = 0;
1219
1220         /* Set physical address of ICT table (4KB aligned). */
1221         IWM_WRITE(sc, IWM_CSR_DRAM_INT_TBL_REG,
1222             IWM_CSR_DRAM_INT_TBL_ENABLE
1223             | IWM_CSR_DRAM_INIT_TBL_WRITE_POINTER
1224             | IWM_CSR_DRAM_INIT_TBL_WRAP_CHECK
1225             | sc->ict_dma.paddr >> IWM_ICT_PADDR_SHIFT);
1226
1227         /* Switch to ICT interrupt mode in driver. */
1228         sc->sc_flags |= IWM_FLAG_USE_ICT;
1229
1230         /* Re-enable interrupts. */
1231         IWM_WRITE(sc, IWM_CSR_INT, ~0);
1232         iwm_enable_interrupts(sc);
1233 }
1234
1235 /* iwlwifi pcie/trans.c */
1236
1237 /*
1238  * Since this .. hard-resets things, it's time to actually
1239  * mark the first vap (if any) as having no mac context.
1240  * It's annoying, but since the driver is potentially being
1241  * stop/start'ed whilst active (thanks openbsd port!) we
1242  * have to correctly track this.
1243  */
1244 static void
1245 iwm_stop_device(struct iwm_softc *sc)
1246 {
1247         struct ieee80211com *ic = &sc->sc_ic;
1248         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
1249         int chnl, qid;
1250         uint32_t mask = 0;
1251
1252         /* tell the device to stop sending interrupts */
1253         iwm_disable_interrupts(sc);
1254
1255         /*
1256          * FreeBSD-local: mark the first vap as not-uploaded,
1257          * so the next transition through auth/assoc
1258          * will correctly populate the MAC context.
1259          */
1260         if (vap) {
1261                 struct iwm_vap *iv = IWM_VAP(vap);
1262                 iv->phy_ctxt = NULL;
1263                 iv->is_uploaded = 0;
1264         }
1265         sc->sc_firmware_state = 0;
1266         sc->sc_flags &= ~IWM_FLAG_TE_ACTIVE;
1267
1268         /* device going down, Stop using ICT table */
1269         sc->sc_flags &= ~IWM_FLAG_USE_ICT;
1270
1271         /* stop tx and rx.  tx and rx bits, as usual, are from if_iwn */
1272
1273         if (iwm_nic_lock(sc)) {
1274                 iwm_write_prph(sc, IWM_SCD_TXFACT, 0);
1275
1276                 /* Stop each Tx DMA channel */
1277                 for (chnl = 0; chnl < IWM_FH_TCSR_CHNL_NUM; chnl++) {
1278                         IWM_WRITE(sc,
1279                             IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl), 0);
1280                         mask |= IWM_FH_TSSR_TX_STATUS_REG_MSK_CHNL_IDLE(chnl);
1281                 }
1282
1283                 /* Wait for DMA channels to be idle */
1284                 if (!iwm_poll_bit(sc, IWM_FH_TSSR_TX_STATUS_REG, mask, mask,
1285                     5000)) {
1286                         device_printf(sc->sc_dev,
1287                             "Failing on timeout while stopping DMA channel: [0x%08x]\n",
1288                             IWM_READ(sc, IWM_FH_TSSR_TX_STATUS_REG));
1289                 }
1290                 iwm_nic_unlock(sc);
1291         }
1292         iwm_pcie_rx_stop(sc);
1293
1294         /* Stop RX ring. */
1295         iwm_reset_rx_ring(sc, &sc->rxq);
1296
1297         /* Reset all TX rings. */
1298         for (qid = 0; qid < nitems(sc->txq); qid++)
1299                 iwm_reset_tx_ring(sc, &sc->txq[qid]);
1300
1301         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
1302                 /* Power-down device's busmaster DMA clocks */
1303                 if (iwm_nic_lock(sc)) {
1304                         iwm_write_prph(sc, IWM_APMG_CLK_DIS_REG,
1305                             IWM_APMG_CLK_VAL_DMA_CLK_RQT);
1306                         iwm_nic_unlock(sc);
1307                 }
1308                 DELAY(5);
1309         }
1310
1311         /* Make sure (redundant) we've released our request to stay awake */
1312         IWM_CLRBITS(sc, IWM_CSR_GP_CNTRL,
1313             IWM_CSR_GP_CNTRL_REG_FLAG_MAC_ACCESS_REQ);
1314
1315         /* Stop the device, and put it in low power state */
1316         iwm_apm_stop(sc);
1317
1318         /* Upon stop, the APM issues an interrupt if HW RF kill is set.
1319          * Clean again the interrupt here
1320          */
1321         iwm_disable_interrupts(sc);
1322         /* stop and reset the on-board processor */
1323         IWM_WRITE(sc, IWM_CSR_RESET, IWM_CSR_RESET_REG_FLAG_SW_RESET);
1324
1325         /*
1326          * Even if we stop the HW, we still want the RF kill
1327          * interrupt
1328          */
1329         iwm_enable_rfkill_int(sc);
1330         iwm_check_rfkill(sc);
1331 }
1332
1333 /* iwlwifi: mvm/ops.c */
1334 static void
1335 iwm_mvm_nic_config(struct iwm_softc *sc)
1336 {
1337         uint8_t radio_cfg_type, radio_cfg_step, radio_cfg_dash;
1338         uint32_t reg_val = 0;
1339         uint32_t phy_config = iwm_mvm_get_phy_config(sc);
1340
1341         radio_cfg_type = (phy_config & IWM_FW_PHY_CFG_RADIO_TYPE) >>
1342             IWM_FW_PHY_CFG_RADIO_TYPE_POS;
1343         radio_cfg_step = (phy_config & IWM_FW_PHY_CFG_RADIO_STEP) >>
1344             IWM_FW_PHY_CFG_RADIO_STEP_POS;
1345         radio_cfg_dash = (phy_config & IWM_FW_PHY_CFG_RADIO_DASH) >>
1346             IWM_FW_PHY_CFG_RADIO_DASH_POS;
1347
1348         /* SKU control */
1349         reg_val |= IWM_CSR_HW_REV_STEP(sc->sc_hw_rev) <<
1350             IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_STEP;
1351         reg_val |= IWM_CSR_HW_REV_DASH(sc->sc_hw_rev) <<
1352             IWM_CSR_HW_IF_CONFIG_REG_POS_MAC_DASH;
1353
1354         /* radio configuration */
1355         reg_val |= radio_cfg_type << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_TYPE;
1356         reg_val |= radio_cfg_step << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_STEP;
1357         reg_val |= radio_cfg_dash << IWM_CSR_HW_IF_CONFIG_REG_POS_PHY_DASH;
1358
1359         IWM_WRITE(sc, IWM_CSR_HW_IF_CONFIG_REG, reg_val);
1360
1361         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
1362             "Radio type=0x%x-0x%x-0x%x\n", radio_cfg_type,
1363             radio_cfg_step, radio_cfg_dash);
1364
1365         /*
1366          * W/A : NIC is stuck in a reset state after Early PCIe power off
1367          * (PCIe power is lost before PERST# is asserted), causing ME FW
1368          * to lose ownership and not being able to obtain it back.
1369          */
1370         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
1371                 iwm_set_bits_mask_prph(sc, IWM_APMG_PS_CTRL_REG,
1372                     IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS,
1373                     ~IWM_APMG_PS_CTRL_EARLY_PWR_OFF_RESET_DIS);
1374         }
1375 }
1376
1377 static int
1378 iwm_nic_rx_init(struct iwm_softc *sc)
1379 {
1380         /*
1381          * Initialize RX ring.  This is from the iwn driver.
1382          */
1383         memset(sc->rxq.stat, 0, sizeof(*sc->rxq.stat));
1384
1385         /* Stop Rx DMA */
1386         iwm_pcie_rx_stop(sc);
1387
1388         if (!iwm_nic_lock(sc))
1389                 return EBUSY;
1390
1391         /* reset and flush pointers */
1392         IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_RBDCB_WPTR, 0);
1393         IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_FLUSH_RB_REQ, 0);
1394         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_RDPTR, 0);
1395         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_RBDCB_WPTR_REG, 0);
1396
1397         /* Set physical address of RX ring (256-byte aligned). */
1398         IWM_WRITE(sc,
1399             IWM_FH_RSCSR_CHNL0_RBDCB_BASE_REG, sc->rxq.desc_dma.paddr >> 8);
1400
1401         /* Set physical address of RX status (16-byte aligned). */
1402         IWM_WRITE(sc,
1403             IWM_FH_RSCSR_CHNL0_STTS_WPTR_REG, sc->rxq.stat_dma.paddr >> 4);
1404
1405         /* Enable Rx DMA
1406          * XXX 5000 HW isn't supported by the iwm(4) driver.
1407          * IWM_FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY is set because of HW bug in
1408          *      the credit mechanism in 5000 HW RX FIFO
1409          * Direct rx interrupts to hosts
1410          * Rx buffer size 4 or 8k or 12k
1411          * RB timeout 0x10
1412          * 256 RBDs
1413          */
1414         IWM_WRITE(sc, IWM_FH_MEM_RCSR_CHNL0_CONFIG_REG,
1415             IWM_FH_RCSR_RX_CONFIG_CHNL_EN_ENABLE_VAL            |
1416             IWM_FH_RCSR_CHNL0_RX_IGNORE_RXF_EMPTY               |  /* HW bug */
1417             IWM_FH_RCSR_CHNL0_RX_CONFIG_IRQ_DEST_INT_HOST_VAL   |
1418             IWM_FH_RCSR_RX_CONFIG_REG_VAL_RB_SIZE_4K            |
1419             (IWM_RX_RB_TIMEOUT << IWM_FH_RCSR_RX_CONFIG_REG_IRQ_RBTH_POS) |
1420             IWM_RX_QUEUE_SIZE_LOG << IWM_FH_RCSR_RX_CONFIG_RBDCB_SIZE_POS);
1421
1422         IWM_WRITE_1(sc, IWM_CSR_INT_COALESCING, IWM_HOST_INT_TIMEOUT_DEF);
1423
1424         /* W/A for interrupt coalescing bug in 7260 and 3160 */
1425         if (sc->cfg->host_interrupt_operation_mode)
1426                 IWM_SETBITS(sc, IWM_CSR_INT_COALESCING, IWM_HOST_INT_OPER_MODE);
1427
1428         /*
1429          * Thus sayeth el jefe (iwlwifi) via a comment:
1430          *
1431          * This value should initially be 0 (before preparing any
1432          * RBs), should be 8 after preparing the first 8 RBs (for example)
1433          */
1434         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_WPTR, 8);
1435
1436         iwm_nic_unlock(sc);
1437
1438         return 0;
1439 }
1440
1441 static int
1442 iwm_nic_tx_init(struct iwm_softc *sc)
1443 {
1444         int qid;
1445
1446         if (!iwm_nic_lock(sc))
1447                 return EBUSY;
1448
1449         /* Deactivate TX scheduler. */
1450         iwm_write_prph(sc, IWM_SCD_TXFACT, 0);
1451
1452         /* Set physical address of "keep warm" page (16-byte aligned). */
1453         IWM_WRITE(sc, IWM_FH_KW_MEM_ADDR_REG, sc->kw_dma.paddr >> 4);
1454
1455         /* Initialize TX rings. */
1456         for (qid = 0; qid < nitems(sc->txq); qid++) {
1457                 struct iwm_tx_ring *txq = &sc->txq[qid];
1458
1459                 /* Set physical address of TX ring (256-byte aligned). */
1460                 IWM_WRITE(sc, IWM_FH_MEM_CBBC_QUEUE(qid),
1461                     txq->desc_dma.paddr >> 8);
1462                 IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
1463                     "%s: loading ring %d descriptors (%p) at %lx\n",
1464                     __func__,
1465                     qid, txq->desc,
1466                     (unsigned long) (txq->desc_dma.paddr >> 8));
1467         }
1468
1469         iwm_write_prph(sc, IWM_SCD_GP_CTRL, IWM_SCD_GP_CTRL_AUTO_ACTIVE_MODE);
1470
1471         iwm_nic_unlock(sc);
1472
1473         return 0;
1474 }
1475
1476 static int
1477 iwm_nic_init(struct iwm_softc *sc)
1478 {
1479         int error;
1480
1481         iwm_apm_init(sc);
1482         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
1483                 iwm_set_pwr(sc);
1484
1485         iwm_mvm_nic_config(sc);
1486
1487         if ((error = iwm_nic_rx_init(sc)) != 0)
1488                 return error;
1489
1490         /*
1491          * Ditto for TX, from iwn
1492          */
1493         if ((error = iwm_nic_tx_init(sc)) != 0)
1494                 return error;
1495
1496         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
1497             "%s: shadow registers enabled\n", __func__);
1498         IWM_SETBITS(sc, IWM_CSR_MAC_SHADOW_REG_CTRL, 0x800fffff);
1499
1500         return 0;
1501 }
1502
1503 int
1504 iwm_enable_txq(struct iwm_softc *sc, int sta_id, int qid, int fifo)
1505 {
1506         if (!iwm_nic_lock(sc)) {
1507                 device_printf(sc->sc_dev,
1508                     "%s: cannot enable txq %d\n",
1509                     __func__,
1510                     qid);
1511                 return EBUSY;
1512         }
1513
1514         IWM_WRITE(sc, IWM_HBUS_TARG_WRPTR, qid << 8 | 0);
1515
1516         if (qid == IWM_MVM_CMD_QUEUE) {
1517                 /* unactivate before configuration */
1518                 iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid),
1519                     (0 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE)
1520                     | (1 << IWM_SCD_QUEUE_STTS_REG_POS_SCD_ACT_EN));
1521
1522                 iwm_nic_unlock(sc);
1523
1524                 iwm_clear_bits_prph(sc, IWM_SCD_AGGR_SEL, (1 << qid));
1525
1526                 if (!iwm_nic_lock(sc)) {
1527                         device_printf(sc->sc_dev,
1528                             "%s: cannot enable txq %d\n", __func__, qid);
1529                         return EBUSY;
1530                 }
1531                 iwm_write_prph(sc, IWM_SCD_QUEUE_RDPTR(qid), 0);
1532                 iwm_nic_unlock(sc);
1533
1534                 iwm_write_mem32(sc, sc->scd_base_addr + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid), 0);
1535                 /* Set scheduler window size and frame limit. */
1536                 iwm_write_mem32(sc,
1537                     sc->scd_base_addr + IWM_SCD_CONTEXT_QUEUE_OFFSET(qid) +
1538                     sizeof(uint32_t),
1539                     ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_POS) &
1540                     IWM_SCD_QUEUE_CTX_REG2_WIN_SIZE_MSK) |
1541                     ((IWM_FRAME_LIMIT << IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_POS) &
1542                     IWM_SCD_QUEUE_CTX_REG2_FRAME_LIMIT_MSK));
1543
1544                 if (!iwm_nic_lock(sc)) {
1545                         device_printf(sc->sc_dev,
1546                             "%s: cannot enable txq %d\n", __func__, qid);
1547                         return EBUSY;
1548                 }
1549                 iwm_write_prph(sc, IWM_SCD_QUEUE_STATUS_BITS(qid),
1550                     (1 << IWM_SCD_QUEUE_STTS_REG_POS_ACTIVE) |
1551                     (fifo << IWM_SCD_QUEUE_STTS_REG_POS_TXF) |
1552                     (1 << IWM_SCD_QUEUE_STTS_REG_POS_WSL) |
1553                     IWM_SCD_QUEUE_STTS_REG_MSK);
1554         } else {
1555                 struct iwm_scd_txq_cfg_cmd cmd;
1556                 int error;
1557
1558                 iwm_nic_unlock(sc);
1559
1560                 memset(&cmd, 0, sizeof(cmd));
1561                 cmd.scd_queue = qid;
1562                 cmd.enable = 1;
1563                 cmd.sta_id = sta_id;
1564                 cmd.tx_fifo = fifo;
1565                 cmd.aggregate = 0;
1566                 cmd.window = IWM_FRAME_LIMIT;
1567
1568                 error = iwm_mvm_send_cmd_pdu(sc, IWM_SCD_QUEUE_CFG, IWM_CMD_SYNC,
1569                     sizeof(cmd), &cmd);
1570                 if (error) {
1571                         device_printf(sc->sc_dev,
1572                             "cannot enable txq %d\n", qid);
1573                         return error;
1574                 }
1575
1576                 if (!iwm_nic_lock(sc))
1577                         return EBUSY;
1578         }
1579
1580         iwm_write_prph(sc, IWM_SCD_EN_CTRL,
1581             iwm_read_prph(sc, IWM_SCD_EN_CTRL) | qid);
1582
1583         iwm_nic_unlock(sc);
1584
1585         IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "%s: enabled txq %d FIFO %d\n",
1586             __func__, qid, fifo);
1587
1588         return 0;
1589 }
1590
1591 static int
1592 iwm_trans_pcie_fw_alive(struct iwm_softc *sc, uint32_t scd_base_addr)
1593 {
1594         int error, chnl;
1595
1596         int clear_dwords = (IWM_SCD_TRANS_TBL_MEM_UPPER_BOUND -
1597             IWM_SCD_CONTEXT_MEM_LOWER_BOUND) / sizeof(uint32_t);
1598
1599         if (!iwm_nic_lock(sc))
1600                 return EBUSY;
1601
1602         iwm_ict_reset(sc);
1603
1604         sc->scd_base_addr = iwm_read_prph(sc, IWM_SCD_SRAM_BASE_ADDR);
1605         if (scd_base_addr != 0 &&
1606             scd_base_addr != sc->scd_base_addr) {
1607                 device_printf(sc->sc_dev,
1608                     "%s: sched addr mismatch: alive: 0x%x prph: 0x%x\n",
1609                     __func__, sc->scd_base_addr, scd_base_addr);
1610         }
1611
1612         iwm_nic_unlock(sc);
1613
1614         /* reset context data, TX status and translation data */
1615         error = iwm_write_mem(sc,
1616             sc->scd_base_addr + IWM_SCD_CONTEXT_MEM_LOWER_BOUND,
1617             NULL, clear_dwords);
1618         if (error)
1619                 return EBUSY;
1620
1621         if (!iwm_nic_lock(sc))
1622                 return EBUSY;
1623
1624         /* Set physical address of TX scheduler rings (1KB aligned). */
1625         iwm_write_prph(sc, IWM_SCD_DRAM_BASE_ADDR, sc->sched_dma.paddr >> 10);
1626
1627         iwm_write_prph(sc, IWM_SCD_CHAINEXT_EN, 0);
1628
1629         iwm_nic_unlock(sc);
1630
1631         /* enable command channel */
1632         error = iwm_enable_txq(sc, 0 /* unused */, IWM_MVM_CMD_QUEUE, 7);
1633         if (error)
1634                 return error;
1635
1636         if (!iwm_nic_lock(sc))
1637                 return EBUSY;
1638
1639         iwm_write_prph(sc, IWM_SCD_TXFACT, 0xff);
1640
1641         /* Enable DMA channels. */
1642         for (chnl = 0; chnl < IWM_FH_TCSR_CHNL_NUM; chnl++) {
1643                 IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(chnl),
1644                     IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE |
1645                     IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_ENABLE);
1646         }
1647
1648         IWM_SETBITS(sc, IWM_FH_TX_CHICKEN_BITS_REG,
1649             IWM_FH_TX_CHICKEN_BITS_SCD_AUTO_RETRY_EN);
1650
1651         iwm_nic_unlock(sc);
1652
1653         /* Enable L1-Active */
1654         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
1655                 iwm_clear_bits_prph(sc, IWM_APMG_PCIDEV_STT_REG,
1656                     IWM_APMG_PCIDEV_STT_VAL_L1_ACT_DIS);
1657         }
1658
1659         return error;
1660 }
1661
1662 /*
1663  * NVM read access and content parsing.  We do not support
1664  * external NVM or writing NVM.
1665  * iwlwifi/mvm/nvm.c
1666  */
1667
1668 /* Default NVM size to read */
1669 #define IWM_NVM_DEFAULT_CHUNK_SIZE      (2*1024)
1670
1671 #define IWM_NVM_WRITE_OPCODE 1
1672 #define IWM_NVM_READ_OPCODE 0
1673
1674 /* load nvm chunk response */
1675 enum {
1676         IWM_READ_NVM_CHUNK_SUCCEED = 0,
1677         IWM_READ_NVM_CHUNK_NOT_VALID_ADDRESS = 1
1678 };
1679
1680 static int
1681 iwm_nvm_read_chunk(struct iwm_softc *sc, uint16_t section,
1682         uint16_t offset, uint16_t length, uint8_t *data, uint16_t *len)
1683 {
1684         struct iwm_nvm_access_cmd nvm_access_cmd = {
1685                 .offset = htole16(offset),
1686                 .length = htole16(length),
1687                 .type = htole16(section),
1688                 .op_code = IWM_NVM_READ_OPCODE,
1689         };
1690         struct iwm_nvm_access_resp *nvm_resp;
1691         struct iwm_rx_packet *pkt;
1692         struct iwm_host_cmd cmd = {
1693                 .id = IWM_NVM_ACCESS_CMD,
1694                 .flags = IWM_CMD_WANT_SKB | IWM_CMD_SEND_IN_RFKILL,
1695                 .data = { &nvm_access_cmd, },
1696         };
1697         int ret, bytes_read, offset_read;
1698         uint8_t *resp_data;
1699
1700         cmd.len[0] = sizeof(struct iwm_nvm_access_cmd);
1701
1702         ret = iwm_send_cmd(sc, &cmd);
1703         if (ret) {
1704                 device_printf(sc->sc_dev,
1705                     "Could not send NVM_ACCESS command (error=%d)\n", ret);
1706                 return ret;
1707         }
1708
1709         pkt = cmd.resp_pkt;
1710
1711         /* Extract NVM response */
1712         nvm_resp = (void *)pkt->data;
1713         ret = le16toh(nvm_resp->status);
1714         bytes_read = le16toh(nvm_resp->length);
1715         offset_read = le16toh(nvm_resp->offset);
1716         resp_data = nvm_resp->data;
1717         if (ret) {
1718                 if ((offset != 0) &&
1719                     (ret == IWM_READ_NVM_CHUNK_NOT_VALID_ADDRESS)) {
1720                         /*
1721                          * meaning of NOT_VALID_ADDRESS:
1722                          * driver try to read chunk from address that is
1723                          * multiple of 2K and got an error since addr is empty.
1724                          * meaning of (offset != 0): driver already
1725                          * read valid data from another chunk so this case
1726                          * is not an error.
1727                          */
1728                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1729                                     "NVM access command failed on offset 0x%x since that section size is multiple 2K\n",
1730                                     offset);
1731                         *len = 0;
1732                         ret = 0;
1733                 } else {
1734                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1735                                     "NVM access command failed with status %d\n", ret);
1736                         ret = EIO;
1737                 }
1738                 goto exit;
1739         }
1740
1741         if (offset_read != offset) {
1742                 device_printf(sc->sc_dev,
1743                     "NVM ACCESS response with invalid offset %d\n",
1744                     offset_read);
1745                 ret = EINVAL;
1746                 goto exit;
1747         }
1748
1749         if (bytes_read > length) {
1750                 device_printf(sc->sc_dev,
1751                     "NVM ACCESS response with too much data "
1752                     "(%d bytes requested, %d bytes received)\n",
1753                     length, bytes_read);
1754                 ret = EINVAL;
1755                 goto exit;
1756         }
1757
1758         /* Write data to NVM */
1759         memcpy(data + offset, resp_data, bytes_read);
1760         *len = bytes_read;
1761
1762  exit:
1763         iwm_free_resp(sc, &cmd);
1764         return ret;
1765 }
1766
1767 /*
1768  * Reads an NVM section completely.
1769  * NICs prior to 7000 family don't have a real NVM, but just read
1770  * section 0 which is the EEPROM. Because the EEPROM reading is unlimited
1771  * by uCode, we need to manually check in this case that we don't
1772  * overflow and try to read more than the EEPROM size.
1773  * For 7000 family NICs, we supply the maximal size we can read, and
1774  * the uCode fills the response with as much data as we can,
1775  * without overflowing, so no check is needed.
1776  */
1777 static int
1778 iwm_nvm_read_section(struct iwm_softc *sc,
1779         uint16_t section, uint8_t *data, uint16_t *len, uint32_t size_read)
1780 {
1781         uint16_t seglen, length, offset = 0;
1782         int ret;
1783
1784         /* Set nvm section read length */
1785         length = IWM_NVM_DEFAULT_CHUNK_SIZE;
1786
1787         seglen = length;
1788
1789         /* Read the NVM until exhausted (reading less than requested) */
1790         while (seglen == length) {
1791                 /* Check no memory assumptions fail and cause an overflow */
1792                 if ((size_read + offset + length) >
1793                     sc->cfg->eeprom_size) {
1794                         device_printf(sc->sc_dev,
1795                             "EEPROM size is too small for NVM\n");
1796                         return ENOBUFS;
1797                 }
1798
1799                 ret = iwm_nvm_read_chunk(sc, section, offset, length, data, &seglen);
1800                 if (ret) {
1801                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1802                                     "Cannot read NVM from section %d offset %d, length %d\n",
1803                                     section, offset, length);
1804                         return ret;
1805                 }
1806                 offset += seglen;
1807         }
1808
1809         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
1810                     "NVM section %d read completed\n", section);
1811         *len = offset;
1812         return 0;
1813 }
1814
1815 /*
1816  * BEGIN IWM_NVM_PARSE
1817  */
1818
1819 /* iwlwifi/iwl-nvm-parse.c */
1820
1821 /* NVM offsets (in words) definitions */
1822 enum iwm_nvm_offsets {
1823         /* NVM HW-Section offset (in words) definitions */
1824         IWM_HW_ADDR = 0x15,
1825
1826 /* NVM SW-Section offset (in words) definitions */
1827         IWM_NVM_SW_SECTION = 0x1C0,
1828         IWM_NVM_VERSION = 0,
1829         IWM_RADIO_CFG = 1,
1830         IWM_SKU = 2,
1831         IWM_N_HW_ADDRS = 3,
1832         IWM_NVM_CHANNELS = 0x1E0 - IWM_NVM_SW_SECTION,
1833
1834 /* NVM calibration section offset (in words) definitions */
1835         IWM_NVM_CALIB_SECTION = 0x2B8,
1836         IWM_XTAL_CALIB = 0x316 - IWM_NVM_CALIB_SECTION
1837 };
1838
1839 enum iwm_8000_nvm_offsets {
1840         /* NVM HW-Section offset (in words) definitions */
1841         IWM_HW_ADDR0_WFPM_8000 = 0x12,
1842         IWM_HW_ADDR1_WFPM_8000 = 0x16,
1843         IWM_HW_ADDR0_PCIE_8000 = 0x8A,
1844         IWM_HW_ADDR1_PCIE_8000 = 0x8E,
1845         IWM_MAC_ADDRESS_OVERRIDE_8000 = 1,
1846
1847         /* NVM SW-Section offset (in words) definitions */
1848         IWM_NVM_SW_SECTION_8000 = 0x1C0,
1849         IWM_NVM_VERSION_8000 = 0,
1850         IWM_RADIO_CFG_8000 = 0,
1851         IWM_SKU_8000 = 2,
1852         IWM_N_HW_ADDRS_8000 = 3,
1853
1854         /* NVM REGULATORY -Section offset (in words) definitions */
1855         IWM_NVM_CHANNELS_8000 = 0,
1856         IWM_NVM_LAR_OFFSET_8000_OLD = 0x4C7,
1857         IWM_NVM_LAR_OFFSET_8000 = 0x507,
1858         IWM_NVM_LAR_ENABLED_8000 = 0x7,
1859
1860         /* NVM calibration section offset (in words) definitions */
1861         IWM_NVM_CALIB_SECTION_8000 = 0x2B8,
1862         IWM_XTAL_CALIB_8000 = 0x316 - IWM_NVM_CALIB_SECTION_8000
1863 };
1864
1865 /* SKU Capabilities (actual values from NVM definition) */
1866 enum nvm_sku_bits {
1867         IWM_NVM_SKU_CAP_BAND_24GHZ      = (1 << 0),
1868         IWM_NVM_SKU_CAP_BAND_52GHZ      = (1 << 1),
1869         IWM_NVM_SKU_CAP_11N_ENABLE      = (1 << 2),
1870         IWM_NVM_SKU_CAP_11AC_ENABLE     = (1 << 3),
1871 };
1872
1873 /* radio config bits (actual values from NVM definition) */
1874 #define IWM_NVM_RF_CFG_DASH_MSK(x)   (x & 0x3)         /* bits 0-1   */
1875 #define IWM_NVM_RF_CFG_STEP_MSK(x)   ((x >> 2)  & 0x3) /* bits 2-3   */
1876 #define IWM_NVM_RF_CFG_TYPE_MSK(x)   ((x >> 4)  & 0x3) /* bits 4-5   */
1877 #define IWM_NVM_RF_CFG_PNUM_MSK(x)   ((x >> 6)  & 0x3) /* bits 6-7   */
1878 #define IWM_NVM_RF_CFG_TX_ANT_MSK(x) ((x >> 8)  & 0xF) /* bits 8-11  */
1879 #define IWM_NVM_RF_CFG_RX_ANT_MSK(x) ((x >> 12) & 0xF) /* bits 12-15 */
1880
1881 #define IWM_NVM_RF_CFG_FLAVOR_MSK_8000(x)       (x & 0xF)
1882 #define IWM_NVM_RF_CFG_DASH_MSK_8000(x)         ((x >> 4) & 0xF)
1883 #define IWM_NVM_RF_CFG_STEP_MSK_8000(x)         ((x >> 8) & 0xF)
1884 #define IWM_NVM_RF_CFG_TYPE_MSK_8000(x)         ((x >> 12) & 0xFFF)
1885 #define IWM_NVM_RF_CFG_TX_ANT_MSK_8000(x)       ((x >> 24) & 0xF)
1886 #define IWM_NVM_RF_CFG_RX_ANT_MSK_8000(x)       ((x >> 28) & 0xF)
1887
1888 /**
1889  * enum iwm_nvm_channel_flags - channel flags in NVM
1890  * @IWM_NVM_CHANNEL_VALID: channel is usable for this SKU/geo
1891  * @IWM_NVM_CHANNEL_IBSS: usable as an IBSS channel
1892  * @IWM_NVM_CHANNEL_ACTIVE: active scanning allowed
1893  * @IWM_NVM_CHANNEL_RADAR: radar detection required
1894  * XXX cannot find this (DFS) flag in iwm-nvm-parse.c
1895  * @IWM_NVM_CHANNEL_DFS: dynamic freq selection candidate
1896  * @IWM_NVM_CHANNEL_WIDE: 20 MHz channel okay (?)
1897  * @IWM_NVM_CHANNEL_40MHZ: 40 MHz channel okay (?)
1898  * @IWM_NVM_CHANNEL_80MHZ: 80 MHz channel okay (?)
1899  * @IWM_NVM_CHANNEL_160MHZ: 160 MHz channel okay (?)
1900  */
1901 enum iwm_nvm_channel_flags {
1902         IWM_NVM_CHANNEL_VALID = (1 << 0),
1903         IWM_NVM_CHANNEL_IBSS = (1 << 1),
1904         IWM_NVM_CHANNEL_ACTIVE = (1 << 3),
1905         IWM_NVM_CHANNEL_RADAR = (1 << 4),
1906         IWM_NVM_CHANNEL_DFS = (1 << 7),
1907         IWM_NVM_CHANNEL_WIDE = (1 << 8),
1908         IWM_NVM_CHANNEL_40MHZ = (1 << 9),
1909         IWM_NVM_CHANNEL_80MHZ = (1 << 10),
1910         IWM_NVM_CHANNEL_160MHZ = (1 << 11),
1911 };
1912
1913 /*
1914  * Translate EEPROM flags to net80211.
1915  */
1916 static uint32_t
1917 iwm_eeprom_channel_flags(uint16_t ch_flags)
1918 {
1919         uint32_t nflags;
1920
1921         nflags = 0;
1922         if ((ch_flags & IWM_NVM_CHANNEL_ACTIVE) == 0)
1923                 nflags |= IEEE80211_CHAN_PASSIVE;
1924         if ((ch_flags & IWM_NVM_CHANNEL_IBSS) == 0)
1925                 nflags |= IEEE80211_CHAN_NOADHOC;
1926         if (ch_flags & IWM_NVM_CHANNEL_RADAR) {
1927                 nflags |= IEEE80211_CHAN_DFS;
1928                 /* Just in case. */
1929                 nflags |= IEEE80211_CHAN_NOADHOC;
1930         }
1931
1932         return (nflags);
1933 }
1934
1935 static void
1936 iwm_add_channel_band(struct iwm_softc *sc, struct ieee80211_channel chans[],
1937     int maxchans, int *nchans, int ch_idx, size_t ch_num,
1938     const uint8_t bands[])
1939 {
1940         const uint16_t * const nvm_ch_flags = sc->nvm_data->nvm_ch_flags;
1941         uint32_t nflags;
1942         uint16_t ch_flags;
1943         uint8_t ieee;
1944         int error;
1945
1946         for (; ch_idx < ch_num; ch_idx++) {
1947                 ch_flags = le16_to_cpup(nvm_ch_flags + ch_idx);
1948                 if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
1949                         ieee = iwm_nvm_channels[ch_idx];
1950                 else
1951                         ieee = iwm_nvm_channels_8000[ch_idx];
1952
1953                 if (!(ch_flags & IWM_NVM_CHANNEL_VALID)) {
1954                         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM,
1955                             "Ch. %d Flags %x [%sGHz] - No traffic\n",
1956                             ieee, ch_flags,
1957                             (ch_idx >= IWM_NUM_2GHZ_CHANNELS) ?
1958                             "5.2" : "2.4");
1959                         continue;
1960                 }
1961
1962                 nflags = iwm_eeprom_channel_flags(ch_flags);
1963                 error = ieee80211_add_channel(chans, maxchans, nchans,
1964                     ieee, 0, 0, nflags, bands);
1965                 if (error != 0)
1966                         break;
1967
1968                 IWM_DPRINTF(sc, IWM_DEBUG_EEPROM,
1969                     "Ch. %d Flags %x [%sGHz] - Added\n",
1970                     ieee, ch_flags,
1971                     (ch_idx >= IWM_NUM_2GHZ_CHANNELS) ?
1972                     "5.2" : "2.4");
1973         }
1974 }
1975
1976 static void
1977 iwm_init_channel_map(struct ieee80211com *ic, int maxchans, int *nchans,
1978     struct ieee80211_channel chans[])
1979 {
1980         struct iwm_softc *sc = ic->ic_softc;
1981         struct iwm_nvm_data *data = sc->nvm_data;
1982         uint8_t bands[IEEE80211_MODE_BYTES];
1983         size_t ch_num;
1984
1985         memset(bands, 0, sizeof(bands));
1986         /* 1-13: 11b/g channels. */
1987         setbit(bands, IEEE80211_MODE_11B);
1988         setbit(bands, IEEE80211_MODE_11G);
1989         iwm_add_channel_band(sc, chans, maxchans, nchans, 0,
1990             IWM_NUM_2GHZ_CHANNELS - 1, bands);
1991
1992         /* 14: 11b channel only. */
1993         clrbit(bands, IEEE80211_MODE_11G);
1994         iwm_add_channel_band(sc, chans, maxchans, nchans,
1995             IWM_NUM_2GHZ_CHANNELS - 1, IWM_NUM_2GHZ_CHANNELS, bands);
1996
1997         if (data->sku_cap_band_52GHz_enable) {
1998                 if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
1999                         ch_num = nitems(iwm_nvm_channels);
2000                 else
2001                         ch_num = nitems(iwm_nvm_channels_8000);
2002                 memset(bands, 0, sizeof(bands));
2003                 setbit(bands, IEEE80211_MODE_11A);
2004                 iwm_add_channel_band(sc, chans, maxchans, nchans,
2005                     IWM_NUM_2GHZ_CHANNELS, ch_num, bands);
2006         }
2007 }
2008
2009 static void
2010 iwm_set_hw_address_family_8000(struct iwm_softc *sc, struct iwm_nvm_data *data,
2011         const uint16_t *mac_override, const uint16_t *nvm_hw)
2012 {
2013         const uint8_t *hw_addr;
2014
2015         if (mac_override) {
2016                 static const uint8_t reserved_mac[] = {
2017                         0x02, 0xcc, 0xaa, 0xff, 0xee, 0x00
2018                 };
2019
2020                 hw_addr = (const uint8_t *)(mac_override +
2021                                  IWM_MAC_ADDRESS_OVERRIDE_8000);
2022
2023                 /*
2024                  * Store the MAC address from MAO section.
2025                  * No byte swapping is required in MAO section
2026                  */
2027                 IEEE80211_ADDR_COPY(data->hw_addr, hw_addr);
2028
2029                 /*
2030                  * Force the use of the OTP MAC address in case of reserved MAC
2031                  * address in the NVM, or if address is given but invalid.
2032                  */
2033                 if (!IEEE80211_ADDR_EQ(reserved_mac, hw_addr) &&
2034                     !IEEE80211_ADDR_EQ(ieee80211broadcastaddr, data->hw_addr) &&
2035                     iwm_is_valid_ether_addr(data->hw_addr) &&
2036                     !IEEE80211_IS_MULTICAST(data->hw_addr))
2037                         return;
2038
2039                 IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2040                     "%s: mac address from nvm override section invalid\n",
2041                     __func__);
2042         }
2043
2044         if (nvm_hw) {
2045                 /* read the mac address from WFMP registers */
2046                 uint32_t mac_addr0 =
2047                     htole32(iwm_read_prph(sc, IWM_WFMP_MAC_ADDR_0));
2048                 uint32_t mac_addr1 =
2049                     htole32(iwm_read_prph(sc, IWM_WFMP_MAC_ADDR_1));
2050
2051                 hw_addr = (const uint8_t *)&mac_addr0;
2052                 data->hw_addr[0] = hw_addr[3];
2053                 data->hw_addr[1] = hw_addr[2];
2054                 data->hw_addr[2] = hw_addr[1];
2055                 data->hw_addr[3] = hw_addr[0];
2056
2057                 hw_addr = (const uint8_t *)&mac_addr1;
2058                 data->hw_addr[4] = hw_addr[1];
2059                 data->hw_addr[5] = hw_addr[0];
2060
2061                 return;
2062         }
2063
2064         device_printf(sc->sc_dev, "%s: mac address not found\n", __func__);
2065         memset(data->hw_addr, 0, sizeof(data->hw_addr));
2066 }
2067
2068 static int
2069 iwm_get_sku(const struct iwm_softc *sc, const uint16_t *nvm_sw,
2070             const uint16_t *phy_sku)
2071 {
2072         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2073                 return le16_to_cpup(nvm_sw + IWM_SKU);
2074
2075         return le32_to_cpup((const uint32_t *)(phy_sku + IWM_SKU_8000));
2076 }
2077
2078 static int
2079 iwm_get_nvm_version(const struct iwm_softc *sc, const uint16_t *nvm_sw)
2080 {
2081         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2082                 return le16_to_cpup(nvm_sw + IWM_NVM_VERSION);
2083         else
2084                 return le32_to_cpup((const uint32_t *)(nvm_sw +
2085                                                 IWM_NVM_VERSION_8000));
2086 }
2087
2088 static int
2089 iwm_get_radio_cfg(const struct iwm_softc *sc, const uint16_t *nvm_sw,
2090                   const uint16_t *phy_sku)
2091 {
2092         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2093                 return le16_to_cpup(nvm_sw + IWM_RADIO_CFG);
2094
2095         return le32_to_cpup((const uint32_t *)(phy_sku + IWM_RADIO_CFG_8000));
2096 }
2097
2098 static int
2099 iwm_get_n_hw_addrs(const struct iwm_softc *sc, const uint16_t *nvm_sw)
2100 {
2101         int n_hw_addr;
2102
2103         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000)
2104                 return le16_to_cpup(nvm_sw + IWM_N_HW_ADDRS);
2105
2106         n_hw_addr = le32_to_cpup((const uint32_t *)(nvm_sw + IWM_N_HW_ADDRS_8000));
2107
2108         return n_hw_addr & IWM_N_HW_ADDR_MASK;
2109 }
2110
2111 static void
2112 iwm_set_radio_cfg(const struct iwm_softc *sc, struct iwm_nvm_data *data,
2113                   uint32_t radio_cfg)
2114 {
2115         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
2116                 data->radio_cfg_type = IWM_NVM_RF_CFG_TYPE_MSK(radio_cfg);
2117                 data->radio_cfg_step = IWM_NVM_RF_CFG_STEP_MSK(radio_cfg);
2118                 data->radio_cfg_dash = IWM_NVM_RF_CFG_DASH_MSK(radio_cfg);
2119                 data->radio_cfg_pnum = IWM_NVM_RF_CFG_PNUM_MSK(radio_cfg);
2120                 return;
2121         }
2122
2123         /* set the radio configuration for family 8000 */
2124         data->radio_cfg_type = IWM_NVM_RF_CFG_TYPE_MSK_8000(radio_cfg);
2125         data->radio_cfg_step = IWM_NVM_RF_CFG_STEP_MSK_8000(radio_cfg);
2126         data->radio_cfg_dash = IWM_NVM_RF_CFG_DASH_MSK_8000(radio_cfg);
2127         data->radio_cfg_pnum = IWM_NVM_RF_CFG_FLAVOR_MSK_8000(radio_cfg);
2128         data->valid_tx_ant = IWM_NVM_RF_CFG_TX_ANT_MSK_8000(radio_cfg);
2129         data->valid_rx_ant = IWM_NVM_RF_CFG_RX_ANT_MSK_8000(radio_cfg);
2130 }
2131
2132 static int
2133 iwm_set_hw_address(struct iwm_softc *sc, struct iwm_nvm_data *data,
2134                    const uint16_t *nvm_hw, const uint16_t *mac_override)
2135 {
2136 #ifdef notyet /* for FAMILY 9000 */
2137         if (cfg->mac_addr_from_csr) {
2138                 iwm_set_hw_address_from_csr(sc, data);
2139         } else
2140 #endif
2141         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
2142                 const uint8_t *hw_addr = (const uint8_t *)(nvm_hw + IWM_HW_ADDR);
2143
2144                 /* The byte order is little endian 16 bit, meaning 214365 */
2145                 data->hw_addr[0] = hw_addr[1];
2146                 data->hw_addr[1] = hw_addr[0];
2147                 data->hw_addr[2] = hw_addr[3];
2148                 data->hw_addr[3] = hw_addr[2];
2149                 data->hw_addr[4] = hw_addr[5];
2150                 data->hw_addr[5] = hw_addr[4];
2151         } else {
2152                 iwm_set_hw_address_family_8000(sc, data, mac_override, nvm_hw);
2153         }
2154
2155         if (!iwm_is_valid_ether_addr(data->hw_addr)) {
2156                 device_printf(sc->sc_dev, "no valid mac address was found\n");
2157                 return EINVAL;
2158         }
2159
2160         return 0;
2161 }
2162
2163 static struct iwm_nvm_data *
2164 iwm_parse_nvm_data(struct iwm_softc *sc,
2165                    const uint16_t *nvm_hw, const uint16_t *nvm_sw,
2166                    const uint16_t *nvm_calib, const uint16_t *mac_override,
2167                    const uint16_t *phy_sku, const uint16_t *regulatory)
2168 {
2169         struct iwm_nvm_data *data;
2170         uint32_t sku, radio_cfg;
2171         uint16_t lar_config;
2172
2173         if (sc->cfg->device_family != IWM_DEVICE_FAMILY_8000) {
2174                 data = malloc(sizeof(*data) +
2175                     IWM_NUM_CHANNELS * sizeof(uint16_t),
2176                     M_DEVBUF, M_NOWAIT | M_ZERO);
2177         } else {
2178                 data = malloc(sizeof(*data) +
2179                     IWM_NUM_CHANNELS_8000 * sizeof(uint16_t),
2180                     M_DEVBUF, M_NOWAIT | M_ZERO);
2181         }
2182         if (!data)
2183                 return NULL;
2184
2185         data->nvm_version = iwm_get_nvm_version(sc, nvm_sw);
2186
2187         radio_cfg = iwm_get_radio_cfg(sc, nvm_sw, phy_sku);
2188         iwm_set_radio_cfg(sc, data, radio_cfg);
2189
2190         sku = iwm_get_sku(sc, nvm_sw, phy_sku);
2191         data->sku_cap_band_24GHz_enable = sku & IWM_NVM_SKU_CAP_BAND_24GHZ;
2192         data->sku_cap_band_52GHz_enable = sku & IWM_NVM_SKU_CAP_BAND_52GHZ;
2193         data->sku_cap_11n_enable = 0;
2194
2195         data->n_hw_addrs = iwm_get_n_hw_addrs(sc, nvm_sw);
2196
2197         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) {
2198                 uint16_t lar_offset = data->nvm_version < 0xE39 ?
2199                                        IWM_NVM_LAR_OFFSET_8000_OLD :
2200                                        IWM_NVM_LAR_OFFSET_8000;
2201
2202                 lar_config = le16_to_cpup(regulatory + lar_offset);
2203                 data->lar_enabled = !!(lar_config &
2204                                        IWM_NVM_LAR_ENABLED_8000);
2205         }
2206
2207         /* If no valid mac address was found - bail out */
2208         if (iwm_set_hw_address(sc, data, nvm_hw, mac_override)) {
2209                 free(data, M_DEVBUF);
2210                 return NULL;
2211         }
2212
2213         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
2214                 memcpy(data->nvm_ch_flags, &nvm_sw[IWM_NVM_CHANNELS],
2215                     IWM_NUM_CHANNELS * sizeof(uint16_t));
2216         } else {
2217                 memcpy(data->nvm_ch_flags, &regulatory[IWM_NVM_CHANNELS_8000],
2218                     IWM_NUM_CHANNELS_8000 * sizeof(uint16_t));
2219         }
2220
2221         return data;
2222 }
2223
2224 static void
2225 iwm_free_nvm_data(struct iwm_nvm_data *data)
2226 {
2227         if (data != NULL)
2228                 free(data, M_DEVBUF);
2229 }
2230
2231 static struct iwm_nvm_data *
2232 iwm_parse_nvm_sections(struct iwm_softc *sc, struct iwm_nvm_section *sections)
2233 {
2234         const uint16_t *hw, *sw, *calib, *regulatory, *mac_override, *phy_sku;
2235
2236         /* Checking for required sections */
2237         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000) {
2238                 if (!sections[IWM_NVM_SECTION_TYPE_SW].data ||
2239                     !sections[sc->cfg->nvm_hw_section_num].data) {
2240                         device_printf(sc->sc_dev,
2241                             "Can't parse empty OTP/NVM sections\n");
2242                         return NULL;
2243                 }
2244         } else if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) {
2245                 /* SW and REGULATORY sections are mandatory */
2246                 if (!sections[IWM_NVM_SECTION_TYPE_SW].data ||
2247                     !sections[IWM_NVM_SECTION_TYPE_REGULATORY].data) {
2248                         device_printf(sc->sc_dev,
2249                             "Can't parse empty OTP/NVM sections\n");
2250                         return NULL;
2251                 }
2252                 /* MAC_OVERRIDE or at least HW section must exist */
2253                 if (!sections[sc->cfg->nvm_hw_section_num].data &&
2254                     !sections[IWM_NVM_SECTION_TYPE_MAC_OVERRIDE].data) {
2255                         device_printf(sc->sc_dev,
2256                             "Can't parse mac_address, empty sections\n");
2257                         return NULL;
2258                 }
2259
2260                 /* PHY_SKU section is mandatory in B0 */
2261                 if (!sections[IWM_NVM_SECTION_TYPE_PHY_SKU].data) {
2262                         device_printf(sc->sc_dev,
2263                             "Can't parse phy_sku in B0, empty sections\n");
2264                         return NULL;
2265                 }
2266         } else {
2267                 panic("unknown device family %d\n", sc->cfg->device_family);
2268         }
2269
2270         hw = (const uint16_t *) sections[sc->cfg->nvm_hw_section_num].data;
2271         sw = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_SW].data;
2272         calib = (const uint16_t *)
2273             sections[IWM_NVM_SECTION_TYPE_CALIBRATION].data;
2274         regulatory = (const uint16_t *)
2275             sections[IWM_NVM_SECTION_TYPE_REGULATORY].data;
2276         mac_override = (const uint16_t *)
2277             sections[IWM_NVM_SECTION_TYPE_MAC_OVERRIDE].data;
2278         phy_sku = (const uint16_t *)sections[IWM_NVM_SECTION_TYPE_PHY_SKU].data;
2279
2280         return iwm_parse_nvm_data(sc, hw, sw, calib, mac_override,
2281             phy_sku, regulatory);
2282 }
2283
2284 static int
2285 iwm_nvm_init(struct iwm_softc *sc)
2286 {
2287         struct iwm_nvm_section nvm_sections[IWM_NVM_MAX_NUM_SECTIONS];
2288         int i, ret, section;
2289         uint32_t size_read = 0;
2290         uint8_t *nvm_buffer, *temp;
2291         uint16_t len;
2292
2293         memset(nvm_sections, 0, sizeof(nvm_sections));
2294
2295         if (sc->cfg->nvm_hw_section_num >= IWM_NVM_MAX_NUM_SECTIONS)
2296                 return EINVAL;
2297
2298         /* load NVM values from nic */
2299         /* Read From FW NVM */
2300         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM, "Read from NVM\n");
2301
2302         nvm_buffer = malloc(sc->cfg->eeprom_size, M_DEVBUF, M_NOWAIT | M_ZERO);
2303         if (!nvm_buffer)
2304                 return ENOMEM;
2305         for (section = 0; section < IWM_NVM_MAX_NUM_SECTIONS; section++) {
2306                 /* we override the constness for initial read */
2307                 ret = iwm_nvm_read_section(sc, section, nvm_buffer,
2308                                            &len, size_read);
2309                 if (ret)
2310                         continue;
2311                 size_read += len;
2312                 temp = malloc(len, M_DEVBUF, M_NOWAIT);
2313                 if (!temp) {
2314                         ret = ENOMEM;
2315                         break;
2316                 }
2317                 memcpy(temp, nvm_buffer, len);
2318
2319                 nvm_sections[section].data = temp;
2320                 nvm_sections[section].length = len;
2321         }
2322         if (!size_read)
2323                 device_printf(sc->sc_dev, "OTP is blank\n");
2324         free(nvm_buffer, M_DEVBUF);
2325
2326         sc->nvm_data = iwm_parse_nvm_sections(sc, nvm_sections);
2327         if (!sc->nvm_data)
2328                 return EINVAL;
2329         IWM_DPRINTF(sc, IWM_DEBUG_EEPROM | IWM_DEBUG_RESET,
2330                     "nvm version = %x\n", sc->nvm_data->nvm_version);
2331
2332         for (i = 0; i < IWM_NVM_MAX_NUM_SECTIONS; i++) {
2333                 if (nvm_sections[i].data != NULL)
2334                         free(nvm_sections[i].data, M_DEVBUF);
2335         }
2336
2337         return 0;
2338 }
2339
2340 static int
2341 iwm_pcie_load_section(struct iwm_softc *sc, uint8_t section_num,
2342         const struct iwm_fw_desc *section)
2343 {
2344         struct iwm_dma_info *dma = &sc->fw_dma;
2345         uint8_t *v_addr;
2346         bus_addr_t p_addr;
2347         uint32_t offset, chunk_sz = MIN(IWM_FH_MEM_TB_MAX_LENGTH, section->len);
2348         int ret = 0;
2349
2350         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2351                     "%s: [%d] uCode section being loaded...\n",
2352                     __func__, section_num);
2353
2354         v_addr = dma->vaddr;
2355         p_addr = dma->paddr;
2356
2357         for (offset = 0; offset < section->len; offset += chunk_sz) {
2358                 uint32_t copy_size, dst_addr;
2359                 int extended_addr = FALSE;
2360
2361                 copy_size = MIN(chunk_sz, section->len - offset);
2362                 dst_addr = section->offset + offset;
2363
2364                 if (dst_addr >= IWM_FW_MEM_EXTENDED_START &&
2365                     dst_addr <= IWM_FW_MEM_EXTENDED_END)
2366                         extended_addr = TRUE;
2367
2368                 if (extended_addr)
2369                         iwm_set_bits_prph(sc, IWM_LMPM_CHICK,
2370                                           IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE);
2371
2372                 memcpy(v_addr, (const uint8_t *)section->data + offset,
2373                     copy_size);
2374                 bus_dmamap_sync(dma->tag, dma->map, BUS_DMASYNC_PREWRITE);
2375                 ret = iwm_pcie_load_firmware_chunk(sc, dst_addr, p_addr,
2376                                                    copy_size);
2377
2378                 if (extended_addr)
2379                         iwm_clear_bits_prph(sc, IWM_LMPM_CHICK,
2380                                             IWM_LMPM_CHICK_EXTENDED_ADDR_SPACE);
2381
2382                 if (ret) {
2383                         device_printf(sc->sc_dev,
2384                             "%s: Could not load the [%d] uCode section\n",
2385                             __func__, section_num);
2386                         break;
2387                 }
2388         }
2389
2390         return ret;
2391 }
2392
2393 /*
2394  * ucode
2395  */
2396 static int
2397 iwm_pcie_load_firmware_chunk(struct iwm_softc *sc, uint32_t dst_addr,
2398                              bus_addr_t phy_addr, uint32_t byte_cnt)
2399 {
2400         sc->sc_fw_chunk_done = 0;
2401
2402         if (!iwm_nic_lock(sc))
2403                 return EBUSY;
2404
2405         IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(IWM_FH_SRVC_CHNL),
2406             IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_PAUSE);
2407
2408         IWM_WRITE(sc, IWM_FH_SRVC_CHNL_SRAM_ADDR_REG(IWM_FH_SRVC_CHNL),
2409             dst_addr);
2410
2411         IWM_WRITE(sc, IWM_FH_TFDIB_CTRL0_REG(IWM_FH_SRVC_CHNL),
2412             phy_addr & IWM_FH_MEM_TFDIB_DRAM_ADDR_LSB_MSK);
2413
2414         IWM_WRITE(sc, IWM_FH_TFDIB_CTRL1_REG(IWM_FH_SRVC_CHNL),
2415             (iwm_get_dma_hi_addr(phy_addr)
2416              << IWM_FH_MEM_TFDIB_REG1_ADDR_BITSHIFT) | byte_cnt);
2417
2418         IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_BUF_STS_REG(IWM_FH_SRVC_CHNL),
2419             1 << IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_NUM |
2420             1 << IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_POS_TB_IDX |
2421             IWM_FH_TCSR_CHNL_TX_BUF_STS_REG_VAL_TFDB_VALID);
2422
2423         IWM_WRITE(sc, IWM_FH_TCSR_CHNL_TX_CONFIG_REG(IWM_FH_SRVC_CHNL),
2424             IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CHNL_ENABLE    |
2425             IWM_FH_TCSR_TX_CONFIG_REG_VAL_DMA_CREDIT_DISABLE |
2426             IWM_FH_TCSR_TX_CONFIG_REG_VAL_CIRQ_HOST_ENDTFD);
2427
2428         iwm_nic_unlock(sc);
2429
2430         /* wait up to 5s for this segment to load */
2431         msleep(&sc->sc_fw, &sc->sc_mtx, 0, "iwmfw", hz * 5);
2432
2433         if (!sc->sc_fw_chunk_done) {
2434                 device_printf(sc->sc_dev,
2435                     "fw chunk addr 0x%x len %d failed to load\n",
2436                     dst_addr, byte_cnt);
2437                 return ETIMEDOUT;
2438         }
2439
2440         return 0;
2441 }
2442
2443 static int
2444 iwm_pcie_load_cpu_sections_8000(struct iwm_softc *sc,
2445         const struct iwm_fw_img *image, int cpu, int *first_ucode_section)
2446 {
2447         int shift_param;
2448         int i, ret = 0, sec_num = 0x1;
2449         uint32_t val, last_read_idx = 0;
2450
2451         if (cpu == 1) {
2452                 shift_param = 0;
2453                 *first_ucode_section = 0;
2454         } else {
2455                 shift_param = 16;
2456                 (*first_ucode_section)++;
2457         }
2458
2459         for (i = *first_ucode_section; i < IWM_UCODE_SECTION_MAX; i++) {
2460                 last_read_idx = i;
2461
2462                 /*
2463                  * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
2464                  * CPU1 to CPU2.
2465                  * PAGING_SEPARATOR_SECTION delimiter - separate between
2466                  * CPU2 non paged to CPU2 paging sec.
2467                  */
2468                 if (!image->sec[i].data ||
2469                     image->sec[i].offset == IWM_CPU1_CPU2_SEPARATOR_SECTION ||
2470                     image->sec[i].offset == IWM_PAGING_SEPARATOR_SECTION) {
2471                         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2472                                     "Break since Data not valid or Empty section, sec = %d\n",
2473                                     i);
2474                         break;
2475                 }
2476                 ret = iwm_pcie_load_section(sc, i, &image->sec[i]);
2477                 if (ret)
2478                         return ret;
2479
2480                 /* Notify the ucode of the loaded section number and status */
2481                 if (iwm_nic_lock(sc)) {
2482                         val = IWM_READ(sc, IWM_FH_UCODE_LOAD_STATUS);
2483                         val = val | (sec_num << shift_param);
2484                         IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, val);
2485                         sec_num = (sec_num << 1) | 0x1;
2486                         iwm_nic_unlock(sc);
2487                 }
2488         }
2489
2490         *first_ucode_section = last_read_idx;
2491
2492         iwm_enable_interrupts(sc);
2493
2494         if (iwm_nic_lock(sc)) {
2495                 if (cpu == 1)
2496                         IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, 0xFFFF);
2497                 else
2498                         IWM_WRITE(sc, IWM_FH_UCODE_LOAD_STATUS, 0xFFFFFFFF);
2499                 iwm_nic_unlock(sc);
2500         }
2501
2502         return 0;
2503 }
2504
2505 static int
2506 iwm_pcie_load_cpu_sections(struct iwm_softc *sc,
2507         const struct iwm_fw_img *image, int cpu, int *first_ucode_section)
2508 {
2509         int shift_param;
2510         int i, ret = 0;
2511         uint32_t last_read_idx = 0;
2512
2513         if (cpu == 1) {
2514                 shift_param = 0;
2515                 *first_ucode_section = 0;
2516         } else {
2517                 shift_param = 16;
2518                 (*first_ucode_section)++;
2519         }
2520
2521         for (i = *first_ucode_section; i < IWM_UCODE_SECTION_MAX; i++) {
2522                 last_read_idx = i;
2523
2524                 /*
2525                  * CPU1_CPU2_SEPARATOR_SECTION delimiter - separate between
2526                  * CPU1 to CPU2.
2527                  * PAGING_SEPARATOR_SECTION delimiter - separate between
2528                  * CPU2 non paged to CPU2 paging sec.
2529                  */
2530                 if (!image->sec[i].data ||
2531                     image->sec[i].offset == IWM_CPU1_CPU2_SEPARATOR_SECTION ||
2532                     image->sec[i].offset == IWM_PAGING_SEPARATOR_SECTION) {
2533                         IWM_DPRINTF(sc, IWM_DEBUG_RESET,
2534                                     "Break since Data not valid or Empty section, sec = %d\n",
2535                                      i);
2536                         break;
2537                 }
2538
2539                 ret = iwm_pcie_load_section(sc, i, &image->sec[i]);
2540                 if (ret)
2541                         return ret;
2542         }
2543
2544         *first_ucode_section = last_read_idx;
2545
2546         return 0;
2547
2548 }
2549
2550 static int
2551 iwm_pcie_load_given_ucode(struct iwm_softc *sc, const struct iwm_fw_img *image)
2552 {
2553         int ret = 0;
2554         int first_ucode_section;
2555
2556         IWM_DPRINTF(sc, IWM_DEBUG_RESET, "working with %s CPU\n",
2557                      image->is_dual_cpus ? "Dual" : "Single");
2558
2559         /* load to FW the binary non secured sections of CPU1 */
2560         ret = iwm_pcie_load_cpu_sections(sc, image, 1, &first_ucode_section);
2561         if (ret)
2562                 return ret;
2563
2564         if (image->is_dual_cpus) {
2565                 /* set CPU2 header address */
2566                 if (iwm_nic_lock(sc)) {
2567                         iwm_write_prph(sc,
2568                                        IWM_LMPM_SECURE_UCODE_LOAD_CPU2_HDR_ADDR,
2569                                        IWM_LMPM_SECURE_CPU2_HDR_MEM_SPACE);
2570                         iwm_nic_unlock(sc);
2571                 }
2572
2573                 /* load to FW the binary sections of CPU2 */
2574                 ret = iwm_pcie_load_cpu_sections(sc, image, 2,
2575                                                  &first_ucode_section);
2576                 if (ret)
2577                         return ret;
2578         }
2579
2580         iwm_enable_interrupts(sc);
2581
2582         /* release CPU reset */
2583         IWM_WRITE(sc, IWM_CSR_RESET, 0);
2584
2585         return 0;
2586 }
2587
2588 int
2589 iwm_pcie_load_given_ucode_8000(struct iwm_softc *sc,
2590         const struct iwm_fw_img *image)
2591 {
2592         int ret = 0;
2593         int first_ucode_section;
2594
2595         IWM_DPRINTF(sc, IWM_DEBUG_RESET, "working with %s CPU\n",
2596                     image->is_dual_cpus ? "Dual" : "Single");
2597
2598         /* configure the ucode to be ready to get the secured image */
2599         /* release CPU reset */
2600         if (iwm_nic_lock(sc)) {
2601                 iwm_write_prph(sc, IWM_RELEASE_CPU_RESET,
2602                     IWM_RELEASE_CPU_RESET_BIT);
2603                 iwm_nic_unlock(sc);
2604         }
2605
2606         /* load to FW the binary Secured sections of CPU1 */
2607         ret = iwm_pcie_load_cpu_sections_8000(sc, image, 1,
2608             &first_ucode_section);
2609         if (ret)
2610                 return ret;
2611
2612         /* load to FW the binary sections of CPU2 */
2613         return iwm_pcie_load_cpu_sections_8000(sc, image, 2,
2614             &first_ucode_section);
2615 }
2616
2617 /* XXX Get rid of this definition */
2618 static inline void
2619 iwm_enable_fw_load_int(struct iwm_softc *sc)
2620 {
2621         IWM_DPRINTF(sc, IWM_DEBUG_INTR, "Enabling FW load interrupt\n");
2622         sc->sc_intmask = IWM_CSR_INT_BIT_FH_TX;
2623         IWM_WRITE(sc, IWM_CSR_INT_MASK, sc->sc_intmask);
2624 }
2625
2626 /* XXX Add proper rfkill support code */
2627 static int
2628 iwm_start_fw(struct iwm_softc *sc, const struct iwm_fw_img *fw)
2629 {
2630         int ret;
2631
2632         /* This may fail if AMT took ownership of the device */
2633         if (iwm_prepare_card_hw(sc)) {
2634                 device_printf(sc->sc_dev,
2635                     "%s: Exit HW not ready\n", __func__);
2636                 ret = EIO;
2637                 goto out;
2638         }
2639
2640         IWM_WRITE(sc, IWM_CSR_INT, 0xFFFFFFFF);
2641
2642         iwm_disable_interrupts(sc);
2643
2644         /* make sure rfkill handshake bits are cleared */
2645         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL);
2646         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR,
2647             IWM_CSR_UCODE_DRV_GP1_BIT_CMD_BLOCKED);
2648
2649         /* clear (again), then enable host interrupts */
2650         IWM_WRITE(sc, IWM_CSR_INT, 0xFFFFFFFF);
2651
2652         ret = iwm_nic_init(sc);
2653         if (ret) {
2654                 device_printf(sc->sc_dev, "%s: Unable to init nic\n", __func__);
2655                 goto out;
2656         }
2657
2658         /*
2659          * Now, we load the firmware and don't want to be interrupted, even
2660          * by the RF-Kill interrupt (hence mask all the interrupt besides the
2661          * FH_TX interrupt which is needed to load the firmware). If the
2662          * RF-Kill switch is toggled, we will find out after having loaded
2663          * the firmware and return the proper value to the caller.
2664          */
2665         iwm_enable_fw_load_int(sc);
2666
2667         /* really make sure rfkill handshake bits are cleared */
2668         /* maybe we should write a few times more?  just to make sure */
2669         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL);
2670         IWM_WRITE(sc, IWM_CSR_UCODE_DRV_GP1_CLR, IWM_CSR_UCODE_SW_BIT_RFKILL);
2671
2672         /* Load the given image to the HW */
2673         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000)
2674                 ret = iwm_pcie_load_given_ucode_8000(sc, fw);
2675         else
2676                 ret = iwm_pcie_load_given_ucode(sc, fw);
2677
2678         /* XXX re-check RF-Kill state */
2679
2680 out:
2681         return ret;
2682 }
2683
2684 static int
2685 iwm_send_tx_ant_cfg(struct iwm_softc *sc, uint8_t valid_tx_ant)
2686 {
2687         struct iwm_tx_ant_cfg_cmd tx_ant_cmd = {
2688                 .valid = htole32(valid_tx_ant),
2689         };
2690
2691         return iwm_mvm_send_cmd_pdu(sc, IWM_TX_ANT_CONFIGURATION_CMD,
2692             IWM_CMD_SYNC, sizeof(tx_ant_cmd), &tx_ant_cmd);
2693 }
2694
2695 /* iwlwifi: mvm/fw.c */
2696 static int
2697 iwm_send_phy_cfg_cmd(struct iwm_softc *sc)
2698 {
2699         struct iwm_phy_cfg_cmd phy_cfg_cmd;
2700         enum iwm_ucode_type ucode_type = sc->cur_ucode;
2701
2702         /* Set parameters */
2703         phy_cfg_cmd.phy_cfg = htole32(iwm_mvm_get_phy_config(sc));
2704         phy_cfg_cmd.calib_control.event_trigger =
2705             sc->sc_default_calib[ucode_type].event_trigger;
2706         phy_cfg_cmd.calib_control.flow_trigger =
2707             sc->sc_default_calib[ucode_type].flow_trigger;
2708
2709         IWM_DPRINTF(sc, IWM_DEBUG_CMD | IWM_DEBUG_RESET,
2710             "Sending Phy CFG command: 0x%x\n", phy_cfg_cmd.phy_cfg);
2711         return iwm_mvm_send_cmd_pdu(sc, IWM_PHY_CONFIGURATION_CMD, IWM_CMD_SYNC,
2712             sizeof(phy_cfg_cmd), &phy_cfg_cmd);
2713 }
2714
2715 static int
2716 iwm_alive_fn(struct iwm_softc *sc, struct iwm_rx_packet *pkt, void *data)
2717 {
2718         struct iwm_mvm_alive_data *alive_data = data;
2719         struct iwm_mvm_alive_resp_v3 *palive3;
2720         struct iwm_mvm_alive_resp *palive;
2721         struct iwm_umac_alive *umac;
2722         struct iwm_lmac_alive *lmac1;
2723         struct iwm_lmac_alive *lmac2 = NULL;
2724         uint16_t status;
2725
2726         if (iwm_rx_packet_payload_len(pkt) == sizeof(*palive)) {
2727                 palive = (void *)pkt->data;
2728                 umac = &palive->umac_data;
2729                 lmac1 = &palive->lmac_data[0];
2730                 lmac2 = &palive->lmac_data[1];
2731                 status = le16toh(palive->status);
2732         } else {
2733                 palive3 = (void *)pkt->data;
2734                 umac = &palive3->umac_data;
2735                 lmac1 = &palive3->lmac_data;
2736                 status = le16toh(palive3->status);
2737         }
2738
2739         sc->error_event_table[0] = le32toh(lmac1->error_event_table_ptr);
2740         if (lmac2)
2741                 sc->error_event_table[1] =
2742                         le32toh(lmac2->error_event_table_ptr);
2743         sc->log_event_table = le32toh(lmac1->log_event_table_ptr);
2744         sc->umac_error_event_table = le32toh(umac->error_info_addr);
2745         alive_data->scd_base_addr = le32toh(lmac1->scd_base_ptr);
2746         alive_data->valid = status == IWM_ALIVE_STATUS_OK;
2747         if (sc->umac_error_event_table)
2748                 sc->support_umac_log = TRUE;
2749
2750         IWM_DPRINTF(sc, IWM_DEBUG_FW,
2751                     "Alive ucode status 0x%04x revision 0x%01X 0x%01X\n",
2752                     status, lmac1->ver_type, lmac1->ver_subtype);
2753
2754         if (lmac2)
2755                 IWM_DPRINTF(sc, IWM_DEBUG_FW, "Alive ucode CDB\n");
2756
2757         IWM_DPRINTF(sc, IWM_DEBUG_FW,
2758                     "UMAC version: Major - 0x%x, Minor - 0x%x\n",
2759                     le32toh(umac->umac_major),
2760                     le32toh(umac->umac_minor));
2761
2762         return TRUE;
2763 }
2764
2765 static int
2766 iwm_wait_phy_db_entry(struct iwm_softc *sc,
2767         struct iwm_rx_packet *pkt, void *data)
2768 {
2769         struct iwm_phy_db *phy_db = data;
2770
2771         if (pkt->hdr.code != IWM_CALIB_RES_NOTIF_PHY_DB) {
2772                 if(pkt->hdr.code != IWM_INIT_COMPLETE_NOTIF) {
2773                         device_printf(sc->sc_dev, "%s: Unexpected cmd: %d\n",
2774                             __func__, pkt->hdr.code);
2775                 }
2776                 return TRUE;
2777         }
2778
2779         if (iwm_phy_db_set_section(phy_db, pkt)) {
2780                 device_printf(sc->sc_dev,
2781                     "%s: iwm_phy_db_set_section failed\n", __func__);
2782         }
2783
2784         return FALSE;
2785 }
2786
2787 static int
2788 iwm_mvm_load_ucode_wait_alive(struct iwm_softc *sc,
2789         enum iwm_ucode_type ucode_type)
2790 {
2791         struct iwm_notification_wait alive_wait;
2792         struct iwm_mvm_alive_data alive_data;
2793         const struct iwm_fw_img *fw;
2794         enum iwm_ucode_type old_type = sc->cur_ucode;
2795         int error;
2796         static const uint16_t alive_cmd[] = { IWM_MVM_ALIVE };
2797
2798         fw = &sc->sc_fw.img[ucode_type];
2799         sc->cur_ucode = ucode_type;
2800         sc->ucode_loaded = FALSE;
2801
2802         memset(&alive_data, 0, sizeof(alive_data));
2803         iwm_init_notification_wait(sc->sc_notif_wait, &alive_wait,
2804                                    alive_cmd, nitems(alive_cmd),
2805                                    iwm_alive_fn, &alive_data);
2806
2807         error = iwm_start_fw(sc, fw);
2808         if (error) {
2809                 device_printf(sc->sc_dev, "iwm_start_fw: failed %d\n", error);
2810                 sc->cur_ucode = old_type;
2811                 iwm_remove_notification(sc->sc_notif_wait, &alive_wait);
2812                 return error;
2813         }
2814
2815         /*
2816          * Some things may run in the background now, but we
2817          * just wait for the ALIVE notification here.
2818          */
2819         IWM_UNLOCK(sc);
2820         error = iwm_wait_notification(sc->sc_notif_wait, &alive_wait,
2821                                       IWM_MVM_UCODE_ALIVE_TIMEOUT);
2822         IWM_LOCK(sc);
2823         if (error) {
2824                 if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) {
2825                         uint32_t a = 0x5a5a5a5a, b = 0x5a5a5a5a;
2826                         if (iwm_nic_lock(sc)) {
2827                                 a = iwm_read_prph(sc, IWM_SB_CPU_1_STATUS);
2828                                 b = iwm_read_prph(sc, IWM_SB_CPU_2_STATUS);
2829                                 iwm_nic_unlock(sc);
2830                         }
2831                         device_printf(sc->sc_dev,
2832                             "SecBoot CPU1 Status: 0x%x, CPU2 Status: 0x%x\n",
2833                             a, b);
2834                 }
2835                 sc->cur_ucode = old_type;
2836                 return error;
2837         }
2838
2839         if (!alive_data.valid) {
2840                 device_printf(sc->sc_dev, "%s: Loaded ucode is not valid\n",
2841                     __func__);
2842                 sc->cur_ucode = old_type;
2843                 return EIO;
2844         }
2845
2846         iwm_trans_pcie_fw_alive(sc, alive_data.scd_base_addr);
2847
2848         /*
2849          * configure and operate fw paging mechanism.
2850          * driver configures the paging flow only once, CPU2 paging image
2851          * included in the IWM_UCODE_INIT image.
2852          */
2853         if (fw->paging_mem_size) {
2854                 error = iwm_save_fw_paging(sc, fw);
2855                 if (error) {
2856                         device_printf(sc->sc_dev,
2857                             "%s: failed to save the FW paging image\n",
2858                             __func__);
2859                         return error;
2860                 }
2861
2862                 error = iwm_send_paging_cmd(sc, fw);
2863                 if (error) {
2864                         device_printf(sc->sc_dev,
2865                             "%s: failed to send the paging cmd\n", __func__);
2866                         iwm_free_fw_paging(sc);
2867                         return error;
2868                 }
2869         }
2870
2871         if (!error)
2872                 sc->ucode_loaded = TRUE;
2873         return error;
2874 }
2875
2876 /*
2877  * mvm misc bits
2878  */
2879
2880 /*
2881  * follows iwlwifi/fw.c
2882  */
2883 static int
2884 iwm_run_init_mvm_ucode(struct iwm_softc *sc, int justnvm)
2885 {
2886         struct iwm_notification_wait calib_wait;
2887         static const uint16_t init_complete[] = {
2888                 IWM_INIT_COMPLETE_NOTIF,
2889                 IWM_CALIB_RES_NOTIF_PHY_DB
2890         };
2891         int ret;
2892
2893         /* do not operate with rfkill switch turned on */
2894         if ((sc->sc_flags & IWM_FLAG_RFKILL) && !justnvm) {
2895                 device_printf(sc->sc_dev,
2896                     "radio is disabled by hardware switch\n");
2897                 return EPERM;
2898         }
2899
2900         iwm_init_notification_wait(sc->sc_notif_wait,
2901                                    &calib_wait,
2902                                    init_complete,
2903                                    nitems(init_complete),
2904                                    iwm_wait_phy_db_entry,
2905                                    sc->sc_phy_db);
2906
2907         /* Will also start the device */
2908         ret = iwm_mvm_load_ucode_wait_alive(sc, IWM_UCODE_INIT);
2909         if (ret) {
2910                 device_printf(sc->sc_dev, "Failed to start INIT ucode: %d\n",
2911                     ret);
2912                 goto error;
2913         }
2914
2915         if (justnvm) {
2916                 /* Read nvm */
2917                 ret = iwm_nvm_init(sc);
2918                 if (ret) {
2919                         device_printf(sc->sc_dev, "failed to read nvm\n");
2920                         goto error;
2921                 }
2922                 IEEE80211_ADDR_COPY(sc->sc_ic.ic_macaddr, sc->nvm_data->hw_addr);
2923                 goto error;
2924         }
2925
2926         ret = iwm_send_bt_init_conf(sc);
2927         if (ret) {
2928                 device_printf(sc->sc_dev,
2929                     "failed to send bt coex configuration: %d\n", ret);
2930                 goto error;
2931         }
2932
2933         /* Send TX valid antennas before triggering calibrations */
2934         ret = iwm_send_tx_ant_cfg(sc, iwm_mvm_get_valid_tx_ant(sc));
2935         if (ret) {
2936                 device_printf(sc->sc_dev,
2937                     "failed to send antennas before calibration: %d\n", ret);
2938                 goto error;
2939         }
2940
2941         /*
2942          * Send phy configurations command to init uCode
2943          * to start the 16.0 uCode init image internal calibrations.
2944          */
2945         ret = iwm_send_phy_cfg_cmd(sc);
2946         if (ret) {
2947                 device_printf(sc->sc_dev,
2948                     "%s: Failed to run INIT calibrations: %d\n",
2949                     __func__, ret);
2950                 goto error;
2951         }
2952
2953         /*
2954          * Nothing to do but wait for the init complete notification
2955          * from the firmware.
2956          */
2957         IWM_UNLOCK(sc);
2958         ret = iwm_wait_notification(sc->sc_notif_wait, &calib_wait,
2959             IWM_MVM_UCODE_CALIB_TIMEOUT);
2960         IWM_LOCK(sc);
2961
2962
2963         goto out;
2964
2965 error:
2966         iwm_remove_notification(sc->sc_notif_wait, &calib_wait);
2967 out:
2968         return ret;
2969 }
2970
2971 static int
2972 iwm_mvm_config_ltr(struct iwm_softc *sc)
2973 {
2974         struct iwm_ltr_config_cmd cmd = {
2975                 .flags = htole32(IWM_LTR_CFG_FLAG_FEATURE_ENABLE),
2976         };
2977
2978         if (!sc->sc_ltr_enabled)
2979                 return 0;
2980
2981         return iwm_mvm_send_cmd_pdu(sc, IWM_LTR_CONFIG, 0, sizeof(cmd), &cmd);
2982 }
2983
2984 /*
2985  * receive side
2986  */
2987
2988 /* (re)stock rx ring, called at init-time and at runtime */
2989 static int
2990 iwm_rx_addbuf(struct iwm_softc *sc, int size, int idx)
2991 {
2992         struct iwm_rx_ring *ring = &sc->rxq;
2993         struct iwm_rx_data *data = &ring->data[idx];
2994         struct mbuf *m;
2995         bus_dmamap_t dmamap;
2996         bus_dma_segment_t seg;
2997         int nsegs, error;
2998
2999         m = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, IWM_RBUF_SIZE);
3000         if (m == NULL)
3001                 return ENOBUFS;
3002
3003         m->m_len = m->m_pkthdr.len = m->m_ext.ext_size;
3004         error = bus_dmamap_load_mbuf_sg(ring->data_dmat, ring->spare_map, m,
3005             &seg, &nsegs, BUS_DMA_NOWAIT);
3006         if (error != 0) {
3007                 device_printf(sc->sc_dev,
3008                     "%s: can't map mbuf, error %d\n", __func__, error);
3009                 m_freem(m);
3010                 return error;
3011         }
3012
3013         if (data->m != NULL)
3014                 bus_dmamap_unload(ring->data_dmat, data->map);
3015
3016         /* Swap ring->spare_map with data->map */
3017         dmamap = data->map;
3018         data->map = ring->spare_map;
3019         ring->spare_map = dmamap;
3020
3021         bus_dmamap_sync(ring->data_dmat, data->map, BUS_DMASYNC_PREREAD);
3022         data->m = m;
3023
3024         /* Update RX descriptor. */
3025         KASSERT((seg.ds_addr & 255) == 0, ("seg.ds_addr not aligned"));
3026         ring->desc[idx] = htole32(seg.ds_addr >> 8);
3027         bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map,
3028             BUS_DMASYNC_PREWRITE);
3029
3030         return 0;
3031 }
3032
3033 /* iwlwifi: mvm/rx.c */
3034 /*
3035  * iwm_mvm_get_signal_strength - use new rx PHY INFO API
3036  * values are reported by the fw as positive values - need to negate
3037  * to obtain their dBM.  Account for missing antennas by replacing 0
3038  * values by -256dBm: practically 0 power and a non-feasible 8 bit value.
3039  */
3040 static int
3041 iwm_mvm_get_signal_strength(struct iwm_softc *sc, struct iwm_rx_phy_info *phy_info)
3042 {
3043         int energy_a, energy_b, energy_c, max_energy;
3044         uint32_t val;
3045
3046         val = le32toh(phy_info->non_cfg_phy[IWM_RX_INFO_ENERGY_ANT_ABC_IDX]);
3047         energy_a = (val & IWM_RX_INFO_ENERGY_ANT_A_MSK) >>
3048             IWM_RX_INFO_ENERGY_ANT_A_POS;
3049         energy_a = energy_a ? -energy_a : -256;
3050         energy_b = (val & IWM_RX_INFO_ENERGY_ANT_B_MSK) >>
3051             IWM_RX_INFO_ENERGY_ANT_B_POS;
3052         energy_b = energy_b ? -energy_b : -256;
3053         energy_c = (val & IWM_RX_INFO_ENERGY_ANT_C_MSK) >>
3054             IWM_RX_INFO_ENERGY_ANT_C_POS;
3055         energy_c = energy_c ? -energy_c : -256;
3056         max_energy = MAX(energy_a, energy_b);
3057         max_energy = MAX(max_energy, energy_c);
3058
3059         IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3060             "energy In A %d B %d C %d , and max %d\n",
3061             energy_a, energy_b, energy_c, max_energy);
3062
3063         return max_energy;
3064 }
3065
3066 static void
3067 iwm_mvm_rx_rx_phy_cmd(struct iwm_softc *sc, struct iwm_rx_packet *pkt)
3068 {
3069         struct iwm_rx_phy_info *phy_info = (void *)pkt->data;
3070
3071         IWM_DPRINTF(sc, IWM_DEBUG_RECV, "received PHY stats\n");
3072
3073         memcpy(&sc->sc_last_phy_info, phy_info, sizeof(sc->sc_last_phy_info));
3074 }
3075
3076 /*
3077  * Retrieve the average noise (in dBm) among receivers.
3078  */
3079 static int
3080 iwm_get_noise(struct iwm_softc *sc,
3081     const struct iwm_mvm_statistics_rx_non_phy *stats)
3082 {
3083         int i, total, nbant, noise;
3084
3085         total = nbant = noise = 0;
3086         for (i = 0; i < 3; i++) {
3087                 noise = le32toh(stats->beacon_silence_rssi[i]) & 0xff;
3088                 IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: i=%d, noise=%d\n",
3089                     __func__,
3090                     i,
3091                     noise);
3092
3093                 if (noise) {
3094                         total += noise;
3095                         nbant++;
3096                 }
3097         }
3098
3099         IWM_DPRINTF(sc, IWM_DEBUG_RECV, "%s: nbant=%d, total=%d\n",
3100             __func__, nbant, total);
3101 #if 0
3102         /* There should be at least one antenna but check anyway. */
3103         return (nbant == 0) ? -127 : (total / nbant) - 107;
3104 #else
3105         /* For now, just hard-code it to -96 to be safe */
3106         return (-96);
3107 #endif
3108 }
3109
3110 static void
3111 iwm_mvm_handle_rx_statistics(struct iwm_softc *sc, struct iwm_rx_packet *pkt)
3112 {
3113         struct iwm_notif_statistics_v10 *stats = (void *)&pkt->data;
3114
3115         memcpy(&sc->sc_stats, stats, sizeof(sc->sc_stats));
3116         sc->sc_noise = iwm_get_noise(sc, &stats->rx.general);
3117 }
3118
3119 /*
3120  * iwm_mvm_rx_rx_mpdu - IWM_REPLY_RX_MPDU_CMD handler
3121  *
3122  * Handles the actual data of the Rx packet from the fw
3123  */
3124 static boolean_t
3125 iwm_mvm_rx_rx_mpdu(struct iwm_softc *sc, struct mbuf *m, uint32_t offset,
3126         boolean_t stolen)
3127 {
3128         struct ieee80211com *ic = &sc->sc_ic;
3129         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
3130         struct ieee80211_frame *wh;
3131         struct ieee80211_node *ni;
3132         struct ieee80211_rx_stats rxs;
3133         struct iwm_rx_phy_info *phy_info;
3134         struct iwm_rx_mpdu_res_start *rx_res;
3135         struct iwm_rx_packet *pkt = mtodoff(m, struct iwm_rx_packet *, offset);
3136         uint32_t len;
3137         uint32_t rx_pkt_status;
3138         int rssi;
3139
3140         phy_info = &sc->sc_last_phy_info;
3141         rx_res = (struct iwm_rx_mpdu_res_start *)pkt->data;
3142         wh = (struct ieee80211_frame *)(pkt->data + sizeof(*rx_res));
3143         len = le16toh(rx_res->byte_count);
3144         rx_pkt_status = le32toh(*(uint32_t *)(pkt->data + sizeof(*rx_res) + len));
3145
3146         if (__predict_false(phy_info->cfg_phy_cnt > 20)) {
3147                 device_printf(sc->sc_dev,
3148                     "dsp size out of range [0,20]: %d\n",
3149                     phy_info->cfg_phy_cnt);
3150                 goto fail;
3151         }
3152
3153         if (!(rx_pkt_status & IWM_RX_MPDU_RES_STATUS_CRC_OK) ||
3154             !(rx_pkt_status & IWM_RX_MPDU_RES_STATUS_OVERRUN_OK)) {
3155                 IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3156                     "Bad CRC or FIFO: 0x%08X.\n", rx_pkt_status);
3157                 goto fail;
3158         }
3159
3160         rssi = iwm_mvm_get_signal_strength(sc, phy_info);
3161
3162         /* Map it to relative value */
3163         rssi = rssi - sc->sc_noise;
3164
3165         /* replenish ring for the buffer we're going to feed to the sharks */
3166         if (!stolen && iwm_rx_addbuf(sc, IWM_RBUF_SIZE, sc->rxq.cur) != 0) {
3167                 device_printf(sc->sc_dev, "%s: unable to add more buffers\n",
3168                     __func__);
3169                 goto fail;
3170         }
3171
3172         m->m_data = pkt->data + sizeof(*rx_res);
3173         m->m_pkthdr.len = m->m_len = len;
3174
3175         IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3176             "%s: rssi=%d, noise=%d\n", __func__, rssi, sc->sc_noise);
3177
3178         ni = ieee80211_find_rxnode(ic, (struct ieee80211_frame_min *)wh);
3179
3180         IWM_DPRINTF(sc, IWM_DEBUG_RECV,
3181             "%s: phy_info: channel=%d, flags=0x%08x\n",
3182             __func__,
3183             le16toh(phy_info->channel),
3184             le16toh(phy_info->phy_flags));
3185
3186         /*
3187          * Populate an RX state struct with the provided information.
3188          */
3189         bzero(&rxs, sizeof(rxs));
3190         rxs.r_flags |= IEEE80211_R_IEEE | IEEE80211_R_FREQ;
3191         rxs.r_flags |= IEEE80211_R_NF | IEEE80211_R_RSSI;
3192         rxs.c_ieee = le16toh(phy_info->channel);
3193         if (le16toh(phy_info->phy_flags & IWM_RX_RES_PHY_FLAGS_BAND_24)) {
3194                 rxs.c_freq = ieee80211_ieee2mhz(rxs.c_ieee, IEEE80211_CHAN_2GHZ);
3195         } else {
3196                 rxs.c_freq = ieee80211_ieee2mhz(rxs.c_ieee, IEEE80211_CHAN_5GHZ);
3197         }
3198
3199         /* rssi is in 1/2db units */
3200         rxs.c_rssi = rssi * 2;
3201         rxs.c_nf = sc->sc_noise;
3202         if (ieee80211_add_rx_params(m, &rxs) == 0) {
3203                 if (ni)
3204                         ieee80211_free_node(ni);
3205                 goto fail;
3206         }
3207
3208         if (ieee80211_radiotap_active_vap(vap)) {
3209                 struct iwm_rx_radiotap_header *tap = &sc->sc_rxtap;
3210
3211                 tap->wr_flags = 0;
3212                 if (phy_info->phy_flags & htole16(IWM_PHY_INFO_FLAG_SHPREAMBLE))
3213                         tap->wr_flags |= IEEE80211_RADIOTAP_F_SHORTPRE;
3214                 tap->wr_chan_freq = htole16(rxs.c_freq);
3215                 /* XXX only if ic->ic_curchan->ic_ieee == rxs.c_ieee */
3216                 tap->wr_chan_flags = htole16(ic->ic_curchan->ic_flags);
3217                 tap->wr_dbm_antsignal = (int8_t)rssi;
3218                 tap->wr_dbm_antnoise = (int8_t)sc->sc_noise;
3219                 tap->wr_tsft = phy_info->system_timestamp;
3220                 switch (phy_info->rate) {
3221                 /* CCK rates. */
3222                 case  10: tap->wr_rate =   2; break;
3223                 case  20: tap->wr_rate =   4; break;
3224                 case  55: tap->wr_rate =  11; break;
3225                 case 110: tap->wr_rate =  22; break;
3226                 /* OFDM rates. */
3227                 case 0xd: tap->wr_rate =  12; break;
3228                 case 0xf: tap->wr_rate =  18; break;
3229                 case 0x5: tap->wr_rate =  24; break;
3230                 case 0x7: tap->wr_rate =  36; break;
3231                 case 0x9: tap->wr_rate =  48; break;
3232                 case 0xb: tap->wr_rate =  72; break;
3233                 case 0x1: tap->wr_rate =  96; break;
3234                 case 0x3: tap->wr_rate = 108; break;
3235                 /* Unknown rate: should not happen. */
3236                 default:  tap->wr_rate =   0;
3237                 }
3238         }
3239
3240         IWM_UNLOCK(sc);
3241         if (ni != NULL) {
3242                 IWM_DPRINTF(sc, IWM_DEBUG_RECV, "input m %p\n", m);
3243                 ieee80211_input_mimo(ni, m);
3244                 ieee80211_free_node(ni);
3245         } else {
3246                 IWM_DPRINTF(sc, IWM_DEBUG_RECV, "inputall m %p\n", m);
3247                 ieee80211_input_mimo_all(ic, m);
3248         }
3249         IWM_LOCK(sc);
3250
3251         return TRUE;
3252
3253 fail:
3254         counter_u64_add(ic->ic_ierrors, 1);
3255         return FALSE;
3256 }
3257
3258 static int
3259 iwm_mvm_rx_tx_cmd_single(struct iwm_softc *sc, struct iwm_rx_packet *pkt,
3260         struct iwm_node *in)
3261 {
3262         struct iwm_mvm_tx_resp *tx_resp = (void *)pkt->data;
3263         struct ieee80211_ratectl_tx_status *txs = &sc->sc_txs;
3264         struct ieee80211_node *ni = &in->in_ni;
3265         struct ieee80211vap *vap = ni->ni_vap;
3266         int status = le16toh(tx_resp->status.status) & IWM_TX_STATUS_MSK;
3267         int new_rate, cur_rate = vap->iv_bss->ni_txrate;
3268         boolean_t rate_matched;
3269         uint8_t tx_resp_rate;
3270
3271         KASSERT(tx_resp->frame_count == 1, ("too many frames"));
3272
3273         /* Update rate control statistics. */
3274         IWM_DPRINTF(sc, IWM_DEBUG_XMIT, "%s: status=0x%04x, seq=%d, fc=%d, btc=%d, frts=%d, ff=%d, irate=%08x, wmt=%d\n",
3275             __func__,
3276             (int) le16toh(tx_resp->status.status),
3277             (int) le16toh(tx_resp->status.sequence),
3278             tx_resp->frame_count,
3279             tx_resp->bt_kill_count,
3280             tx_resp->failure_rts,
3281             tx_resp->failure_frame,
3282             le32toh(tx_resp->initial_rate),
3283             (int) le16toh(tx_resp->wireless_media_time));
3284
3285         tx_resp_rate = iwm_rate_from_ucode_rate(le32toh(tx_resp->initial_rate));
3286
3287         /* For rate control, ignore frames sent at different initial rate */
3288         rate_matched = (tx_resp_rate != 0 && tx_resp_rate == cur_rate);
3289
3290         if (tx_resp_rate != 0 && cur_rate != 0 && !rate_matched) {
3291                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3292                     "tx_resp_rate doesn't match ni_txrate (tx_resp_rate=%u "
3293                     "ni_txrate=%d)\n", tx_resp_rate, cur_rate);
3294         }
3295
3296         txs->flags = IEEE80211_RATECTL_STATUS_SHORT_RETRY |
3297                      IEEE80211_RATECTL_STATUS_LONG_RETRY;
3298         txs->short_retries = tx_resp->failure_rts;
3299         txs->long_retries = tx_resp->failure_frame;
3300         if (status != IWM_TX_STATUS_SUCCESS &&
3301             status != IWM_TX_STATUS_DIRECT_DONE) {
3302                 switch (status) {
3303                 case IWM_TX_STATUS_FAIL_SHORT_LIMIT:
3304                         txs->status = IEEE80211_RATECTL_TX_FAIL_SHORT;
3305                         break;
3306                 case IWM_TX_STATUS_FAIL_LONG_LIMIT:
3307                         txs->status = IEEE80211_RATECTL_TX_FAIL_LONG;
3308                         break;
3309                 case IWM_TX_STATUS_FAIL_LIFE_EXPIRE:
3310                         txs->status = IEEE80211_RATECTL_TX_FAIL_EXPIRED;
3311                         break;
3312                 default:
3313                         txs->status = IEEE80211_RATECTL_TX_FAIL_UNSPECIFIED;
3314                         break;
3315                 }
3316         } else {
3317                 txs->status = IEEE80211_RATECTL_TX_SUCCESS;
3318         }
3319
3320         if (rate_matched) {
3321                 ieee80211_ratectl_tx_complete(ni, txs);
3322
3323                 int rix = ieee80211_ratectl_rate(vap->iv_bss, NULL, 0);
3324                 new_rate = vap->iv_bss->ni_txrate;
3325                 if (new_rate != 0 && new_rate != cur_rate) {
3326                         struct iwm_node *in = IWM_NODE(vap->iv_bss);
3327                         iwm_setrates(sc, in, rix);
3328                         iwm_mvm_send_lq_cmd(sc, &in->in_lq, FALSE);
3329                 }
3330         }
3331
3332         return (txs->status != IEEE80211_RATECTL_TX_SUCCESS);
3333 }
3334
3335 static void
3336 iwm_mvm_rx_tx_cmd(struct iwm_softc *sc, struct iwm_rx_packet *pkt)
3337 {
3338         struct iwm_cmd_header *cmd_hdr = &pkt->hdr;
3339         int idx = cmd_hdr->idx;
3340         int qid = cmd_hdr->qid;
3341         struct iwm_tx_ring *ring = &sc->txq[qid];
3342         struct iwm_tx_data *txd = &ring->data[idx];
3343         struct iwm_node *in = txd->in;
3344         struct mbuf *m = txd->m;
3345         int status;
3346
3347         KASSERT(txd->done == 0, ("txd not done"));
3348         KASSERT(txd->in != NULL, ("txd without node"));
3349         KASSERT(txd->m != NULL, ("txd without mbuf"));
3350
3351         sc->sc_tx_timer = 0;
3352
3353         status = iwm_mvm_rx_tx_cmd_single(sc, pkt, in);
3354
3355         /* Unmap and free mbuf. */
3356         bus_dmamap_sync(ring->data_dmat, txd->map, BUS_DMASYNC_POSTWRITE);
3357         bus_dmamap_unload(ring->data_dmat, txd->map);
3358
3359         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3360             "free txd %p, in %p\n", txd, txd->in);
3361         txd->done = 1;
3362         txd->m = NULL;
3363         txd->in = NULL;
3364
3365         ieee80211_tx_complete(&in->in_ni, m, status);
3366
3367         if (--ring->queued < IWM_TX_RING_LOMARK) {
3368                 sc->qfullmsk &= ~(1 << ring->qid);
3369                 if (sc->qfullmsk == 0) {
3370                         iwm_start(sc);
3371                 }
3372         }
3373 }
3374
3375 /*
3376  * transmit side
3377  */
3378
3379 /*
3380  * Process a "command done" firmware notification.  This is where we wakeup
3381  * processes waiting for a synchronous command completion.
3382  * from if_iwn
3383  */
3384 static void
3385 iwm_cmd_done(struct iwm_softc *sc, struct iwm_rx_packet *pkt)
3386 {
3387         struct iwm_tx_ring *ring = &sc->txq[IWM_MVM_CMD_QUEUE];
3388         struct iwm_tx_data *data;
3389
3390         if (pkt->hdr.qid != IWM_MVM_CMD_QUEUE) {
3391                 return; /* Not a command ack. */
3392         }
3393
3394         /* XXX wide commands? */
3395         IWM_DPRINTF(sc, IWM_DEBUG_CMD,
3396             "cmd notification type 0x%x qid %d idx %d\n",
3397             pkt->hdr.code, pkt->hdr.qid, pkt->hdr.idx);
3398
3399         data = &ring->data[pkt->hdr.idx];
3400
3401         /* If the command was mapped in an mbuf, free it. */
3402         if (data->m != NULL) {
3403                 bus_dmamap_sync(ring->data_dmat, data->map,
3404                     BUS_DMASYNC_POSTWRITE);
3405                 bus_dmamap_unload(ring->data_dmat, data->map);
3406                 m_freem(data->m);
3407                 data->m = NULL;
3408         }
3409         wakeup(&ring->desc[pkt->hdr.idx]);
3410
3411         if (((pkt->hdr.idx + ring->queued) % IWM_TX_RING_COUNT) != ring->cur) {
3412                 device_printf(sc->sc_dev,
3413                     "%s: Some HCMDs skipped?: idx=%d queued=%d cur=%d\n",
3414                     __func__, pkt->hdr.idx, ring->queued, ring->cur);
3415                 /* XXX call iwm_force_nmi() */
3416         }
3417
3418         KASSERT(ring->queued > 0, ("ring->queued is empty?"));
3419         ring->queued--;
3420         if (ring->queued == 0)
3421                 iwm_pcie_clear_cmd_in_flight(sc);
3422 }
3423
3424 #if 0
3425 /*
3426  * necessary only for block ack mode
3427  */
3428 void
3429 iwm_update_sched(struct iwm_softc *sc, int qid, int idx, uint8_t sta_id,
3430         uint16_t len)
3431 {
3432         struct iwm_agn_scd_bc_tbl *scd_bc_tbl;
3433         uint16_t w_val;
3434
3435         scd_bc_tbl = sc->sched_dma.vaddr;
3436
3437         len += 8; /* magic numbers came naturally from paris */
3438         len = roundup(len, 4) / 4;
3439
3440         w_val = htole16(sta_id << 12 | len);
3441
3442         /* Update TX scheduler. */
3443         scd_bc_tbl[qid].tfd_offset[idx] = w_val;
3444         bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map,
3445             BUS_DMASYNC_PREWRITE);
3446
3447         /* I really wonder what this is ?!? */
3448         if (idx < IWM_TFD_QUEUE_SIZE_BC_DUP) {
3449                 scd_bc_tbl[qid].tfd_offset[IWM_TFD_QUEUE_SIZE_MAX + idx] = w_val;
3450                 bus_dmamap_sync(sc->sched_dma.tag, sc->sched_dma.map,
3451                     BUS_DMASYNC_PREWRITE);
3452         }
3453 }
3454 #endif
3455
3456 static int
3457 iwm_tx_rateidx_global_lookup(struct iwm_softc *sc, uint8_t rate)
3458 {
3459         int i;
3460
3461         for (i = 0; i < nitems(iwm_rates); i++) {
3462                 if (iwm_rates[i].rate == rate)
3463                         return (i);
3464         }
3465         /* XXX error? */
3466         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE,
3467             "%s: couldn't find an entry for rate=%d\n",
3468             __func__,
3469             rate);
3470         return (0);
3471 }
3472
3473 /*
3474  * Fill in the rate related information for a transmit command.
3475  */
3476 static const struct iwm_rate *
3477 iwm_tx_fill_cmd(struct iwm_softc *sc, struct iwm_node *in,
3478         struct mbuf *m, struct iwm_tx_cmd *tx)
3479 {
3480         struct ieee80211_node *ni = &in->in_ni;
3481         struct ieee80211_frame *wh;
3482         const struct ieee80211_txparam *tp = ni->ni_txparms;
3483         const struct iwm_rate *rinfo;
3484         int type;
3485         int ridx, rate_flags;
3486
3487         wh = mtod(m, struct ieee80211_frame *);
3488         type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK;
3489
3490         tx->rts_retry_limit = IWM_RTS_DFAULT_RETRY_LIMIT;
3491         tx->data_retry_limit = IWM_DEFAULT_TX_RETRY;
3492
3493         if (type == IEEE80211_FC0_TYPE_MGT ||
3494             type == IEEE80211_FC0_TYPE_CTL ||
3495             (m->m_flags & M_EAPOL) != 0) {
3496                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->mgmtrate);
3497                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3498                     "%s: MGT (%d)\n", __func__, tp->mgmtrate);
3499         } else if (IEEE80211_IS_MULTICAST(wh->i_addr1)) {
3500                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->mcastrate);
3501                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3502                     "%s: MCAST (%d)\n", __func__, tp->mcastrate);
3503         } else if (tp->ucastrate != IEEE80211_FIXED_RATE_NONE) {
3504                 ridx = iwm_tx_rateidx_global_lookup(sc, tp->ucastrate);
3505                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
3506                     "%s: FIXED_RATE (%d)\n", __func__, tp->ucastrate);
3507         } else {
3508                 /* for data frames, use RS table */
3509                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: DATA\n", __func__);
3510                 ridx = iwm_rate2ridx(sc, ni->ni_txrate);
3511                 if (ridx == -1)
3512                         ridx = 0;
3513
3514                 /* This is the index into the programmed table */
3515                 tx->initial_rate_index = 0;
3516                 tx->tx_flags |= htole32(IWM_TX_CMD_FLG_STA_RATE);
3517         }
3518
3519         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TXRATE,
3520             "%s: frame type=%d txrate %d\n",
3521                 __func__, type, iwm_rates[ridx].rate);
3522
3523         rinfo = &iwm_rates[ridx];
3524
3525         IWM_DPRINTF(sc, IWM_DEBUG_TXRATE, "%s: ridx=%d; rate=%d, CCK=%d\n",
3526             __func__, ridx,
3527             rinfo->rate,
3528             !! (IWM_RIDX_IS_CCK(ridx))
3529             );
3530
3531         /* XXX TODO: hard-coded TX antenna? */
3532         rate_flags = 1 << IWM_RATE_MCS_ANT_POS;
3533         if (IWM_RIDX_IS_CCK(ridx))
3534                 rate_flags |= IWM_RATE_MCS_CCK_MSK;
3535         tx->rate_n_flags = htole32(rate_flags | rinfo->plcp);
3536
3537         return rinfo;
3538 }
3539
3540 #define TB0_SIZE 16
3541 static int
3542 iwm_tx(struct iwm_softc *sc, struct mbuf *m, struct ieee80211_node *ni, int ac)
3543 {
3544         struct ieee80211com *ic = &sc->sc_ic;
3545         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
3546         struct iwm_node *in = IWM_NODE(ni);
3547         struct iwm_tx_ring *ring;
3548         struct iwm_tx_data *data;
3549         struct iwm_tfd *desc;
3550         struct iwm_device_cmd *cmd;
3551         struct iwm_tx_cmd *tx;
3552         struct ieee80211_frame *wh;
3553         struct ieee80211_key *k = NULL;
3554         struct mbuf *m1;
3555         const struct iwm_rate *rinfo;
3556         uint32_t flags;
3557         u_int hdrlen;
3558         bus_dma_segment_t *seg, segs[IWM_MAX_SCATTER];
3559         int nsegs;
3560         uint8_t tid, type;
3561         int i, totlen, error, pad;
3562
3563         wh = mtod(m, struct ieee80211_frame *);
3564         hdrlen = ieee80211_anyhdrsize(wh);
3565         type = wh->i_fc[0] & IEEE80211_FC0_TYPE_MASK;
3566         tid = 0;
3567         ring = &sc->txq[ac];
3568         desc = &ring->desc[ring->cur];
3569         memset(desc, 0, sizeof(*desc));
3570         data = &ring->data[ring->cur];
3571
3572         /* Fill out iwm_tx_cmd to send to the firmware */
3573         cmd = &ring->cmd[ring->cur];
3574         cmd->hdr.code = IWM_TX_CMD;
3575         cmd->hdr.flags = 0;
3576         cmd->hdr.qid = ring->qid;
3577         cmd->hdr.idx = ring->cur;
3578
3579         tx = (void *)cmd->data;
3580         memset(tx, 0, sizeof(*tx));
3581
3582         rinfo = iwm_tx_fill_cmd(sc, in, m, tx);
3583
3584         /* Encrypt the frame if need be. */
3585         if (wh->i_fc[1] & IEEE80211_FC1_PROTECTED) {
3586                 /* Retrieve key for TX && do software encryption. */
3587                 k = ieee80211_crypto_encap(ni, m);
3588                 if (k == NULL) {
3589                         m_freem(m);
3590                         return (ENOBUFS);
3591                 }
3592                 /* 802.11 header may have moved. */
3593                 wh = mtod(m, struct ieee80211_frame *);
3594         }
3595
3596         if (ieee80211_radiotap_active_vap(vap)) {
3597                 struct iwm_tx_radiotap_header *tap = &sc->sc_txtap;
3598
3599                 tap->wt_flags = 0;
3600                 tap->wt_chan_freq = htole16(ni->ni_chan->ic_freq);
3601                 tap->wt_chan_flags = htole16(ni->ni_chan->ic_flags);
3602                 tap->wt_rate = rinfo->rate;
3603                 if (k != NULL)
3604                         tap->wt_flags |= IEEE80211_RADIOTAP_F_WEP;
3605                 ieee80211_radiotap_tx(vap, m);
3606         }
3607
3608
3609         totlen = m->m_pkthdr.len;
3610
3611         flags = 0;
3612         if (!IEEE80211_IS_MULTICAST(wh->i_addr1)) {
3613                 flags |= IWM_TX_CMD_FLG_ACK;
3614         }
3615
3616         if (type == IEEE80211_FC0_TYPE_DATA
3617             && (totlen + IEEE80211_CRC_LEN > vap->iv_rtsthreshold)
3618             && !IEEE80211_IS_MULTICAST(wh->i_addr1)) {
3619                 flags |= IWM_TX_CMD_FLG_PROT_REQUIRE;
3620         }
3621
3622         if (IEEE80211_IS_MULTICAST(wh->i_addr1) ||
3623             type != IEEE80211_FC0_TYPE_DATA)
3624                 tx->sta_id = sc->sc_aux_sta.sta_id;
3625         else
3626                 tx->sta_id = IWM_STATION_ID;
3627
3628         if (type == IEEE80211_FC0_TYPE_MGT) {
3629                 uint8_t subtype = wh->i_fc[0] & IEEE80211_FC0_SUBTYPE_MASK;
3630
3631                 if (subtype == IEEE80211_FC0_SUBTYPE_ASSOC_REQ ||
3632                     subtype == IEEE80211_FC0_SUBTYPE_REASSOC_REQ) {
3633                         tx->pm_frame_timeout = htole16(IWM_PM_FRAME_ASSOC);
3634                 } else if (subtype == IEEE80211_FC0_SUBTYPE_ACTION) {
3635                         tx->pm_frame_timeout = htole16(IWM_PM_FRAME_NONE);
3636                 } else {
3637                         tx->pm_frame_timeout = htole16(IWM_PM_FRAME_MGMT);
3638                 }
3639         } else {
3640                 tx->pm_frame_timeout = htole16(IWM_PM_FRAME_NONE);
3641         }
3642
3643         if (hdrlen & 3) {
3644                 /* First segment length must be a multiple of 4. */
3645                 flags |= IWM_TX_CMD_FLG_MH_PAD;
3646                 pad = 4 - (hdrlen & 3);
3647         } else
3648                 pad = 0;
3649
3650         tx->driver_txop = 0;
3651         tx->next_frame_len = 0;
3652
3653         tx->len = htole16(totlen);
3654         tx->tid_tspec = tid;
3655         tx->life_time = htole32(IWM_TX_CMD_LIFE_TIME_INFINITE);
3656
3657         /* Set physical address of "scratch area". */
3658         tx->dram_lsb_ptr = htole32(data->scratch_paddr);
3659         tx->dram_msb_ptr = iwm_get_dma_hi_addr(data->scratch_paddr);
3660
3661         /* Copy 802.11 header in TX command. */
3662         memcpy(((uint8_t *)tx) + sizeof(*tx), wh, hdrlen);
3663
3664         flags |= IWM_TX_CMD_FLG_BT_DIS | IWM_TX_CMD_FLG_SEQ_CTL;
3665
3666         tx->sec_ctl = 0;
3667         tx->tx_flags |= htole32(flags);
3668
3669         /* Trim 802.11 header. */
3670         m_adj(m, hdrlen);
3671         error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m,
3672             segs, &nsegs, BUS_DMA_NOWAIT);
3673         if (error != 0) {
3674                 if (error != EFBIG) {
3675                         device_printf(sc->sc_dev, "can't map mbuf (error %d)\n",
3676                             error);
3677                         m_freem(m);
3678                         return error;
3679                 }
3680                 /* Too many DMA segments, linearize mbuf. */
3681                 m1 = m_collapse(m, M_NOWAIT, IWM_MAX_SCATTER - 2);
3682                 if (m1 == NULL) {
3683                         device_printf(sc->sc_dev,
3684                             "%s: could not defrag mbuf\n", __func__);
3685                         m_freem(m);
3686                         return (ENOBUFS);
3687                 }
3688                 m = m1;
3689
3690                 error = bus_dmamap_load_mbuf_sg(ring->data_dmat, data->map, m,
3691                     segs, &nsegs, BUS_DMA_NOWAIT);
3692                 if (error != 0) {
3693                         device_printf(sc->sc_dev, "can't map mbuf (error %d)\n",
3694                             error);
3695                         m_freem(m);
3696                         return error;
3697                 }
3698         }
3699         data->m = m;
3700         data->in = in;
3701         data->done = 0;
3702
3703         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3704             "sending txd %p, in %p\n", data, data->in);
3705         KASSERT(data->in != NULL, ("node is NULL"));
3706
3707         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3708             "sending data: qid=%d idx=%d len=%d nsegs=%d txflags=0x%08x rate_n_flags=0x%08x rateidx=%u\n",
3709             ring->qid, ring->cur, totlen, nsegs,
3710             le32toh(tx->tx_flags),
3711             le32toh(tx->rate_n_flags),
3712             tx->initial_rate_index
3713             );
3714
3715         /* Fill TX descriptor. */
3716         desc->num_tbs = 2 + nsegs;
3717
3718         desc->tbs[0].lo = htole32(data->cmd_paddr);
3719         desc->tbs[0].hi_n_len = htole16(iwm_get_dma_hi_addr(data->cmd_paddr)) |
3720             (TB0_SIZE << 4);
3721         desc->tbs[1].lo = htole32(data->cmd_paddr + TB0_SIZE);
3722         desc->tbs[1].hi_n_len = htole16(iwm_get_dma_hi_addr(data->cmd_paddr)) |
3723             ((sizeof(struct iwm_cmd_header) + sizeof(*tx)
3724               + hdrlen + pad - TB0_SIZE) << 4);
3725
3726         /* Other DMA segments are for data payload. */
3727         for (i = 0; i < nsegs; i++) {
3728                 seg = &segs[i];
3729                 desc->tbs[i+2].lo = htole32(seg->ds_addr);
3730                 desc->tbs[i+2].hi_n_len = \
3731                     htole16(iwm_get_dma_hi_addr(seg->ds_addr))
3732                     | ((seg->ds_len) << 4);
3733         }
3734
3735         bus_dmamap_sync(ring->data_dmat, data->map,
3736             BUS_DMASYNC_PREWRITE);
3737         bus_dmamap_sync(ring->cmd_dma.tag, ring->cmd_dma.map,
3738             BUS_DMASYNC_PREWRITE);
3739         bus_dmamap_sync(ring->desc_dma.tag, ring->desc_dma.map,
3740             BUS_DMASYNC_PREWRITE);
3741
3742 #if 0
3743         iwm_update_sched(sc, ring->qid, ring->cur, tx->sta_id, le16toh(tx->len));
3744 #endif
3745
3746         /* Kick TX ring. */
3747         ring->cur = (ring->cur + 1) % IWM_TX_RING_COUNT;
3748         IWM_WRITE(sc, IWM_HBUS_TARG_WRPTR, ring->qid << 8 | ring->cur);
3749
3750         /* Mark TX ring as full if we reach a certain threshold. */
3751         if (++ring->queued > IWM_TX_RING_HIMARK) {
3752                 sc->qfullmsk |= 1 << ring->qid;
3753         }
3754
3755         return 0;
3756 }
3757
3758 static int
3759 iwm_raw_xmit(struct ieee80211_node *ni, struct mbuf *m,
3760     const struct ieee80211_bpf_params *params)
3761 {
3762         struct ieee80211com *ic = ni->ni_ic;
3763         struct iwm_softc *sc = ic->ic_softc;
3764         int error = 0;
3765
3766         IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3767             "->%s begin\n", __func__);
3768
3769         if ((sc->sc_flags & IWM_FLAG_HW_INITED) == 0) {
3770                 m_freem(m);
3771                 IWM_DPRINTF(sc, IWM_DEBUG_XMIT,
3772                     "<-%s not RUNNING\n", __func__);
3773                 return (ENETDOWN);
3774         }
3775
3776         IWM_LOCK(sc);
3777         /* XXX fix this */
3778         if (params == NULL) {
3779                 error = iwm_tx(sc, m, ni, 0);
3780         } else {
3781                 error = iwm_tx(sc, m, ni, 0);
3782         }
3783         if (sc->sc_tx_timer == 0)
3784                 callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog, sc);
3785         sc->sc_tx_timer = 5;
3786         IWM_UNLOCK(sc);
3787
3788         return (error);
3789 }
3790
3791 /*
3792  * mvm/tx.c
3793  */
3794
3795 /*
3796  * Note that there are transports that buffer frames before they reach
3797  * the firmware. This means that after flush_tx_path is called, the
3798  * queue might not be empty. The race-free way to handle this is to:
3799  * 1) set the station as draining
3800  * 2) flush the Tx path
3801  * 3) wait for the transport queues to be empty
3802  */
3803 int
3804 iwm_mvm_flush_tx_path(struct iwm_softc *sc, uint32_t tfd_msk, uint32_t flags)
3805 {
3806         int ret;
3807         struct iwm_tx_path_flush_cmd flush_cmd = {
3808                 .queues_ctl = htole32(tfd_msk),
3809                 .flush_ctl = htole16(IWM_DUMP_TX_FIFO_FLUSH),
3810         };
3811
3812         ret = iwm_mvm_send_cmd_pdu(sc, IWM_TXPATH_FLUSH, flags,
3813             sizeof(flush_cmd), &flush_cmd);
3814         if (ret)
3815                 device_printf(sc->sc_dev,
3816                     "Flushing tx queue failed: %d\n", ret);
3817         return ret;
3818 }
3819
3820 /*
3821  * BEGIN mvm/quota.c
3822  */
3823
3824 static int
3825 iwm_mvm_update_quotas(struct iwm_softc *sc, struct iwm_vap *ivp)
3826 {
3827         struct iwm_time_quota_cmd cmd;
3828         int i, idx, ret, num_active_macs, quota, quota_rem;
3829         int colors[IWM_MAX_BINDINGS] = { -1, -1, -1, -1, };
3830         int n_ifs[IWM_MAX_BINDINGS] = {0, };
3831         uint16_t id;
3832
3833         memset(&cmd, 0, sizeof(cmd));
3834
3835         /* currently, PHY ID == binding ID */
3836         if (ivp) {
3837                 id = ivp->phy_ctxt->id;
3838                 KASSERT(id < IWM_MAX_BINDINGS, ("invalid id"));
3839                 colors[id] = ivp->phy_ctxt->color;
3840
3841                 if (1)
3842                         n_ifs[id] = 1;
3843         }
3844
3845         /*
3846          * The FW's scheduling session consists of
3847          * IWM_MVM_MAX_QUOTA fragments. Divide these fragments
3848          * equally between all the bindings that require quota
3849          */
3850         num_active_macs = 0;
3851         for (i = 0; i < IWM_MAX_BINDINGS; i++) {
3852                 cmd.quotas[i].id_and_color = htole32(IWM_FW_CTXT_INVALID);
3853                 num_active_macs += n_ifs[i];
3854         }
3855
3856         quota = 0;
3857         quota_rem = 0;
3858         if (num_active_macs) {
3859                 quota = IWM_MVM_MAX_QUOTA / num_active_macs;
3860                 quota_rem = IWM_MVM_MAX_QUOTA % num_active_macs;
3861         }
3862
3863         for (idx = 0, i = 0; i < IWM_MAX_BINDINGS; i++) {
3864                 if (colors[i] < 0)
3865                         continue;
3866
3867                 cmd.quotas[idx].id_and_color =
3868                         htole32(IWM_FW_CMD_ID_AND_COLOR(i, colors[i]));
3869
3870                 if (n_ifs[i] <= 0) {
3871                         cmd.quotas[idx].quota = htole32(0);
3872                         cmd.quotas[idx].max_duration = htole32(0);
3873                 } else {
3874                         cmd.quotas[idx].quota = htole32(quota * n_ifs[i]);
3875                         cmd.quotas[idx].max_duration = htole32(0);
3876                 }
3877                 idx++;
3878         }
3879
3880         /* Give the remainder of the session to the first binding */
3881         cmd.quotas[0].quota = htole32(le32toh(cmd.quotas[0].quota) + quota_rem);
3882
3883         ret = iwm_mvm_send_cmd_pdu(sc, IWM_TIME_QUOTA_CMD, IWM_CMD_SYNC,
3884             sizeof(cmd), &cmd);
3885         if (ret)
3886                 device_printf(sc->sc_dev,
3887                     "%s: Failed to send quota: %d\n", __func__, ret);
3888         return ret;
3889 }
3890
3891 /*
3892  * END mvm/quota.c
3893  */
3894
3895 /*
3896  * ieee80211 routines
3897  */
3898
3899 /*
3900  * Change to AUTH state in 80211 state machine.  Roughly matches what
3901  * Linux does in bss_info_changed().
3902  */
3903 static int
3904 iwm_auth(struct ieee80211vap *vap, struct iwm_softc *sc)
3905 {
3906         struct ieee80211_node *ni;
3907         struct iwm_node *in;
3908         struct iwm_vap *iv = IWM_VAP(vap);
3909         uint32_t duration;
3910         int error;
3911
3912         /*
3913          * XXX i have a feeling that the vap node is being
3914          * freed from underneath us. Grr.
3915          */
3916         ni = ieee80211_ref_node(vap->iv_bss);
3917         in = IWM_NODE(ni);
3918         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_STATE,
3919             "%s: called; vap=%p, bss ni=%p\n",
3920             __func__,
3921             vap,
3922             ni);
3923         IWM_DPRINTF(sc, IWM_DEBUG_STATE, "%s: Current node bssid: %s\n",
3924             __func__, ether_sprintf(ni->ni_bssid));
3925
3926         in->in_assoc = 0;
3927         iv->iv_auth = 1;
3928
3929         /*
3930          * Firmware bug - it'll crash if the beacon interval is less
3931          * than 16. We can't avoid connecting at all, so refuse the
3932          * station state change, this will cause net80211 to abandon
3933          * attempts to connect to this AP, and eventually wpa_s will
3934          * blacklist the AP...
3935          */
3936         if (ni->ni_intval < 16) {
3937                 device_printf(sc->sc_dev,
3938                     "AP %s beacon interval is %d, refusing due to firmware bug!\n",
3939                     ether_sprintf(ni->ni_bssid), ni->ni_intval);
3940                 error = EINVAL;
3941                 goto out;
3942         }
3943
3944         error = iwm_allow_mcast(vap, sc);
3945         if (error) {
3946                 device_printf(sc->sc_dev,
3947                     "%s: failed to set multicast\n", __func__);
3948                 goto out;
3949         }
3950
3951         /*
3952          * This is where it deviates from what Linux does.
3953          *
3954          * Linux iwlwifi doesn't reset the nic each time, nor does it
3955          * call ctxt_add() here.  Instead, it adds it during vap creation,
3956          * and always does a mac_ctx_changed().
3957          *
3958          * The openbsd port doesn't attempt to do that - it reset things
3959          * at odd states and does the add here.
3960          *
3961          * So, until the state handling is fixed (ie, we never reset
3962          * the NIC except for a firmware failure, which should drag
3963          * the NIC back to IDLE, re-setup and re-add all the mac/phy
3964          * contexts that are required), let's do a dirty hack here.
3965          */
3966         if (iv->is_uploaded) {
3967                 if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) {
3968                         device_printf(sc->sc_dev,
3969                             "%s: failed to update MAC\n", __func__);
3970                         goto out;
3971                 }
3972         } else {
3973                 if ((error = iwm_mvm_mac_ctxt_add(sc, vap)) != 0) {
3974                         device_printf(sc->sc_dev,
3975                             "%s: failed to add MAC\n", __func__);
3976                         goto out;
3977                 }
3978         }
3979         sc->sc_firmware_state = 1;
3980
3981         if ((error = iwm_mvm_phy_ctxt_changed(sc, &sc->sc_phyctxt[0],
3982             in->in_ni.ni_chan, 1, 1)) != 0) {
3983                 device_printf(sc->sc_dev,
3984                     "%s: failed update phy ctxt\n", __func__);
3985                 goto out;
3986         }
3987         iv->phy_ctxt = &sc->sc_phyctxt[0];
3988
3989         if ((error = iwm_mvm_binding_add_vif(sc, iv)) != 0) {
3990                 device_printf(sc->sc_dev,
3991                     "%s: binding update cmd\n", __func__);
3992                 goto out;
3993         }
3994         sc->sc_firmware_state = 2;
3995         /*
3996          * Authentication becomes unreliable when powersaving is left enabled
3997          * here. Powersaving will be activated again when association has
3998          * finished or is aborted.
3999          */
4000         iv->ps_disabled = TRUE;
4001         error = iwm_mvm_power_update_mac(sc);
4002         iv->ps_disabled = FALSE;
4003         if (error != 0) {
4004                 device_printf(sc->sc_dev,
4005                     "%s: failed to update power management\n",
4006                     __func__);
4007                 goto out;
4008         }
4009         if ((error = iwm_mvm_add_sta(sc, in)) != 0) {
4010                 device_printf(sc->sc_dev,
4011                     "%s: failed to add sta\n", __func__);
4012                 goto out;
4013         }
4014         sc->sc_firmware_state = 3;
4015
4016         /*
4017          * Prevent the FW from wandering off channel during association
4018          * by "protecting" the session with a time event.
4019          */
4020         /* XXX duration is in units of TU, not MS */
4021         duration = IWM_MVM_TE_SESSION_PROTECTION_MAX_TIME_MS;
4022         iwm_mvm_protect_session(sc, iv, duration, 500 /* XXX magic number */, TRUE);
4023
4024         error = 0;
4025 out:
4026         if (error != 0)
4027                 iv->iv_auth = 0;
4028         ieee80211_free_node(ni);
4029         return (error);
4030 }
4031
4032 static struct ieee80211_node *
4033 iwm_node_alloc(struct ieee80211vap *vap, const uint8_t mac[IEEE80211_ADDR_LEN])
4034 {
4035         return malloc(sizeof (struct iwm_node), M_80211_NODE,
4036             M_NOWAIT | M_ZERO);
4037 }
4038
4039 static uint8_t
4040 iwm_rate_from_ucode_rate(uint32_t rate_n_flags)
4041 {
4042         uint8_t plcp = rate_n_flags & 0xff;
4043         int i;
4044
4045         for (i = 0; i <= IWM_RIDX_MAX; i++) {
4046                 if (iwm_rates[i].plcp == plcp)
4047                         return iwm_rates[i].rate;
4048         }
4049         return 0;
4050 }
4051
4052 uint8_t
4053 iwm_ridx2rate(struct ieee80211_rateset *rs, int ridx)
4054 {
4055         int i;
4056         uint8_t rval;
4057
4058         for (i = 0; i < rs->rs_nrates; i++) {
4059                 rval = (rs->rs_rates[i] & IEEE80211_RATE_VAL);
4060                 if (rval == iwm_rates[ridx].rate)
4061                         return rs->rs_rates[i];
4062         }
4063
4064         return 0;
4065 }
4066
4067 static int
4068 iwm_rate2ridx(struct iwm_softc *sc, uint8_t rate)
4069 {
4070         int i;
4071
4072         for (i = 0; i <= IWM_RIDX_MAX; i++) {
4073                 if (iwm_rates[i].rate == rate)
4074                         return i;
4075         }
4076
4077         device_printf(sc->sc_dev,
4078             "%s: WARNING: device rate for %u not found!\n",
4079             __func__, rate);
4080
4081         return -1;
4082 }
4083
4084
4085 static void
4086 iwm_setrates(struct iwm_softc *sc, struct iwm_node *in, int rix)
4087 {
4088         struct ieee80211_node *ni = &in->in_ni;
4089         struct iwm_lq_cmd *lq = &in->in_lq;
4090         struct ieee80211_rateset *rs = &ni->ni_rates;
4091         int nrates = rs->rs_nrates;
4092         int i, ridx, tab = 0;
4093 //      int txant = 0;
4094
4095         KASSERT(rix >= 0 && rix < nrates, ("invalid rix"));
4096
4097         if (nrates > nitems(lq->rs_table)) {
4098                 device_printf(sc->sc_dev,
4099                     "%s: node supports %d rates, driver handles "
4100                     "only %zu\n", __func__, nrates, nitems(lq->rs_table));
4101                 return;
4102         }
4103         if (nrates == 0) {
4104                 device_printf(sc->sc_dev,
4105                     "%s: node supports 0 rates, odd!\n", __func__);
4106                 return;
4107         }
4108         nrates = imin(rix + 1, nrates);
4109
4110         IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
4111             "%s: nrates=%d\n", __func__, nrates);
4112
4113         /* then construct a lq_cmd based on those */
4114         memset(lq, 0, sizeof(*lq));
4115         lq->sta_id = IWM_STATION_ID;
4116
4117         /* For HT, always enable RTS/CTS to avoid excessive retries. */
4118         if (ni->ni_flags & IEEE80211_NODE_HT)
4119                 lq->flags |= IWM_LQ_FLAG_USE_RTS_MSK;
4120
4121         /*
4122          * are these used? (we don't do SISO or MIMO)
4123          * need to set them to non-zero, though, or we get an error.
4124          */
4125         lq->single_stream_ant_msk = 1;
4126         lq->dual_stream_ant_msk = 1;
4127
4128         /*
4129          * Build the actual rate selection table.
4130          * The lowest bits are the rates.  Additionally,
4131          * CCK needs bit 9 to be set.  The rest of the bits
4132          * we add to the table select the tx antenna
4133          * Note that we add the rates in the highest rate first
4134          * (opposite of ni_rates).
4135          */
4136         for (i = 0; i < nrates; i++) {
4137                 int rate = rs->rs_rates[rix - i] & IEEE80211_RATE_VAL;
4138                 int nextant;
4139
4140                 /* Map 802.11 rate to HW rate index. */
4141                 ridx = iwm_rate2ridx(sc, rate);
4142                 if (ridx == -1)
4143                         continue;
4144
4145 #if 0
4146                 if (txant == 0)
4147                         txant = iwm_mvm_get_valid_tx_ant(sc);
4148                 nextant = 1<<(ffs(txant)-1);
4149                 txant &= ~nextant;
4150 #else
4151                 nextant = iwm_mvm_get_valid_tx_ant(sc);
4152 #endif
4153                 tab = iwm_rates[ridx].plcp;
4154                 tab |= nextant << IWM_RATE_MCS_ANT_POS;
4155                 if (IWM_RIDX_IS_CCK(ridx))
4156                         tab |= IWM_RATE_MCS_CCK_MSK;
4157                 IWM_DPRINTF(sc, IWM_DEBUG_TXRATE,
4158                     "station rate i=%d, rate=%d, hw=%x\n",
4159                     i, iwm_rates[ridx].rate, tab);
4160                 lq->rs_table[i] = htole32(tab);
4161         }
4162         /* then fill the rest with the lowest possible rate */
4163         for (i = nrates; i < nitems(lq->rs_table); i++) {
4164                 KASSERT(tab != 0, ("invalid tab"));
4165                 lq->rs_table[i] = htole32(tab);
4166         }
4167 }
4168
4169 static int
4170 iwm_media_change(struct ifnet *ifp)
4171 {
4172         struct ieee80211vap *vap = ifp->if_softc;
4173         struct ieee80211com *ic = vap->iv_ic;
4174         struct iwm_softc *sc = ic->ic_softc;
4175         int error;
4176
4177         error = ieee80211_media_change(ifp);
4178         if (error != ENETRESET)
4179                 return error;
4180
4181         IWM_LOCK(sc);
4182         if (ic->ic_nrunning > 0) {
4183                 iwm_stop(sc);
4184                 iwm_init(sc);
4185         }
4186         IWM_UNLOCK(sc);
4187         return error;
4188 }
4189
4190 static void
4191 iwm_bring_down_firmware(struct iwm_softc *sc, struct ieee80211vap *vap)
4192 {
4193         struct iwm_vap *ivp = IWM_VAP(vap);
4194         int error;
4195
4196         /* Avoid Tx watchdog triggering, when transfers get dropped here. */
4197         sc->sc_tx_timer = 0;
4198
4199         ivp->iv_auth = 0;
4200         if (sc->sc_firmware_state == 3) {
4201                 iwm_xmit_queue_drain(sc);
4202 //              iwm_mvm_flush_tx_path(sc, 0xf, IWM_CMD_SYNC);
4203                 error = iwm_mvm_rm_sta(sc, vap, TRUE);
4204                 if (error) {
4205                         device_printf(sc->sc_dev,
4206                             "%s: Failed to remove station: %d\n",
4207                             __func__, error);
4208                 }
4209         }
4210         if (sc->sc_firmware_state == 3) {
4211                 error = iwm_mvm_mac_ctxt_changed(sc, vap);
4212                 if (error) {
4213                         device_printf(sc->sc_dev,
4214                             "%s: Failed to change mac context: %d\n",
4215                             __func__, error);
4216                 }
4217         }
4218         if (sc->sc_firmware_state == 3) {
4219                 error = iwm_mvm_sf_update(sc, vap, FALSE);
4220                 if (error) {
4221                         device_printf(sc->sc_dev,
4222                             "%s: Failed to update smart FIFO: %d\n",
4223                             __func__, error);
4224                 }
4225         }
4226         if (sc->sc_firmware_state == 3) {
4227                 error = iwm_mvm_rm_sta_id(sc, vap);
4228                 if (error) {
4229                         device_printf(sc->sc_dev,
4230                             "%s: Failed to remove station id: %d\n",
4231                             __func__, error);
4232                 }
4233         }
4234         if (sc->sc_firmware_state == 3) {
4235                 error = iwm_mvm_update_quotas(sc, NULL);
4236                 if (error) {
4237                         device_printf(sc->sc_dev,
4238                             "%s: Failed to update PHY quota: %d\n",
4239                             __func__, error);
4240                 }
4241         }
4242         if (sc->sc_firmware_state == 3) {
4243                 /* XXX Might need to specify bssid correctly. */
4244                 error = iwm_mvm_mac_ctxt_changed(sc, vap);
4245                 if (error) {
4246                         device_printf(sc->sc_dev,
4247                             "%s: Failed to change mac context: %d\n",
4248                             __func__, error);
4249                 }
4250         }
4251         if (sc->sc_firmware_state == 3) {
4252                 sc->sc_firmware_state = 2;
4253         }
4254         if (sc->sc_firmware_state > 1) {
4255                 error = iwm_mvm_binding_remove_vif(sc, ivp);
4256                 if (error) {
4257                         device_printf(sc->sc_dev,
4258                             "%s: Failed to remove channel ctx: %d\n",
4259                             __func__, error);
4260                 }
4261         }
4262         if (sc->sc_firmware_state > 1) {
4263                 sc->sc_firmware_state = 1;
4264         }
4265         ivp->phy_ctxt = NULL;
4266         if (sc->sc_firmware_state > 0) {
4267                 error = iwm_mvm_mac_ctxt_changed(sc, vap);
4268                 if (error) {
4269                         device_printf(sc->sc_dev,
4270                             "%s: Failed to change mac context: %d\n",
4271                             __func__, error);
4272                 }
4273         }
4274         if (sc->sc_firmware_state > 0) {
4275                 error = iwm_mvm_power_update_mac(sc);
4276                 if (error != 0) {
4277                         device_printf(sc->sc_dev,
4278                             "%s: failed to update power management\n",
4279                             __func__);
4280                 }
4281         }
4282         sc->sc_firmware_state = 0;
4283 }
4284
4285 static int
4286 iwm_newstate(struct ieee80211vap *vap, enum ieee80211_state nstate, int arg)
4287 {
4288         struct iwm_vap *ivp = IWM_VAP(vap);
4289         struct ieee80211com *ic = vap->iv_ic;
4290         struct iwm_softc *sc = ic->ic_softc;
4291         struct iwm_node *in;
4292         int error;
4293
4294         IWM_DPRINTF(sc, IWM_DEBUG_STATE,
4295             "switching state %s -> %s arg=0x%x\n",
4296             ieee80211_state_name[vap->iv_state],
4297             ieee80211_state_name[nstate],
4298             arg);
4299
4300         IEEE80211_UNLOCK(ic);
4301         IWM_LOCK(sc);
4302
4303         if ((sc->sc_flags & IWM_FLAG_SCAN_RUNNING) &&
4304             (nstate == IEEE80211_S_AUTH ||
4305              nstate == IEEE80211_S_ASSOC ||
4306              nstate == IEEE80211_S_RUN)) {
4307                 /* Stop blinking for a scan, when authenticating. */
4308                 iwm_led_blink_stop(sc);
4309         }
4310
4311         if (vap->iv_state == IEEE80211_S_RUN && nstate != IEEE80211_S_RUN) {
4312                 iwm_mvm_led_disable(sc);
4313                 /* disable beacon filtering if we're hopping out of RUN */
4314                 iwm_mvm_disable_beacon_filter(sc);
4315                 if (((in = IWM_NODE(vap->iv_bss)) != NULL))
4316                         in->in_assoc = 0;
4317         }
4318
4319         if ((vap->iv_state == IEEE80211_S_AUTH ||
4320              vap->iv_state == IEEE80211_S_ASSOC ||
4321              vap->iv_state == IEEE80211_S_RUN) &&
4322             (nstate == IEEE80211_S_INIT ||
4323              nstate == IEEE80211_S_SCAN ||
4324              nstate == IEEE80211_S_AUTH)) {
4325                 iwm_mvm_stop_session_protection(sc, ivp);
4326         }
4327
4328         if ((vap->iv_state == IEEE80211_S_RUN ||
4329              vap->iv_state == IEEE80211_S_ASSOC) &&
4330             nstate == IEEE80211_S_INIT) {
4331                 /*
4332                  * In this case, iv_newstate() wants to send an 80211 frame on
4333                  * the network that we are leaving. So we need to call it,
4334                  * before tearing down all the firmware state.
4335                  */
4336                 IWM_UNLOCK(sc);
4337                 IEEE80211_LOCK(ic);
4338                 ivp->iv_newstate(vap, nstate, arg);
4339                 IEEE80211_UNLOCK(ic);
4340                 IWM_LOCK(sc);
4341                 iwm_bring_down_firmware(sc, vap);
4342                 IWM_UNLOCK(sc);
4343                 IEEE80211_LOCK(ic);
4344                 return 0;
4345         }
4346
4347         switch (nstate) {
4348         case IEEE80211_S_INIT:
4349         case IEEE80211_S_SCAN:
4350                 break;
4351
4352         case IEEE80211_S_AUTH:
4353                 iwm_bring_down_firmware(sc, vap);
4354                 if ((error = iwm_auth(vap, sc)) != 0) {
4355                         device_printf(sc->sc_dev,
4356                             "%s: could not move to auth state: %d\n",
4357                             __func__, error);
4358                         iwm_bring_down_firmware(sc, vap);
4359                         IWM_UNLOCK(sc);
4360                         IEEE80211_LOCK(ic);
4361                         return 1;
4362                 }
4363                 break;
4364
4365         case IEEE80211_S_ASSOC:
4366                 /*
4367                  * EBS may be disabled due to previous failures reported by FW.
4368                  * Reset EBS status here assuming environment has been changed.
4369                  */
4370                 sc->last_ebs_successful = TRUE;
4371                 break;
4372
4373         case IEEE80211_S_RUN:
4374                 in = IWM_NODE(vap->iv_bss);
4375                 /* Update the association state, now we have it all */
4376                 /* (eg associd comes in at this point */
4377                 error = iwm_mvm_update_sta(sc, in);
4378                 if (error != 0) {
4379                         device_printf(sc->sc_dev,
4380                             "%s: failed to update STA\n", __func__);
4381                         IWM_UNLOCK(sc);
4382                         IEEE80211_LOCK(ic);
4383                         return error;
4384                 }
4385                 in->in_assoc = 1;
4386                 error = iwm_mvm_mac_ctxt_changed(sc, vap);
4387                 if (error != 0) {
4388                         device_printf(sc->sc_dev,
4389                             "%s: failed to update MAC: %d\n", __func__, error);
4390                 }
4391
4392                 iwm_mvm_sf_update(sc, vap, FALSE);
4393                 iwm_mvm_enable_beacon_filter(sc, ivp);
4394                 iwm_mvm_power_update_mac(sc);
4395                 iwm_mvm_update_quotas(sc, ivp);
4396                 int rix = ieee80211_ratectl_rate(&in->in_ni, NULL, 0);
4397                 iwm_setrates(sc, in, rix);
4398
4399                 if ((error = iwm_mvm_send_lq_cmd(sc, &in->in_lq, TRUE)) != 0) {
4400                         device_printf(sc->sc_dev,
4401                             "%s: IWM_LQ_CMD failed: %d\n", __func__, error);
4402                 }
4403
4404                 iwm_mvm_led_enable(sc);
4405                 break;
4406
4407         default:
4408                 break;
4409         }
4410         IWM_UNLOCK(sc);
4411         IEEE80211_LOCK(ic);
4412
4413         return (ivp->iv_newstate(vap, nstate, arg));
4414 }
4415
4416 void
4417 iwm_endscan_cb(void *arg, int pending)
4418 {
4419         struct iwm_softc *sc = arg;
4420         struct ieee80211com *ic = &sc->sc_ic;
4421
4422         IWM_DPRINTF(sc, IWM_DEBUG_SCAN | IWM_DEBUG_TRACE,
4423             "%s: scan ended\n",
4424             __func__);
4425
4426         ieee80211_scan_done(TAILQ_FIRST(&ic->ic_vaps));
4427 }
4428
4429 static int
4430 iwm_send_bt_init_conf(struct iwm_softc *sc)
4431 {
4432         struct iwm_bt_coex_cmd bt_cmd;
4433
4434         bt_cmd.mode = htole32(IWM_BT_COEX_WIFI);
4435         bt_cmd.enabled_modules = htole32(IWM_BT_COEX_HIGH_BAND_RET);
4436
4437         return iwm_mvm_send_cmd_pdu(sc, IWM_BT_CONFIG, 0, sizeof(bt_cmd),
4438             &bt_cmd);
4439 }
4440
4441 static boolean_t
4442 iwm_mvm_is_lar_supported(struct iwm_softc *sc)
4443 {
4444         boolean_t nvm_lar = sc->nvm_data->lar_enabled;
4445         boolean_t tlv_lar = fw_has_capa(&sc->sc_fw.ucode_capa,
4446                                         IWM_UCODE_TLV_CAPA_LAR_SUPPORT);
4447
4448         if (iwm_lar_disable)
4449                 return FALSE;
4450
4451         /*
4452          * Enable LAR only if it is supported by the FW (TLV) &&
4453          * enabled in the NVM
4454          */
4455         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000)
4456                 return nvm_lar && tlv_lar;
4457         else
4458                 return tlv_lar;
4459 }
4460
4461 static boolean_t
4462 iwm_mvm_is_wifi_mcc_supported(struct iwm_softc *sc)
4463 {
4464         return fw_has_api(&sc->sc_fw.ucode_capa,
4465                           IWM_UCODE_TLV_API_WIFI_MCC_UPDATE) ||
4466                fw_has_capa(&sc->sc_fw.ucode_capa,
4467                            IWM_UCODE_TLV_CAPA_LAR_MULTI_MCC);
4468 }
4469
4470 static int
4471 iwm_send_update_mcc_cmd(struct iwm_softc *sc, const char *alpha2)
4472 {
4473         struct iwm_mcc_update_cmd mcc_cmd;
4474         struct iwm_host_cmd hcmd = {
4475                 .id = IWM_MCC_UPDATE_CMD,
4476                 .flags = (IWM_CMD_SYNC | IWM_CMD_WANT_SKB),
4477                 .data = { &mcc_cmd },
4478         };
4479         int ret;
4480 #ifdef IWM_DEBUG
4481         struct iwm_rx_packet *pkt;
4482         struct iwm_mcc_update_resp_v1 *mcc_resp_v1 = NULL;
4483         struct iwm_mcc_update_resp *mcc_resp;
4484         int n_channels;
4485         uint16_t mcc;
4486 #endif
4487         int resp_v2 = fw_has_capa(&sc->sc_fw.ucode_capa,
4488             IWM_UCODE_TLV_CAPA_LAR_SUPPORT_V2);
4489
4490         if (!iwm_mvm_is_lar_supported(sc)) {
4491                 IWM_DPRINTF(sc, IWM_DEBUG_LAR, "%s: no LAR support\n",
4492                     __func__);
4493                 return 0;
4494         }
4495
4496         memset(&mcc_cmd, 0, sizeof(mcc_cmd));
4497         mcc_cmd.mcc = htole16(alpha2[0] << 8 | alpha2[1]);
4498         if (iwm_mvm_is_wifi_mcc_supported(sc))
4499                 mcc_cmd.source_id = IWM_MCC_SOURCE_GET_CURRENT;
4500         else
4501                 mcc_cmd.source_id = IWM_MCC_SOURCE_OLD_FW;
4502
4503         if (resp_v2)
4504                 hcmd.len[0] = sizeof(struct iwm_mcc_update_cmd);
4505         else
4506                 hcmd.len[0] = sizeof(struct iwm_mcc_update_cmd_v1);
4507
4508         IWM_DPRINTF(sc, IWM_DEBUG_LAR,
4509             "send MCC update to FW with '%c%c' src = %d\n",
4510             alpha2[0], alpha2[1], mcc_cmd.source_id);
4511
4512         ret = iwm_send_cmd(sc, &hcmd);
4513         if (ret)
4514                 return ret;
4515
4516 #ifdef IWM_DEBUG
4517         pkt = hcmd.resp_pkt;
4518
4519         /* Extract MCC response */
4520         if (resp_v2) {
4521                 mcc_resp = (void *)pkt->data;
4522                 mcc = mcc_resp->mcc;
4523                 n_channels =  le32toh(mcc_resp->n_channels);
4524         } else {
4525                 mcc_resp_v1 = (void *)pkt->data;
4526                 mcc = mcc_resp_v1->mcc;
4527                 n_channels =  le32toh(mcc_resp_v1->n_channels);
4528         }
4529
4530         /* W/A for a FW/NVM issue - returns 0x00 for the world domain */
4531         if (mcc == 0)
4532                 mcc = 0x3030;  /* "00" - world */
4533
4534         IWM_DPRINTF(sc, IWM_DEBUG_LAR,
4535             "regulatory domain '%c%c' (%d channels available)\n",
4536             mcc >> 8, mcc & 0xff, n_channels);
4537 #endif
4538         iwm_free_resp(sc, &hcmd);
4539
4540         return 0;
4541 }
4542
4543 static void
4544 iwm_mvm_tt_tx_backoff(struct iwm_softc *sc, uint32_t backoff)
4545 {
4546         struct iwm_host_cmd cmd = {
4547                 .id = IWM_REPLY_THERMAL_MNG_BACKOFF,
4548                 .len = { sizeof(uint32_t), },
4549                 .data = { &backoff, },
4550         };
4551
4552         if (iwm_send_cmd(sc, &cmd) != 0) {
4553                 device_printf(sc->sc_dev,
4554                     "failed to change thermal tx backoff\n");
4555         }
4556 }
4557
4558 static int
4559 iwm_init_hw(struct iwm_softc *sc)
4560 {
4561         struct ieee80211com *ic = &sc->sc_ic;
4562         int error, i, ac;
4563
4564         sc->sf_state = IWM_SF_UNINIT;
4565
4566         if ((error = iwm_start_hw(sc)) != 0) {
4567                 printf("iwm_start_hw: failed %d\n", error);
4568                 return error;
4569         }
4570
4571         if ((error = iwm_run_init_mvm_ucode(sc, 0)) != 0) {
4572                 printf("iwm_run_init_mvm_ucode: failed %d\n", error);
4573                 return error;
4574         }
4575
4576         /*
4577          * should stop and start HW since that INIT
4578          * image just loaded
4579          */
4580         iwm_stop_device(sc);
4581         sc->sc_ps_disabled = FALSE;
4582         if ((error = iwm_start_hw(sc)) != 0) {
4583                 device_printf(sc->sc_dev, "could not initialize hardware\n");
4584                 return error;
4585         }
4586
4587         /* omstart, this time with the regular firmware */
4588         error = iwm_mvm_load_ucode_wait_alive(sc, IWM_UCODE_REGULAR);
4589         if (error) {
4590                 device_printf(sc->sc_dev, "could not load firmware\n");
4591                 goto error;
4592         }
4593
4594         error = iwm_mvm_sf_update(sc, NULL, FALSE);
4595         if (error)
4596                 device_printf(sc->sc_dev, "Failed to initialize Smart Fifo\n");
4597
4598         if ((error = iwm_send_bt_init_conf(sc)) != 0) {
4599                 device_printf(sc->sc_dev, "bt init conf failed\n");
4600                 goto error;
4601         }
4602
4603         error = iwm_send_tx_ant_cfg(sc, iwm_mvm_get_valid_tx_ant(sc));
4604         if (error != 0) {
4605                 device_printf(sc->sc_dev, "antenna config failed\n");
4606                 goto error;
4607         }
4608
4609         /* Send phy db control command and then phy db calibration */
4610         if ((error = iwm_send_phy_db_data(sc->sc_phy_db)) != 0)
4611                 goto error;
4612
4613         if ((error = iwm_send_phy_cfg_cmd(sc)) != 0) {
4614                 device_printf(sc->sc_dev, "phy_cfg_cmd failed\n");
4615                 goto error;
4616         }
4617
4618         /* Add auxiliary station for scanning */
4619         if ((error = iwm_mvm_add_aux_sta(sc)) != 0) {
4620                 device_printf(sc->sc_dev, "add_aux_sta failed\n");
4621                 goto error;
4622         }
4623
4624         for (i = 0; i < IWM_NUM_PHY_CTX; i++) {
4625                 /*
4626                  * The channel used here isn't relevant as it's
4627                  * going to be overwritten in the other flows.
4628                  * For now use the first channel we have.
4629                  */
4630                 if ((error = iwm_mvm_phy_ctxt_add(sc,
4631                     &sc->sc_phyctxt[i], &ic->ic_channels[1], 1, 1)) != 0)
4632                         goto error;
4633         }
4634
4635         /* Initialize tx backoffs to the minimum. */
4636         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_7000)
4637                 iwm_mvm_tt_tx_backoff(sc, 0);
4638
4639         if (iwm_mvm_config_ltr(sc) != 0)
4640                 device_printf(sc->sc_dev, "PCIe LTR configuration failed\n");
4641
4642         error = iwm_mvm_power_update_device(sc);
4643         if (error)
4644                 goto error;
4645
4646         if ((error = iwm_send_update_mcc_cmd(sc, "ZZ")) != 0)
4647                 goto error;
4648
4649         if (fw_has_capa(&sc->sc_fw.ucode_capa, IWM_UCODE_TLV_CAPA_UMAC_SCAN)) {
4650                 if ((error = iwm_mvm_config_umac_scan(sc)) != 0)
4651                         goto error;
4652         }
4653
4654         /* Enable Tx queues. */
4655         for (ac = 0; ac < WME_NUM_AC; ac++) {
4656                 error = iwm_enable_txq(sc, IWM_STATION_ID, ac,
4657                     iwm_mvm_ac_to_tx_fifo[ac]);
4658                 if (error)
4659                         goto error;
4660         }
4661
4662         if ((error = iwm_mvm_disable_beacon_filter(sc)) != 0) {
4663                 device_printf(sc->sc_dev, "failed to disable beacon filter\n");
4664                 goto error;
4665         }
4666
4667         return 0;
4668
4669  error:
4670         iwm_stop_device(sc);
4671         return error;
4672 }
4673
4674 /* Allow multicast from our BSSID. */
4675 static int
4676 iwm_allow_mcast(struct ieee80211vap *vap, struct iwm_softc *sc)
4677 {
4678         struct ieee80211_node *ni = vap->iv_bss;
4679         struct iwm_mcast_filter_cmd *cmd;
4680         size_t size;
4681         int error;
4682
4683         size = roundup(sizeof(*cmd), 4);
4684         cmd = malloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
4685         if (cmd == NULL)
4686                 return ENOMEM;
4687         cmd->filter_own = 1;
4688         cmd->port_id = 0;
4689         cmd->count = 0;
4690         cmd->pass_all = 1;
4691         IEEE80211_ADDR_COPY(cmd->bssid, ni->ni_bssid);
4692
4693         error = iwm_mvm_send_cmd_pdu(sc, IWM_MCAST_FILTER_CMD,
4694             IWM_CMD_SYNC, size, cmd);
4695         free(cmd, M_DEVBUF);
4696
4697         return (error);
4698 }
4699
4700 /*
4701  * ifnet interfaces
4702  */
4703
4704 static void
4705 iwm_init(struct iwm_softc *sc)
4706 {
4707         int error;
4708
4709         if (sc->sc_flags & IWM_FLAG_HW_INITED) {
4710                 return;
4711         }
4712         sc->sc_generation++;
4713         sc->sc_flags &= ~IWM_FLAG_STOPPED;
4714
4715         if ((error = iwm_init_hw(sc)) != 0) {
4716                 printf("iwm_init_hw failed %d\n", error);
4717                 iwm_stop(sc);
4718                 return;
4719         }
4720
4721         /*
4722          * Ok, firmware loaded and we are jogging
4723          */
4724         sc->sc_flags |= IWM_FLAG_HW_INITED;
4725 }
4726
4727 static int
4728 iwm_transmit(struct ieee80211com *ic, struct mbuf *m)
4729 {
4730         struct iwm_softc *sc;
4731         int error;
4732
4733         sc = ic->ic_softc;
4734
4735         IWM_LOCK(sc);
4736         if ((sc->sc_flags & IWM_FLAG_HW_INITED) == 0) {
4737                 IWM_UNLOCK(sc);
4738                 return (ENXIO);
4739         }
4740         error = mbufq_enqueue(&sc->sc_snd, m);
4741         if (error) {
4742                 IWM_UNLOCK(sc);
4743                 return (error);
4744         }
4745         iwm_start(sc);
4746         IWM_UNLOCK(sc);
4747         return (0);
4748 }
4749
4750 /*
4751  * Dequeue packets from sendq and call send.
4752  */
4753 static void
4754 iwm_start(struct iwm_softc *sc)
4755 {
4756         struct ieee80211_node *ni;
4757         struct mbuf *m;
4758         int ac = 0;
4759
4760         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TRACE, "->%s\n", __func__);
4761         while (sc->qfullmsk == 0 &&
4762                 (m = mbufq_dequeue(&sc->sc_snd)) != NULL) {
4763                 ni = (struct ieee80211_node *)m->m_pkthdr.rcvif;
4764                 if (iwm_tx(sc, m, ni, ac) != 0) {
4765                         if_inc_counter(ni->ni_vap->iv_ifp,
4766                             IFCOUNTER_OERRORS, 1);
4767                         ieee80211_free_node(ni);
4768                         continue;
4769                 }
4770                 if (sc->sc_tx_timer == 0) {
4771                         callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog,
4772                             sc);
4773                 }
4774                 sc->sc_tx_timer = 15;
4775         }
4776         IWM_DPRINTF(sc, IWM_DEBUG_XMIT | IWM_DEBUG_TRACE, "<-%s\n", __func__);
4777 }
4778
4779 static void
4780 iwm_stop(struct iwm_softc *sc)
4781 {
4782
4783         sc->sc_flags &= ~IWM_FLAG_HW_INITED;
4784         sc->sc_flags |= IWM_FLAG_STOPPED;
4785         sc->sc_generation++;
4786         iwm_led_blink_stop(sc);
4787         sc->sc_tx_timer = 0;
4788         iwm_stop_device(sc);
4789         sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
4790 }
4791
4792 static void
4793 iwm_watchdog(void *arg)
4794 {
4795         struct iwm_softc *sc = arg;
4796         struct ieee80211com *ic = &sc->sc_ic;
4797
4798         if (sc->sc_attached == 0)
4799                 return;
4800
4801         if (sc->sc_tx_timer > 0) {
4802                 if (--sc->sc_tx_timer == 0) {
4803                         device_printf(sc->sc_dev, "device timeout\n");
4804 #ifdef IWM_DEBUG
4805                         iwm_nic_error(sc);
4806 #endif
4807                         ieee80211_restart_all(ic);
4808                         counter_u64_add(sc->sc_ic.ic_oerrors, 1);
4809                         return;
4810                 }
4811                 callout_reset(&sc->sc_watchdog_to, hz, iwm_watchdog, sc);
4812         }
4813 }
4814
4815 static void
4816 iwm_parent(struct ieee80211com *ic)
4817 {
4818         struct iwm_softc *sc = ic->ic_softc;
4819         int startall = 0;
4820
4821         IWM_LOCK(sc);
4822         if (ic->ic_nrunning > 0) {
4823                 if (!(sc->sc_flags & IWM_FLAG_HW_INITED)) {
4824                         iwm_init(sc);
4825                         startall = 1;
4826                 }
4827         } else if (sc->sc_flags & IWM_FLAG_HW_INITED)
4828                 iwm_stop(sc);
4829         IWM_UNLOCK(sc);
4830         if (startall)
4831                 ieee80211_start_all(ic);
4832 }
4833
4834 /*
4835  * The interrupt side of things
4836  */
4837
4838 /*
4839  * error dumping routines are from iwlwifi/mvm/utils.c
4840  */
4841
4842 /*
4843  * Note: This structure is read from the device with IO accesses,
4844  * and the reading already does the endian conversion. As it is
4845  * read with uint32_t-sized accesses, any members with a different size
4846  * need to be ordered correctly though!
4847  */
4848 struct iwm_error_event_table {
4849         uint32_t valid;         /* (nonzero) valid, (0) log is empty */
4850         uint32_t error_id;              /* type of error */
4851         uint32_t trm_hw_status0;        /* TRM HW status */
4852         uint32_t trm_hw_status1;        /* TRM HW status */
4853         uint32_t blink2;                /* branch link */
4854         uint32_t ilink1;                /* interrupt link */
4855         uint32_t ilink2;                /* interrupt link */
4856         uint32_t data1;         /* error-specific data */
4857         uint32_t data2;         /* error-specific data */
4858         uint32_t data3;         /* error-specific data */
4859         uint32_t bcon_time;             /* beacon timer */
4860         uint32_t tsf_low;               /* network timestamp function timer */
4861         uint32_t tsf_hi;                /* network timestamp function timer */
4862         uint32_t gp1;           /* GP1 timer register */
4863         uint32_t gp2;           /* GP2 timer register */
4864         uint32_t fw_rev_type;   /* firmware revision type */
4865         uint32_t major;         /* uCode version major */
4866         uint32_t minor;         /* uCode version minor */
4867         uint32_t hw_ver;                /* HW Silicon version */
4868         uint32_t brd_ver;               /* HW board version */
4869         uint32_t log_pc;                /* log program counter */
4870         uint32_t frame_ptr;             /* frame pointer */
4871         uint32_t stack_ptr;             /* stack pointer */
4872         uint32_t hcmd;          /* last host command header */
4873         uint32_t isr0;          /* isr status register LMPM_NIC_ISR0:
4874                                  * rxtx_flag */
4875         uint32_t isr1;          /* isr status register LMPM_NIC_ISR1:
4876                                  * host_flag */
4877         uint32_t isr2;          /* isr status register LMPM_NIC_ISR2:
4878                                  * enc_flag */
4879         uint32_t isr3;          /* isr status register LMPM_NIC_ISR3:
4880                                  * time_flag */
4881         uint32_t isr4;          /* isr status register LMPM_NIC_ISR4:
4882                                  * wico interrupt */
4883         uint32_t last_cmd_id;   /* last HCMD id handled by the firmware */
4884         uint32_t wait_event;            /* wait event() caller address */
4885         uint32_t l2p_control;   /* L2pControlField */
4886         uint32_t l2p_duration;  /* L2pDurationField */
4887         uint32_t l2p_mhvalid;   /* L2pMhValidBits */
4888         uint32_t l2p_addr_match;        /* L2pAddrMatchStat */
4889         uint32_t lmpm_pmg_sel;  /* indicate which clocks are turned on
4890                                  * (LMPM_PMG_SEL) */
4891         uint32_t u_timestamp;   /* indicate when the date and time of the
4892                                  * compilation */
4893         uint32_t flow_handler;  /* FH read/write pointers, RX credit */
4894 } __packed /* LOG_ERROR_TABLE_API_S_VER_3 */;
4895
4896 /*
4897  * UMAC error struct - relevant starting from family 8000 chip.
4898  * Note: This structure is read from the device with IO accesses,
4899  * and the reading already does the endian conversion. As it is
4900  * read with u32-sized accesses, any members with a different size
4901  * need to be ordered correctly though!
4902  */
4903 struct iwm_umac_error_event_table {
4904         uint32_t valid;         /* (nonzero) valid, (0) log is empty */
4905         uint32_t error_id;      /* type of error */
4906         uint32_t blink1;        /* branch link */
4907         uint32_t blink2;        /* branch link */
4908         uint32_t ilink1;        /* interrupt link */
4909         uint32_t ilink2;        /* interrupt link */
4910         uint32_t data1;         /* error-specific data */
4911         uint32_t data2;         /* error-specific data */
4912         uint32_t data3;         /* error-specific data */
4913         uint32_t umac_major;
4914         uint32_t umac_minor;
4915         uint32_t frame_pointer; /* core register 27*/
4916         uint32_t stack_pointer; /* core register 28 */
4917         uint32_t cmd_header;    /* latest host cmd sent to UMAC */
4918         uint32_t nic_isr_pref;  /* ISR status register */
4919 } __packed;
4920
4921 #define ERROR_START_OFFSET  (1 * sizeof(uint32_t))
4922 #define ERROR_ELEM_SIZE     (7 * sizeof(uint32_t))
4923
4924 #ifdef IWM_DEBUG
4925 struct {
4926         const char *name;
4927         uint8_t num;
4928 } advanced_lookup[] = {
4929         { "NMI_INTERRUPT_WDG", 0x34 },
4930         { "SYSASSERT", 0x35 },
4931         { "UCODE_VERSION_MISMATCH", 0x37 },
4932         { "BAD_COMMAND", 0x38 },
4933         { "NMI_INTERRUPT_DATA_ACTION_PT", 0x3C },
4934         { "FATAL_ERROR", 0x3D },
4935         { "NMI_TRM_HW_ERR", 0x46 },
4936         { "NMI_INTERRUPT_TRM", 0x4C },
4937         { "NMI_INTERRUPT_BREAK_POINT", 0x54 },
4938         { "NMI_INTERRUPT_WDG_RXF_FULL", 0x5C },
4939         { "NMI_INTERRUPT_WDG_NO_RBD_RXF_FULL", 0x64 },
4940         { "NMI_INTERRUPT_HOST", 0x66 },
4941         { "NMI_INTERRUPT_ACTION_PT", 0x7C },
4942         { "NMI_INTERRUPT_UNKNOWN", 0x84 },
4943         { "NMI_INTERRUPT_INST_ACTION_PT", 0x86 },
4944         { "ADVANCED_SYSASSERT", 0 },
4945 };
4946
4947 static const char *
4948 iwm_desc_lookup(uint32_t num)
4949 {
4950         int i;
4951
4952         for (i = 0; i < nitems(advanced_lookup) - 1; i++)
4953                 if (advanced_lookup[i].num == num)
4954                         return advanced_lookup[i].name;
4955
4956         /* No entry matches 'num', so it is the last: ADVANCED_SYSASSERT */
4957         return advanced_lookup[i].name;
4958 }
4959
4960 static void
4961 iwm_nic_umac_error(struct iwm_softc *sc)
4962 {
4963         struct iwm_umac_error_event_table table;
4964         uint32_t base;
4965
4966         base = sc->umac_error_event_table;
4967
4968         if (base < 0x800000) {
4969                 device_printf(sc->sc_dev, "Invalid error log pointer 0x%08x\n",
4970                     base);
4971                 return;
4972         }
4973
4974         if (iwm_read_mem(sc, base, &table, sizeof(table)/sizeof(uint32_t))) {
4975                 device_printf(sc->sc_dev, "reading errlog failed\n");
4976                 return;
4977         }
4978
4979         if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) {
4980                 device_printf(sc->sc_dev, "Start UMAC Error Log Dump:\n");
4981                 device_printf(sc->sc_dev, "Status: 0x%x, count: %d\n",
4982                     sc->sc_flags, table.valid);
4983         }
4984
4985         device_printf(sc->sc_dev, "0x%08X | %s\n", table.error_id,
4986                 iwm_desc_lookup(table.error_id));
4987         device_printf(sc->sc_dev, "0x%08X | umac branchlink1\n", table.blink1);
4988         device_printf(sc->sc_dev, "0x%08X | umac branchlink2\n", table.blink2);
4989         device_printf(sc->sc_dev, "0x%08X | umac interruptlink1\n",
4990             table.ilink1);
4991         device_printf(sc->sc_dev, "0x%08X | umac interruptlink2\n",
4992             table.ilink2);
4993         device_printf(sc->sc_dev, "0x%08X | umac data1\n", table.data1);
4994         device_printf(sc->sc_dev, "0x%08X | umac data2\n", table.data2);
4995         device_printf(sc->sc_dev, "0x%08X | umac data3\n", table.data3);
4996         device_printf(sc->sc_dev, "0x%08X | umac major\n", table.umac_major);
4997         device_printf(sc->sc_dev, "0x%08X | umac minor\n", table.umac_minor);
4998         device_printf(sc->sc_dev, "0x%08X | frame pointer\n",
4999             table.frame_pointer);
5000         device_printf(sc->sc_dev, "0x%08X | stack pointer\n",
5001             table.stack_pointer);
5002         device_printf(sc->sc_dev, "0x%08X | last host cmd\n", table.cmd_header);
5003         device_printf(sc->sc_dev, "0x%08X | isr status reg\n",
5004             table.nic_isr_pref);
5005 }
5006
5007 /*
5008  * Support for dumping the error log seemed like a good idea ...
5009  * but it's mostly hex junk and the only sensible thing is the
5010  * hw/ucode revision (which we know anyway).  Since it's here,
5011  * I'll just leave it in, just in case e.g. the Intel guys want to
5012  * help us decipher some "ADVANCED_SYSASSERT" later.
5013  */
5014 static void
5015 iwm_nic_error(struct iwm_softc *sc)
5016 {
5017         struct iwm_error_event_table table;
5018         uint32_t base;
5019
5020         device_printf(sc->sc_dev, "dumping device error log\n");
5021         base = sc->error_event_table[0];
5022         if (base < 0x800000) {
5023                 device_printf(sc->sc_dev,
5024                     "Invalid error log pointer 0x%08x\n", base);
5025                 return;
5026         }
5027
5028         if (iwm_read_mem(sc, base, &table, sizeof(table)/sizeof(uint32_t))) {
5029                 device_printf(sc->sc_dev, "reading errlog failed\n");
5030                 return;
5031         }
5032
5033         if (!table.valid) {
5034                 device_printf(sc->sc_dev, "errlog not found, skipping\n");
5035                 return;
5036         }
5037
5038         if (ERROR_START_OFFSET <= table.valid * ERROR_ELEM_SIZE) {
5039                 device_printf(sc->sc_dev, "Start Error Log Dump:\n");
5040                 device_printf(sc->sc_dev, "Status: 0x%x, count: %d\n",
5041                     sc->sc_flags, table.valid);
5042         }
5043
5044         device_printf(sc->sc_dev, "0x%08X | %-28s\n", table.error_id,
5045             iwm_desc_lookup(table.error_id));
5046         device_printf(sc->sc_dev, "%08X | trm_hw_status0\n",
5047             table.trm_hw_status0);
5048         device_printf(sc->sc_dev, "%08X | trm_hw_status1\n",
5049             table.trm_hw_status1);
5050         device_printf(sc->sc_dev, "%08X | branchlink2\n", table.blink2);
5051         device_printf(sc->sc_dev, "%08X | interruptlink1\n", table.ilink1);
5052         device_printf(sc->sc_dev, "%08X | interruptlink2\n", table.ilink2);
5053         device_printf(sc->sc_dev, "%08X | data1\n", table.data1);
5054         device_printf(sc->sc_dev, "%08X | data2\n", table.data2);
5055         device_printf(sc->sc_dev, "%08X | data3\n", table.data3);
5056         device_printf(sc->sc_dev, "%08X | beacon time\n", table.bcon_time);
5057         device_printf(sc->sc_dev, "%08X | tsf low\n", table.tsf_low);
5058         device_printf(sc->sc_dev, "%08X | tsf hi\n", table.tsf_hi);
5059         device_printf(sc->sc_dev, "%08X | time gp1\n", table.gp1);
5060         device_printf(sc->sc_dev, "%08X | time gp2\n", table.gp2);
5061         device_printf(sc->sc_dev, "%08X | uCode revision type\n",
5062             table.fw_rev_type);
5063         device_printf(sc->sc_dev, "%08X | uCode version major\n", table.major);
5064         device_printf(sc->sc_dev, "%08X | uCode version minor\n", table.minor);
5065         device_printf(sc->sc_dev, "%08X | hw version\n", table.hw_ver);
5066         device_printf(sc->sc_dev, "%08X | board version\n", table.brd_ver);
5067         device_printf(sc->sc_dev, "%08X | hcmd\n", table.hcmd);
5068         device_printf(sc->sc_dev, "%08X | isr0\n", table.isr0);
5069         device_printf(sc->sc_dev, "%08X | isr1\n", table.isr1);
5070         device_printf(sc->sc_dev, "%08X | isr2\n", table.isr2);
5071         device_printf(sc->sc_dev, "%08X | isr3\n", table.isr3);
5072         device_printf(sc->sc_dev, "%08X | isr4\n", table.isr4);
5073         device_printf(sc->sc_dev, "%08X | last cmd Id\n", table.last_cmd_id);
5074         device_printf(sc->sc_dev, "%08X | wait_event\n", table.wait_event);
5075         device_printf(sc->sc_dev, "%08X | l2p_control\n", table.l2p_control);
5076         device_printf(sc->sc_dev, "%08X | l2p_duration\n", table.l2p_duration);
5077         device_printf(sc->sc_dev, "%08X | l2p_mhvalid\n", table.l2p_mhvalid);
5078         device_printf(sc->sc_dev, "%08X | l2p_addr_match\n", table.l2p_addr_match);
5079         device_printf(sc->sc_dev, "%08X | lmpm_pmg_sel\n", table.lmpm_pmg_sel);
5080         device_printf(sc->sc_dev, "%08X | timestamp\n", table.u_timestamp);
5081         device_printf(sc->sc_dev, "%08X | flow_handler\n", table.flow_handler);
5082
5083         if (sc->umac_error_event_table)
5084                 iwm_nic_umac_error(sc);
5085 }
5086 #endif
5087
5088 static void
5089 iwm_handle_rxb(struct iwm_softc *sc, struct mbuf *m)
5090 {
5091         struct ieee80211com *ic = &sc->sc_ic;
5092         struct iwm_cmd_response *cresp;
5093         struct mbuf *m1;
5094         uint32_t offset = 0;
5095         uint32_t maxoff = IWM_RBUF_SIZE;
5096         uint32_t nextoff;
5097         boolean_t stolen = FALSE;
5098
5099 #define HAVEROOM(a)     \
5100     ((a) + sizeof(uint32_t) + sizeof(struct iwm_cmd_header) < maxoff)
5101
5102         while (HAVEROOM(offset)) {
5103                 struct iwm_rx_packet *pkt = mtodoff(m, struct iwm_rx_packet *,
5104                     offset);
5105                 int qid, idx, code, len;
5106
5107                 qid = pkt->hdr.qid;
5108                 idx = pkt->hdr.idx;
5109
5110                 code = IWM_WIDE_ID(pkt->hdr.flags, pkt->hdr.code);
5111
5112                 /*
5113                  * randomly get these from the firmware, no idea why.
5114                  * they at least seem harmless, so just ignore them for now
5115                  */
5116                 if ((pkt->hdr.code == 0 && (qid & ~0x80) == 0 && idx == 0) ||
5117                     pkt->len_n_flags == htole32(IWM_FH_RSCSR_FRAME_INVALID)) {
5118                         break;
5119                 }
5120
5121                 IWM_DPRINTF(sc, IWM_DEBUG_INTR,
5122                     "rx packet qid=%d idx=%d type=%x\n",
5123                     qid & ~0x80, pkt->hdr.idx, code);
5124
5125                 len = iwm_rx_packet_len(pkt);
5126                 len += sizeof(uint32_t); /* account for status word */
5127                 nextoff = offset + roundup2(len, IWM_FH_RSCSR_FRAME_ALIGN);
5128
5129                 iwm_notification_wait_notify(sc->sc_notif_wait, code, pkt);
5130
5131                 switch (code) {
5132                 case IWM_REPLY_RX_PHY_CMD:
5133                         iwm_mvm_rx_rx_phy_cmd(sc, pkt);
5134                         break;
5135
5136                 case IWM_REPLY_RX_MPDU_CMD: {
5137                         /*
5138                          * If this is the last frame in the RX buffer, we
5139                          * can directly feed the mbuf to the sharks here.
5140                          */
5141                         struct iwm_rx_packet *nextpkt = mtodoff(m,
5142                             struct iwm_rx_packet *, nextoff);
5143                         if (!HAVEROOM(nextoff) ||
5144                             (nextpkt->hdr.code == 0 &&
5145                              (nextpkt->hdr.qid & ~0x80) == 0 &&
5146                              nextpkt->hdr.idx == 0) ||
5147                             (nextpkt->len_n_flags ==
5148                              htole32(IWM_FH_RSCSR_FRAME_INVALID))) {
5149                                 if (iwm_mvm_rx_rx_mpdu(sc, m, offset, stolen)) {
5150                                         stolen = FALSE;
5151                                         /* Make sure we abort the loop */
5152                                         nextoff = maxoff;
5153                                 }
5154                                 break;
5155                         }
5156
5157                         /*
5158                          * Use m_copym instead of m_split, because that
5159                          * makes it easier to keep a valid rx buffer in
5160                          * the ring, when iwm_mvm_rx_rx_mpdu() fails.
5161                          *
5162                          * We need to start m_copym() at offset 0, to get the
5163                          * M_PKTHDR flag preserved.
5164                          */
5165                         m1 = m_copym(m, 0, M_COPYALL, M_NOWAIT);
5166                         if (m1) {
5167                                 if (iwm_mvm_rx_rx_mpdu(sc, m1, offset, stolen))
5168                                         stolen = TRUE;
5169                                 else
5170                                         m_freem(m1);
5171                         }
5172                         break;
5173                 }
5174
5175                 case IWM_TX_CMD:
5176                         iwm_mvm_rx_tx_cmd(sc, pkt);
5177                         break;
5178
5179                 case IWM_MISSED_BEACONS_NOTIFICATION: {
5180                         struct iwm_missed_beacons_notif *resp;
5181                         int missed;
5182
5183                         /* XXX look at mac_id to determine interface ID */
5184                         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
5185
5186                         resp = (void *)pkt->data;
5187                         missed = le32toh(resp->consec_missed_beacons);
5188
5189                         IWM_DPRINTF(sc, IWM_DEBUG_BEACON | IWM_DEBUG_STATE,
5190                             "%s: MISSED_BEACON: mac_id=%d, "
5191                             "consec_since_last_rx=%d, consec=%d, num_expect=%d "
5192                             "num_rx=%d\n",
5193                             __func__,
5194                             le32toh(resp->mac_id),
5195                             le32toh(resp->consec_missed_beacons_since_last_rx),
5196                             le32toh(resp->consec_missed_beacons),
5197                             le32toh(resp->num_expected_beacons),
5198                             le32toh(resp->num_recvd_beacons));
5199
5200                         /* Be paranoid */
5201                         if (vap == NULL)
5202                                 break;
5203
5204                         /* XXX no net80211 locking? */
5205                         if (vap->iv_state == IEEE80211_S_RUN &&
5206                             (ic->ic_flags & IEEE80211_F_SCAN) == 0) {
5207                                 if (missed > vap->iv_bmissthreshold) {
5208                                         /* XXX bad locking; turn into task */
5209                                         IWM_UNLOCK(sc);
5210                                         ieee80211_beacon_miss(ic);
5211                                         IWM_LOCK(sc);
5212                                 }
5213                         }
5214
5215                         break;
5216                 }
5217
5218                 case IWM_MFUART_LOAD_NOTIFICATION:
5219                         break;
5220
5221                 case IWM_MVM_ALIVE:
5222                         break;
5223
5224                 case IWM_CALIB_RES_NOTIF_PHY_DB:
5225                         break;
5226
5227                 case IWM_STATISTICS_NOTIFICATION:
5228                         iwm_mvm_handle_rx_statistics(sc, pkt);
5229                         break;
5230
5231                 case IWM_NVM_ACCESS_CMD:
5232                 case IWM_MCC_UPDATE_CMD:
5233                         if (sc->sc_wantresp == (((qid & ~0x80) << 16) | idx)) {
5234                                 memcpy(sc->sc_cmd_resp,
5235                                     pkt, sizeof(sc->sc_cmd_resp));
5236                         }
5237                         break;
5238
5239                 case IWM_MCC_CHUB_UPDATE_CMD: {
5240                         struct iwm_mcc_chub_notif *notif;
5241                         notif = (void *)pkt->data;
5242
5243                         sc->sc_fw_mcc[0] = (notif->mcc & 0xff00) >> 8;
5244                         sc->sc_fw_mcc[1] = notif->mcc & 0xff;
5245                         sc->sc_fw_mcc[2] = '\0';
5246                         IWM_DPRINTF(sc, IWM_DEBUG_LAR,
5247                             "fw source %d sent CC '%s'\n",
5248                             notif->source_id, sc->sc_fw_mcc);
5249                         break;
5250                 }
5251
5252                 case IWM_DTS_MEASUREMENT_NOTIFICATION:
5253                 case IWM_WIDE_ID(IWM_PHY_OPS_GROUP,
5254                                  IWM_DTS_MEASUREMENT_NOTIF_WIDE): {
5255                         struct iwm_dts_measurement_notif_v1 *notif;
5256
5257                         if (iwm_rx_packet_payload_len(pkt) < sizeof(*notif)) {
5258                                 device_printf(sc->sc_dev,
5259                                     "Invalid DTS_MEASUREMENT_NOTIFICATION\n");
5260                                 break;
5261                         }
5262                         notif = (void *)pkt->data;
5263                         IWM_DPRINTF(sc, IWM_DEBUG_TEMP,
5264                             "IWM_DTS_MEASUREMENT_NOTIFICATION - %d\n",
5265                             notif->temp);
5266                         break;
5267                 }
5268
5269                 case IWM_PHY_CONFIGURATION_CMD:
5270                 case IWM_TX_ANT_CONFIGURATION_CMD:
5271                 case IWM_ADD_STA:
5272                 case IWM_MAC_CONTEXT_CMD:
5273                 case IWM_REPLY_SF_CFG_CMD:
5274                 case IWM_POWER_TABLE_CMD:
5275                 case IWM_LTR_CONFIG:
5276                 case IWM_PHY_CONTEXT_CMD:
5277                 case IWM_BINDING_CONTEXT_CMD:
5278                 case IWM_TIME_EVENT_CMD:
5279                 case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_CFG_CMD):
5280                 case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_REQ_UMAC):
5281                 case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP, IWM_SCAN_ABORT_UMAC):
5282                 case IWM_SCAN_OFFLOAD_REQUEST_CMD:
5283                 case IWM_SCAN_OFFLOAD_ABORT_CMD:
5284                 case IWM_REPLY_BEACON_FILTERING_CMD:
5285                 case IWM_MAC_PM_POWER_TABLE:
5286                 case IWM_TIME_QUOTA_CMD:
5287                 case IWM_REMOVE_STA:
5288                 case IWM_TXPATH_FLUSH:
5289                 case IWM_LQ_CMD:
5290                 case IWM_WIDE_ID(IWM_ALWAYS_LONG_GROUP,
5291                                  IWM_FW_PAGING_BLOCK_CMD):
5292                 case IWM_BT_CONFIG:
5293                 case IWM_REPLY_THERMAL_MNG_BACKOFF:
5294                         cresp = (void *)pkt->data;
5295                         if (sc->sc_wantresp == (((qid & ~0x80) << 16) | idx)) {
5296                                 memcpy(sc->sc_cmd_resp,
5297                                     pkt, sizeof(*pkt)+sizeof(*cresp));
5298                         }
5299                         break;
5300
5301                 /* ignore */
5302                 case IWM_PHY_DB_CMD:
5303                         break;
5304
5305                 case IWM_INIT_COMPLETE_NOTIF:
5306                         break;
5307
5308                 case IWM_SCAN_OFFLOAD_COMPLETE:
5309                         iwm_mvm_rx_lmac_scan_complete_notif(sc, pkt);
5310                         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
5311                                 sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
5312                                 ieee80211_runtask(ic, &sc->sc_es_task);
5313                         }
5314                         break;
5315
5316                 case IWM_SCAN_ITERATION_COMPLETE: {
5317                         struct iwm_lmac_scan_complete_notif *notif;
5318                         notif = (void *)pkt->data;
5319                         break;
5320                 }
5321
5322                 case IWM_SCAN_COMPLETE_UMAC:
5323                         iwm_mvm_rx_umac_scan_complete_notif(sc, pkt);
5324                         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
5325                                 sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
5326                                 ieee80211_runtask(ic, &sc->sc_es_task);
5327                         }
5328                         break;
5329
5330                 case IWM_SCAN_ITERATION_COMPLETE_UMAC: {
5331                         struct iwm_umac_scan_iter_complete_notif *notif;
5332                         notif = (void *)pkt->data;
5333
5334                         IWM_DPRINTF(sc, IWM_DEBUG_SCAN, "UMAC scan iteration "
5335                             "complete, status=0x%x, %d channels scanned\n",
5336                             notif->status, notif->scanned_channels);
5337                         break;
5338                 }
5339
5340                 case IWM_REPLY_ERROR: {
5341                         struct iwm_error_resp *resp;
5342                         resp = (void *)pkt->data;
5343
5344                         device_printf(sc->sc_dev,
5345                             "firmware error 0x%x, cmd 0x%x\n",
5346                             le32toh(resp->error_type),
5347                             resp->cmd_id);
5348                         break;
5349                 }
5350
5351                 case IWM_TIME_EVENT_NOTIFICATION:
5352                         iwm_mvm_rx_time_event_notif(sc, pkt);
5353                         break;
5354
5355                 /*
5356                  * Firmware versions 21 and 22 generate some DEBUG_LOG_MSG
5357                  * messages. Just ignore them for now.
5358                  */
5359                 case IWM_DEBUG_LOG_MSG:
5360                         break;
5361
5362                 case IWM_MCAST_FILTER_CMD:
5363                         break;
5364
5365                 case IWM_SCD_QUEUE_CFG: {
5366                         struct iwm_scd_txq_cfg_rsp *rsp;
5367                         rsp = (void *)pkt->data;
5368
5369                         IWM_DPRINTF(sc, IWM_DEBUG_CMD,
5370                             "queue cfg token=0x%x sta_id=%d "
5371                             "tid=%d scd_queue=%d\n",
5372                             rsp->token, rsp->sta_id, rsp->tid,
5373                             rsp->scd_queue);
5374                         break;
5375                 }
5376
5377                 default:
5378                         device_printf(sc->sc_dev,
5379                             "frame %d/%d %x UNHANDLED (this should "
5380                             "not happen)\n", qid & ~0x80, idx,
5381                             pkt->len_n_flags);
5382                         break;
5383                 }
5384
5385                 /*
5386                  * Why test bit 0x80?  The Linux driver:
5387                  *
5388                  * There is one exception:  uCode sets bit 15 when it
5389                  * originates the response/notification, i.e. when the
5390                  * response/notification is not a direct response to a
5391                  * command sent by the driver.  For example, uCode issues
5392                  * IWM_REPLY_RX when it sends a received frame to the driver;
5393                  * it is not a direct response to any driver command.
5394                  *
5395                  * Ok, so since when is 7 == 15?  Well, the Linux driver
5396                  * uses a slightly different format for pkt->hdr, and "qid"
5397                  * is actually the upper byte of a two-byte field.
5398                  */
5399                 if (!(qid & (1 << 7)))
5400                         iwm_cmd_done(sc, pkt);
5401
5402                 offset = nextoff;
5403         }
5404         if (stolen)
5405                 m_freem(m);
5406 #undef HAVEROOM
5407 }
5408
5409 /*
5410  * Process an IWM_CSR_INT_BIT_FH_RX or IWM_CSR_INT_BIT_SW_RX interrupt.
5411  * Basic structure from if_iwn
5412  */
5413 static void
5414 iwm_notif_intr(struct iwm_softc *sc)
5415 {
5416         uint16_t hw;
5417
5418         bus_dmamap_sync(sc->rxq.stat_dma.tag, sc->rxq.stat_dma.map,
5419             BUS_DMASYNC_POSTREAD);
5420
5421         hw = le16toh(sc->rxq.stat->closed_rb_num) & 0xfff;
5422
5423         /*
5424          * Process responses
5425          */
5426         while (sc->rxq.cur != hw) {
5427                 struct iwm_rx_ring *ring = &sc->rxq;
5428                 struct iwm_rx_data *data = &ring->data[ring->cur];
5429
5430                 bus_dmamap_sync(ring->data_dmat, data->map,
5431                     BUS_DMASYNC_POSTREAD);
5432
5433                 IWM_DPRINTF(sc, IWM_DEBUG_INTR,
5434                     "%s: hw = %d cur = %d\n", __func__, hw, ring->cur);
5435                 iwm_handle_rxb(sc, data->m);
5436
5437                 ring->cur = (ring->cur + 1) % IWM_RX_RING_COUNT;
5438         }
5439
5440         /*
5441          * Tell the firmware that it can reuse the ring entries that
5442          * we have just processed.
5443          * Seems like the hardware gets upset unless we align
5444          * the write by 8??
5445          */
5446         hw = (hw == 0) ? IWM_RX_RING_COUNT - 1 : hw - 1;
5447         IWM_WRITE(sc, IWM_FH_RSCSR_CHNL0_WPTR, rounddown2(hw, 8));
5448 }
5449
5450 static void
5451 iwm_intr(void *arg)
5452 {
5453         struct iwm_softc *sc = arg;
5454         int handled = 0;
5455         int r1, r2, rv = 0;
5456         int isperiodic = 0;
5457
5458         IWM_LOCK(sc);
5459         IWM_WRITE(sc, IWM_CSR_INT_MASK, 0);
5460
5461         if (sc->sc_flags & IWM_FLAG_USE_ICT) {
5462                 uint32_t *ict = sc->ict_dma.vaddr;
5463                 int tmp;
5464
5465                 tmp = htole32(ict[sc->ict_cur]);
5466                 if (!tmp)
5467                         goto out_ena;
5468
5469                 /*
5470                  * ok, there was something.  keep plowing until we have all.
5471                  */
5472                 r1 = r2 = 0;
5473                 while (tmp) {
5474                         r1 |= tmp;
5475                         ict[sc->ict_cur] = 0;
5476                         sc->ict_cur = (sc->ict_cur+1) % IWM_ICT_COUNT;
5477                         tmp = htole32(ict[sc->ict_cur]);
5478                 }
5479
5480                 /* this is where the fun begins.  don't ask */
5481                 if (r1 == 0xffffffff)
5482                         r1 = 0;
5483
5484                 /* i am not expected to understand this */
5485                 if (r1 & 0xc0000)
5486                         r1 |= 0x8000;
5487                 r1 = (0xff & r1) | ((0xff00 & r1) << 16);
5488         } else {
5489                 r1 = IWM_READ(sc, IWM_CSR_INT);
5490                 /* "hardware gone" (where, fishing?) */
5491                 if (r1 == 0xffffffff || (r1 & 0xfffffff0) == 0xa5a5a5a0)
5492                         goto out;
5493                 r2 = IWM_READ(sc, IWM_CSR_FH_INT_STATUS);
5494         }
5495         if (r1 == 0 && r2 == 0) {
5496                 goto out_ena;
5497         }
5498
5499         IWM_WRITE(sc, IWM_CSR_INT, r1 | ~sc->sc_intmask);
5500
5501         /* Safely ignore these bits for debug checks below */
5502         r1 &= ~(IWM_CSR_INT_BIT_ALIVE | IWM_CSR_INT_BIT_SCD);
5503
5504         if (r1 & IWM_CSR_INT_BIT_SW_ERR) {
5505                 int i;
5506                 struct ieee80211com *ic = &sc->sc_ic;
5507                 struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
5508
5509 #ifdef IWM_DEBUG
5510                 iwm_nic_error(sc);
5511 #endif
5512                 /* Dump driver status (TX and RX rings) while we're here. */
5513                 device_printf(sc->sc_dev, "driver status:\n");
5514                 for (i = 0; i < IWM_MVM_MAX_QUEUES; i++) {
5515                         struct iwm_tx_ring *ring = &sc->txq[i];
5516                         device_printf(sc->sc_dev,
5517                             "  tx ring %2d: qid=%-2d cur=%-3d "
5518                             "queued=%-3d\n",
5519                             i, ring->qid, ring->cur, ring->queued);
5520                 }
5521                 device_printf(sc->sc_dev,
5522                     "  rx ring: cur=%d\n", sc->rxq.cur);
5523                 device_printf(sc->sc_dev,
5524                     "  802.11 state %d\n", (vap == NULL) ? -1 : vap->iv_state);
5525
5526                 /* Reset our firmware state tracking. */
5527                 sc->sc_firmware_state = 0;
5528                 /* Don't stop the device; just do a VAP restart */
5529                 IWM_UNLOCK(sc);
5530
5531                 if (vap == NULL) {
5532                         printf("%s: null vap\n", __func__);
5533                         return;
5534                 }
5535
5536                 device_printf(sc->sc_dev, "%s: controller panicked, iv_state = %d; "
5537                     "restarting\n", __func__, vap->iv_state);
5538
5539                 ieee80211_restart_all(ic);
5540                 return;
5541         }
5542
5543         if (r1 & IWM_CSR_INT_BIT_HW_ERR) {
5544                 handled |= IWM_CSR_INT_BIT_HW_ERR;
5545                 device_printf(sc->sc_dev, "hardware error, stopping device\n");
5546                 iwm_stop(sc);
5547                 rv = 1;
5548                 goto out;
5549         }
5550
5551         /* firmware chunk loaded */
5552         if (r1 & IWM_CSR_INT_BIT_FH_TX) {
5553                 IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, IWM_CSR_FH_INT_TX_MASK);
5554                 handled |= IWM_CSR_INT_BIT_FH_TX;
5555                 sc->sc_fw_chunk_done = 1;
5556                 wakeup(&sc->sc_fw);
5557         }
5558
5559         if (r1 & IWM_CSR_INT_BIT_RF_KILL) {
5560                 handled |= IWM_CSR_INT_BIT_RF_KILL;
5561                 if (iwm_check_rfkill(sc)) {
5562                         device_printf(sc->sc_dev,
5563                             "%s: rfkill switch, disabling interface\n",
5564                             __func__);
5565                         iwm_stop(sc);
5566                 }
5567         }
5568
5569         /*
5570          * The Linux driver uses periodic interrupts to avoid races.
5571          * We cargo-cult like it's going out of fashion.
5572          */
5573         if (r1 & IWM_CSR_INT_BIT_RX_PERIODIC) {
5574                 handled |= IWM_CSR_INT_BIT_RX_PERIODIC;
5575                 IWM_WRITE(sc, IWM_CSR_INT, IWM_CSR_INT_BIT_RX_PERIODIC);
5576                 if ((r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX)) == 0)
5577                         IWM_WRITE_1(sc,
5578                             IWM_CSR_INT_PERIODIC_REG, IWM_CSR_INT_PERIODIC_DIS);
5579                 isperiodic = 1;
5580         }
5581
5582         if ((r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX)) || isperiodic) {
5583                 handled |= (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX);
5584                 IWM_WRITE(sc, IWM_CSR_FH_INT_STATUS, IWM_CSR_FH_INT_RX_MASK);
5585
5586                 iwm_notif_intr(sc);
5587
5588                 /* enable periodic interrupt, see above */
5589                 if (r1 & (IWM_CSR_INT_BIT_FH_RX | IWM_CSR_INT_BIT_SW_RX) && !isperiodic)
5590                         IWM_WRITE_1(sc, IWM_CSR_INT_PERIODIC_REG,
5591                             IWM_CSR_INT_PERIODIC_ENA);
5592         }
5593
5594         if (__predict_false(r1 & ~handled))
5595                 IWM_DPRINTF(sc, IWM_DEBUG_INTR,
5596                     "%s: unhandled interrupts: %x\n", __func__, r1);
5597         rv = 1;
5598
5599  out_ena:
5600         iwm_restore_interrupts(sc);
5601  out:
5602         IWM_UNLOCK(sc);
5603         return;
5604 }
5605
5606 /*
5607  * Autoconf glue-sniffing
5608  */
5609 #define PCI_VENDOR_INTEL                0x8086
5610 #define PCI_PRODUCT_INTEL_WL_3160_1     0x08b3
5611 #define PCI_PRODUCT_INTEL_WL_3160_2     0x08b4
5612 #define PCI_PRODUCT_INTEL_WL_3165_1     0x3165
5613 #define PCI_PRODUCT_INTEL_WL_3165_2     0x3166
5614 #define PCI_PRODUCT_INTEL_WL_3168_1     0x24fb
5615 #define PCI_PRODUCT_INTEL_WL_7260_1     0x08b1
5616 #define PCI_PRODUCT_INTEL_WL_7260_2     0x08b2
5617 #define PCI_PRODUCT_INTEL_WL_7265_1     0x095a
5618 #define PCI_PRODUCT_INTEL_WL_7265_2     0x095b
5619 #define PCI_PRODUCT_INTEL_WL_8260_1     0x24f3
5620 #define PCI_PRODUCT_INTEL_WL_8260_2     0x24f4
5621 #define PCI_PRODUCT_INTEL_WL_8265_1     0x24fd
5622
5623 static const struct iwm_devices {
5624         uint16_t                device;
5625         const struct iwm_cfg    *cfg;
5626 } iwm_devices[] = {
5627         { PCI_PRODUCT_INTEL_WL_3160_1, &iwm3160_cfg },
5628         { PCI_PRODUCT_INTEL_WL_3160_2, &iwm3160_cfg },
5629         { PCI_PRODUCT_INTEL_WL_3165_1, &iwm3165_cfg },
5630         { PCI_PRODUCT_INTEL_WL_3165_2, &iwm3165_cfg },
5631         { PCI_PRODUCT_INTEL_WL_3168_1, &iwm3168_cfg },
5632         { PCI_PRODUCT_INTEL_WL_7260_1, &iwm7260_cfg },
5633         { PCI_PRODUCT_INTEL_WL_7260_2, &iwm7260_cfg },
5634         { PCI_PRODUCT_INTEL_WL_7265_1, &iwm7265_cfg },
5635         { PCI_PRODUCT_INTEL_WL_7265_2, &iwm7265_cfg },
5636         { PCI_PRODUCT_INTEL_WL_8260_1, &iwm8260_cfg },
5637         { PCI_PRODUCT_INTEL_WL_8260_2, &iwm8260_cfg },
5638         { PCI_PRODUCT_INTEL_WL_8265_1, &iwm8265_cfg },
5639 };
5640
5641 static int
5642 iwm_probe(device_t dev)
5643 {
5644         int i;
5645
5646         for (i = 0; i < nitems(iwm_devices); i++) {
5647                 if (pci_get_vendor(dev) == PCI_VENDOR_INTEL &&
5648                     pci_get_device(dev) == iwm_devices[i].device) {
5649                         device_set_desc(dev, iwm_devices[i].cfg->name);
5650                         return (BUS_PROBE_DEFAULT);
5651                 }
5652         }
5653
5654         return (ENXIO);
5655 }
5656
5657 static int
5658 iwm_dev_check(device_t dev)
5659 {
5660         struct iwm_softc *sc;
5661         uint16_t devid;
5662         int i;
5663
5664         sc = device_get_softc(dev);
5665
5666         devid = pci_get_device(dev);
5667         for (i = 0; i < nitems(iwm_devices); i++) {
5668                 if (iwm_devices[i].device == devid) {
5669                         sc->cfg = iwm_devices[i].cfg;
5670                         return (0);
5671                 }
5672         }
5673         device_printf(dev, "unknown adapter type\n");
5674         return ENXIO;
5675 }
5676
5677 /* PCI registers */
5678 #define PCI_CFG_RETRY_TIMEOUT   0x041
5679
5680 static int
5681 iwm_pci_attach(device_t dev)
5682 {
5683         struct iwm_softc *sc;
5684         int count, error, rid;
5685         uint16_t reg;
5686
5687         sc = device_get_softc(dev);
5688
5689         /* We disable the RETRY_TIMEOUT register (0x41) to keep
5690          * PCI Tx retries from interfering with C3 CPU state */
5691         pci_write_config(dev, PCI_CFG_RETRY_TIMEOUT, 0x00, 1);
5692
5693         /* Enable bus-mastering and hardware bug workaround. */
5694         pci_enable_busmaster(dev);
5695         reg = pci_read_config(dev, PCIR_STATUS, sizeof(reg));
5696         /* if !MSI */
5697         if (reg & PCIM_STATUS_INTxSTATE) {
5698                 reg &= ~PCIM_STATUS_INTxSTATE;
5699         }
5700         pci_write_config(dev, PCIR_STATUS, reg, sizeof(reg));
5701
5702         rid = PCIR_BAR(0);
5703         sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
5704             RF_ACTIVE);
5705         if (sc->sc_mem == NULL) {
5706                 device_printf(sc->sc_dev, "can't map mem space\n");
5707                 return (ENXIO);
5708         }
5709         sc->sc_st = rman_get_bustag(sc->sc_mem);
5710         sc->sc_sh = rman_get_bushandle(sc->sc_mem);
5711
5712         /* Install interrupt handler. */
5713         count = 1;
5714         rid = 0;
5715         if (pci_alloc_msi(dev, &count) == 0)
5716                 rid = 1;
5717         sc->sc_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, RF_ACTIVE |
5718             (rid != 0 ? 0 : RF_SHAREABLE));
5719         if (sc->sc_irq == NULL) {
5720                 device_printf(dev, "can't map interrupt\n");
5721                         return (ENXIO);
5722         }
5723         error = bus_setup_intr(dev, sc->sc_irq, INTR_TYPE_NET | INTR_MPSAFE,
5724             NULL, iwm_intr, sc, &sc->sc_ih);
5725         if (sc->sc_ih == NULL) {
5726                 device_printf(dev, "can't establish interrupt");
5727                         return (ENXIO);
5728         }
5729         sc->sc_dmat = bus_get_dma_tag(sc->sc_dev);
5730
5731         return (0);
5732 }
5733
5734 static void
5735 iwm_pci_detach(device_t dev)
5736 {
5737         struct iwm_softc *sc = device_get_softc(dev);
5738
5739         if (sc->sc_irq != NULL) {
5740                 bus_teardown_intr(dev, sc->sc_irq, sc->sc_ih);
5741                 bus_release_resource(dev, SYS_RES_IRQ,
5742                     rman_get_rid(sc->sc_irq), sc->sc_irq);
5743                 pci_release_msi(dev);
5744         }
5745         if (sc->sc_mem != NULL)
5746                 bus_release_resource(dev, SYS_RES_MEMORY,
5747                     rman_get_rid(sc->sc_mem), sc->sc_mem);
5748 }
5749
5750
5751
5752 static int
5753 iwm_attach(device_t dev)
5754 {
5755         struct iwm_softc *sc = device_get_softc(dev);
5756         struct ieee80211com *ic = &sc->sc_ic;
5757         int error;
5758         int txq_i, i;
5759
5760         sc->sc_dev = dev;
5761         sc->sc_attached = 1;
5762         IWM_LOCK_INIT(sc);
5763         mbufq_init(&sc->sc_snd, ifqmaxlen);
5764         callout_init_mtx(&sc->sc_watchdog_to, &sc->sc_mtx, 0);
5765         callout_init_mtx(&sc->sc_led_blink_to, &sc->sc_mtx, 0);
5766         TASK_INIT(&sc->sc_es_task, 0, iwm_endscan_cb, sc);
5767
5768         sc->sc_notif_wait = iwm_notification_wait_init(sc);
5769         if (sc->sc_notif_wait == NULL) {
5770                 device_printf(dev, "failed to init notification wait struct\n");
5771                 goto fail;
5772         }
5773
5774         sc->sf_state = IWM_SF_UNINIT;
5775
5776         /* Init phy db */
5777         sc->sc_phy_db = iwm_phy_db_init(sc);
5778         if (!sc->sc_phy_db) {
5779                 device_printf(dev, "Cannot init phy_db\n");
5780                 goto fail;
5781         }
5782
5783         /* Set EBS as successful as long as not stated otherwise by the FW. */
5784         sc->last_ebs_successful = TRUE;
5785
5786         /* PCI attach */
5787         error = iwm_pci_attach(dev);
5788         if (error != 0)
5789                 goto fail;
5790
5791         sc->sc_wantresp = -1;
5792
5793         /* Match device id */
5794         error = iwm_dev_check(dev);
5795         if (error != 0)
5796                 goto fail;
5797
5798         sc->sc_hw_rev = IWM_READ(sc, IWM_CSR_HW_REV);
5799         /*
5800          * In the 8000 HW family the format of the 4 bytes of CSR_HW_REV have
5801          * changed, and now the revision step also includes bit 0-1 (no more
5802          * "dash" value). To keep hw_rev backwards compatible - we'll store it
5803          * in the old format.
5804          */
5805         if (sc->cfg->device_family == IWM_DEVICE_FAMILY_8000) {
5806                 int ret;
5807                 uint32_t hw_step;
5808
5809                 sc->sc_hw_rev = (sc->sc_hw_rev & 0xfff0) |
5810                                 (IWM_CSR_HW_REV_STEP(sc->sc_hw_rev << 2) << 2);
5811
5812                 if (iwm_prepare_card_hw(sc) != 0) {
5813                         device_printf(dev, "could not initialize hardware\n");
5814                         goto fail;
5815                 }
5816
5817                 /*
5818                  * In order to recognize C step the driver should read the
5819                  * chip version id located at the AUX bus MISC address.
5820                  */
5821                 IWM_SETBITS(sc, IWM_CSR_GP_CNTRL,
5822                             IWM_CSR_GP_CNTRL_REG_FLAG_INIT_DONE);
5823                 DELAY(2);
5824
5825                 ret = iwm_poll_bit(sc, IWM_CSR_GP_CNTRL,
5826                                    IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
5827                                    IWM_CSR_GP_CNTRL_REG_FLAG_MAC_CLOCK_READY,
5828                                    25000);
5829                 if (!ret) {
5830                         device_printf(sc->sc_dev,
5831                             "Failed to wake up the nic\n");
5832                         goto fail;
5833                 }
5834
5835                 if (iwm_nic_lock(sc)) {
5836                         hw_step = iwm_read_prph(sc, IWM_WFPM_CTRL_REG);
5837                         hw_step |= IWM_ENABLE_WFPM;
5838                         iwm_write_prph(sc, IWM_WFPM_CTRL_REG, hw_step);
5839                         hw_step = iwm_read_prph(sc, IWM_AUX_MISC_REG);
5840                         hw_step = (hw_step >> IWM_HW_STEP_LOCATION_BITS) & 0xF;
5841                         if (hw_step == 0x3)
5842                                 sc->sc_hw_rev = (sc->sc_hw_rev & 0xFFFFFFF3) |
5843                                                 (IWM_SILICON_C_STEP << 2);
5844                         iwm_nic_unlock(sc);
5845                 } else {
5846                         device_printf(sc->sc_dev, "Failed to lock the nic\n");
5847                         goto fail;
5848                 }
5849         }
5850
5851         /* special-case 7265D, it has the same PCI IDs. */
5852         if (sc->cfg == &iwm7265_cfg &&
5853             (sc->sc_hw_rev & IWM_CSR_HW_REV_TYPE_MSK) == IWM_CSR_HW_REV_TYPE_7265D) {
5854                 sc->cfg = &iwm7265d_cfg;
5855         }
5856
5857         /* Allocate DMA memory for firmware transfers. */
5858         if ((error = iwm_alloc_fwmem(sc)) != 0) {
5859                 device_printf(dev, "could not allocate memory for firmware\n");
5860                 goto fail;
5861         }
5862
5863         /* Allocate "Keep Warm" page. */
5864         if ((error = iwm_alloc_kw(sc)) != 0) {
5865                 device_printf(dev, "could not allocate keep warm page\n");
5866                 goto fail;
5867         }
5868
5869         /* We use ICT interrupts */
5870         if ((error = iwm_alloc_ict(sc)) != 0) {
5871                 device_printf(dev, "could not allocate ICT table\n");
5872                 goto fail;
5873         }
5874
5875         /* Allocate TX scheduler "rings". */
5876         if ((error = iwm_alloc_sched(sc)) != 0) {
5877                 device_printf(dev, "could not allocate TX scheduler rings\n");
5878                 goto fail;
5879         }
5880
5881         /* Allocate TX rings */
5882         for (txq_i = 0; txq_i < nitems(sc->txq); txq_i++) {
5883                 if ((error = iwm_alloc_tx_ring(sc,
5884                     &sc->txq[txq_i], txq_i)) != 0) {
5885                         device_printf(dev,
5886                             "could not allocate TX ring %d\n",
5887                             txq_i);
5888                         goto fail;
5889                 }
5890         }
5891
5892         /* Allocate RX ring. */
5893         if ((error = iwm_alloc_rx_ring(sc, &sc->rxq)) != 0) {
5894                 device_printf(dev, "could not allocate RX ring\n");
5895                 goto fail;
5896         }
5897
5898         /* Clear pending interrupts. */
5899         IWM_WRITE(sc, IWM_CSR_INT, 0xffffffff);
5900
5901         ic->ic_softc = sc;
5902         ic->ic_name = device_get_nameunit(sc->sc_dev);
5903         ic->ic_phytype = IEEE80211_T_OFDM;      /* not only, but not used */
5904         ic->ic_opmode = IEEE80211_M_STA;        /* default to BSS mode */
5905
5906         /* Set device capabilities. */
5907         ic->ic_caps =
5908             IEEE80211_C_STA |
5909             IEEE80211_C_WPA |           /* WPA/RSN */
5910             IEEE80211_C_WME |
5911             IEEE80211_C_PMGT |
5912             IEEE80211_C_SHSLOT |        /* short slot time supported */
5913             IEEE80211_C_SHPREAMBLE      /* short preamble supported */
5914 //          IEEE80211_C_BGSCAN          /* capable of bg scanning */
5915             ;
5916         /* Advertise full-offload scanning */
5917         ic->ic_flags_ext = IEEE80211_FEXT_SCAN_OFFLOAD;
5918         for (i = 0; i < nitems(sc->sc_phyctxt); i++) {
5919                 sc->sc_phyctxt[i].id = i;
5920                 sc->sc_phyctxt[i].color = 0;
5921                 sc->sc_phyctxt[i].ref = 0;
5922                 sc->sc_phyctxt[i].channel = NULL;
5923         }
5924
5925         /* Default noise floor */
5926         sc->sc_noise = -96;
5927
5928         /* Max RSSI */
5929         sc->sc_max_rssi = IWM_MAX_DBM - IWM_MIN_DBM;
5930
5931 #ifdef IWM_DEBUG
5932         SYSCTL_ADD_INT(device_get_sysctl_ctx(dev),
5933             SYSCTL_CHILDREN(device_get_sysctl_tree(dev)), OID_AUTO, "debug",
5934             CTLFLAG_RW, &sc->sc_debug, 0, "control debugging");
5935 #endif
5936
5937         error = iwm_read_firmware(sc);
5938         if (error) {
5939                 goto fail;
5940         } else if (sc->sc_fw.fw_fp == NULL) {
5941                 /*
5942                  * XXX Add a solution for properly deferring firmware load
5943                  *     during bootup.
5944                  */
5945                 goto fail;
5946         } else {
5947                 sc->sc_preinit_hook.ich_func = iwm_preinit;
5948                 sc->sc_preinit_hook.ich_arg = sc;
5949                 if (config_intrhook_establish(&sc->sc_preinit_hook) != 0) {
5950                         device_printf(dev,
5951                             "config_intrhook_establish failed\n");
5952                         goto fail;
5953                 }
5954         }
5955
5956         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
5957             "<-%s\n", __func__);
5958
5959         return 0;
5960
5961         /* Free allocated memory if something failed during attachment. */
5962 fail:
5963         iwm_detach_local(sc, 0);
5964
5965         return ENXIO;
5966 }
5967
5968 static int
5969 iwm_is_valid_ether_addr(uint8_t *addr)
5970 {
5971         char zero_addr[IEEE80211_ADDR_LEN] = { 0, 0, 0, 0, 0, 0 };
5972
5973         if ((addr[0] & 1) || IEEE80211_ADDR_EQ(zero_addr, addr))
5974                 return (FALSE);
5975
5976         return (TRUE);
5977 }
5978
5979 static int
5980 iwm_wme_update(struct ieee80211com *ic)
5981 {
5982 #define IWM_EXP2(x)     ((1 << (x)) - 1)        /* CWmin = 2^ECWmin - 1 */
5983         struct iwm_softc *sc = ic->ic_softc;
5984         struct chanAccParams chp;
5985         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
5986         struct iwm_vap *ivp = IWM_VAP(vap);
5987         struct iwm_node *in;
5988         struct wmeParams tmp[WME_NUM_AC];
5989         int aci, error;
5990
5991         if (vap == NULL)
5992                 return (0);
5993
5994         ieee80211_wme_ic_getparams(ic, &chp);
5995
5996         IEEE80211_LOCK(ic);
5997         for (aci = 0; aci < WME_NUM_AC; aci++)
5998                 tmp[aci] = chp.cap_wmeParams[aci];
5999         IEEE80211_UNLOCK(ic);
6000
6001         IWM_LOCK(sc);
6002         for (aci = 0; aci < WME_NUM_AC; aci++) {
6003                 const struct wmeParams *ac = &tmp[aci];
6004                 ivp->queue_params[aci].aifsn = ac->wmep_aifsn;
6005                 ivp->queue_params[aci].cw_min = IWM_EXP2(ac->wmep_logcwmin);
6006                 ivp->queue_params[aci].cw_max = IWM_EXP2(ac->wmep_logcwmax);
6007                 ivp->queue_params[aci].edca_txop =
6008                     IEEE80211_TXOP_TO_US(ac->wmep_txopLimit);
6009         }
6010         ivp->have_wme = TRUE;
6011         if (ivp->is_uploaded && vap->iv_bss != NULL) {
6012                 in = IWM_NODE(vap->iv_bss);
6013                 if (in->in_assoc) {
6014                         if ((error = iwm_mvm_mac_ctxt_changed(sc, vap)) != 0) {
6015                                 device_printf(sc->sc_dev,
6016                                     "%s: failed to update MAC\n", __func__);
6017                         }
6018                 }
6019         }
6020         IWM_UNLOCK(sc);
6021
6022         return (0);
6023 #undef IWM_EXP2
6024 }
6025
6026 static void
6027 iwm_preinit(void *arg)
6028 {
6029         struct iwm_softc *sc = arg;
6030         device_t dev = sc->sc_dev;
6031         struct ieee80211com *ic = &sc->sc_ic;
6032         int error;
6033
6034         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6035             "->%s\n", __func__);
6036
6037         IWM_LOCK(sc);
6038         if ((error = iwm_start_hw(sc)) != 0) {
6039                 device_printf(dev, "could not initialize hardware\n");
6040                 IWM_UNLOCK(sc);
6041                 goto fail;
6042         }
6043
6044         error = iwm_run_init_mvm_ucode(sc, 1);
6045         iwm_stop_device(sc);
6046         if (error) {
6047                 IWM_UNLOCK(sc);
6048                 goto fail;
6049         }
6050         device_printf(dev,
6051             "hw rev 0x%x, fw ver %s, address %s\n",
6052             sc->sc_hw_rev & IWM_CSR_HW_REV_TYPE_MSK,
6053             sc->sc_fwver, ether_sprintf(sc->nvm_data->hw_addr));
6054
6055         /* not all hardware can do 5GHz band */
6056         if (!sc->nvm_data->sku_cap_band_52GHz_enable)
6057                 memset(&ic->ic_sup_rates[IEEE80211_MODE_11A], 0,
6058                     sizeof(ic->ic_sup_rates[IEEE80211_MODE_11A]));
6059         IWM_UNLOCK(sc);
6060
6061         iwm_init_channel_map(ic, IEEE80211_CHAN_MAX, &ic->ic_nchans,
6062             ic->ic_channels);
6063
6064         /*
6065          * At this point we've committed - if we fail to do setup,
6066          * we now also have to tear down the net80211 state.
6067          */
6068         ieee80211_ifattach(ic);
6069         ic->ic_vap_create = iwm_vap_create;
6070         ic->ic_vap_delete = iwm_vap_delete;
6071         ic->ic_raw_xmit = iwm_raw_xmit;
6072         ic->ic_node_alloc = iwm_node_alloc;
6073         ic->ic_scan_start = iwm_scan_start;
6074         ic->ic_scan_end = iwm_scan_end;
6075         ic->ic_update_mcast = iwm_update_mcast;
6076         ic->ic_getradiocaps = iwm_init_channel_map;
6077         ic->ic_set_channel = iwm_set_channel;
6078         ic->ic_scan_curchan = iwm_scan_curchan;
6079         ic->ic_scan_mindwell = iwm_scan_mindwell;
6080         ic->ic_wme.wme_update = iwm_wme_update;
6081         ic->ic_parent = iwm_parent;
6082         ic->ic_transmit = iwm_transmit;
6083         iwm_radiotap_attach(sc);
6084         if (bootverbose)
6085                 ieee80211_announce(ic);
6086
6087         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6088             "<-%s\n", __func__);
6089         config_intrhook_disestablish(&sc->sc_preinit_hook);
6090
6091         return;
6092 fail:
6093         config_intrhook_disestablish(&sc->sc_preinit_hook);
6094         iwm_detach_local(sc, 0);
6095 }
6096
6097 /*
6098  * Attach the interface to 802.11 radiotap.
6099  */
6100 static void
6101 iwm_radiotap_attach(struct iwm_softc *sc)
6102 {
6103         struct ieee80211com *ic = &sc->sc_ic;
6104
6105         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6106             "->%s begin\n", __func__);
6107         ieee80211_radiotap_attach(ic,
6108             &sc->sc_txtap.wt_ihdr, sizeof(sc->sc_txtap),
6109                 IWM_TX_RADIOTAP_PRESENT,
6110             &sc->sc_rxtap.wr_ihdr, sizeof(sc->sc_rxtap),
6111                 IWM_RX_RADIOTAP_PRESENT);
6112         IWM_DPRINTF(sc, IWM_DEBUG_RESET | IWM_DEBUG_TRACE,
6113             "->%s end\n", __func__);
6114 }
6115
6116 static struct ieee80211vap *
6117 iwm_vap_create(struct ieee80211com *ic, const char name[IFNAMSIZ], int unit,
6118     enum ieee80211_opmode opmode, int flags,
6119     const uint8_t bssid[IEEE80211_ADDR_LEN],
6120     const uint8_t mac[IEEE80211_ADDR_LEN])
6121 {
6122         struct iwm_vap *ivp;
6123         struct ieee80211vap *vap;
6124
6125         if (!TAILQ_EMPTY(&ic->ic_vaps))         /* only one at a time */
6126                 return NULL;
6127         ivp = malloc(sizeof(struct iwm_vap), M_80211_VAP, M_WAITOK | M_ZERO);
6128         vap = &ivp->iv_vap;
6129         ieee80211_vap_setup(ic, vap, name, unit, opmode, flags, bssid);
6130         vap->iv_bmissthreshold = 10;            /* override default */
6131         /* Override with driver methods. */
6132         ivp->iv_newstate = vap->iv_newstate;
6133         vap->iv_newstate = iwm_newstate;
6134
6135         ivp->id = IWM_DEFAULT_MACID;
6136         ivp->color = IWM_DEFAULT_COLOR;
6137
6138         ivp->have_wme = FALSE;
6139         ivp->ps_disabled = FALSE;
6140
6141         ieee80211_ratectl_init(vap);
6142         /* Complete setup. */
6143         ieee80211_vap_attach(vap, iwm_media_change, ieee80211_media_status,
6144             mac);
6145         ic->ic_opmode = opmode;
6146
6147         return vap;
6148 }
6149
6150 static void
6151 iwm_vap_delete(struct ieee80211vap *vap)
6152 {
6153         struct iwm_vap *ivp = IWM_VAP(vap);
6154
6155         ieee80211_ratectl_deinit(vap);
6156         ieee80211_vap_detach(vap);
6157         free(ivp, M_80211_VAP);
6158 }
6159
6160 static void
6161 iwm_xmit_queue_drain(struct iwm_softc *sc)
6162 {
6163         struct mbuf *m;
6164         struct ieee80211_node *ni;
6165
6166         while ((m = mbufq_dequeue(&sc->sc_snd)) != NULL) {
6167                 ni = (struct ieee80211_node *)m->m_pkthdr.rcvif;
6168                 ieee80211_free_node(ni);
6169                 m_freem(m);
6170         }
6171 }
6172
6173 static void
6174 iwm_scan_start(struct ieee80211com *ic)
6175 {
6176         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
6177         struct iwm_softc *sc = ic->ic_softc;
6178         int error;
6179
6180         IWM_LOCK(sc);
6181         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
6182                 /* This should not be possible */
6183                 device_printf(sc->sc_dev,
6184                     "%s: Previous scan not completed yet\n", __func__);
6185         }
6186         if (fw_has_capa(&sc->sc_fw.ucode_capa, IWM_UCODE_TLV_CAPA_UMAC_SCAN))
6187                 error = iwm_mvm_umac_scan(sc);
6188         else
6189                 error = iwm_mvm_lmac_scan(sc);
6190         if (error != 0) {
6191                 device_printf(sc->sc_dev, "could not initiate scan\n");
6192                 IWM_UNLOCK(sc);
6193                 ieee80211_cancel_scan(vap);
6194         } else {
6195                 sc->sc_flags |= IWM_FLAG_SCAN_RUNNING;
6196                 iwm_led_blink_start(sc);
6197                 IWM_UNLOCK(sc);
6198         }
6199 }
6200
6201 static void
6202 iwm_scan_end(struct ieee80211com *ic)
6203 {
6204         struct ieee80211vap *vap = TAILQ_FIRST(&ic->ic_vaps);
6205         struct iwm_softc *sc = ic->ic_softc;
6206
6207         IWM_LOCK(sc);
6208         iwm_led_blink_stop(sc);
6209         if (vap->iv_state == IEEE80211_S_RUN)
6210                 iwm_mvm_led_enable(sc);
6211         if (sc->sc_flags & IWM_FLAG_SCAN_RUNNING) {
6212                 /*
6213                  * Removing IWM_FLAG_SCAN_RUNNING now, is fine because
6214                  * both iwm_scan_end and iwm_scan_start run in the ic->ic_tq
6215                  * taskqueue.
6216                  */
6217                 sc->sc_flags &= ~IWM_FLAG_SCAN_RUNNING;
6218                 iwm_mvm_scan_stop_wait(sc);
6219         }
6220         IWM_UNLOCK(sc);
6221
6222         /*
6223          * Make sure we don't race, if sc_es_task is still enqueued here.
6224          * This is to make sure that it won't call ieee80211_scan_done
6225          * when we have already started the next scan.
6226          */
6227         taskqueue_cancel(ic->ic_tq, &sc->sc_es_task, NULL);
6228 }
6229
6230 static void
6231 iwm_update_mcast(struct ieee80211com *ic)
6232 {
6233 }
6234
6235 static void
6236 iwm_set_channel(struct ieee80211com *ic)
6237 {
6238 }
6239
6240 static void
6241 iwm_scan_curchan(struct ieee80211_scan_state *ss, unsigned long maxdwell)
6242 {
6243 }
6244
6245 static void
6246 iwm_scan_mindwell(struct ieee80211_scan_state *ss)
6247 {
6248         return;
6249 }
6250
6251 void
6252 iwm_init_task(void *arg1)
6253 {
6254         struct iwm_softc *sc = arg1;
6255
6256         IWM_LOCK(sc);
6257         while (sc->sc_flags & IWM_FLAG_BUSY)
6258                 msleep(&sc->sc_flags, &sc->sc_mtx, 0, "iwmpwr", 0);
6259         sc->sc_flags |= IWM_FLAG_BUSY;
6260         iwm_stop(sc);
6261         if (sc->sc_ic.ic_nrunning > 0)
6262                 iwm_init(sc);
6263         sc->sc_flags &= ~IWM_FLAG_BUSY;
6264         wakeup(&sc->sc_flags);
6265         IWM_UNLOCK(sc);
6266 }
6267
6268 static int
6269 iwm_resume(device_t dev)
6270 {
6271         struct iwm_softc *sc = device_get_softc(dev);
6272         int do_reinit = 0;
6273
6274         /*
6275          * We disable the RETRY_TIMEOUT register (0x41) to keep
6276          * PCI Tx retries from interfering with C3 CPU state.
6277          */
6278         pci_write_config(dev, PCI_CFG_RETRY_TIMEOUT, 0x00, 1);
6279
6280         if (!sc->sc_attached)
6281                 return 0;
6282
6283         iwm_init_task(device_get_softc(dev));
6284
6285         IWM_LOCK(sc);
6286         if (sc->sc_flags & IWM_FLAG_SCANNING) {
6287                 sc->sc_flags &= ~IWM_FLAG_SCANNING;
6288                 do_reinit = 1;
6289         }
6290         IWM_UNLOCK(sc);
6291
6292         if (do_reinit)
6293                 ieee80211_resume_all(&sc->sc_ic);
6294
6295         return 0;
6296 }
6297
6298 static int
6299 iwm_suspend(device_t dev)
6300 {
6301         int do_stop = 0;
6302         struct iwm_softc *sc = device_get_softc(dev);
6303
6304         do_stop = !! (sc->sc_ic.ic_nrunning > 0);
6305
6306         if (!sc->sc_attached)
6307                 return (0);
6308
6309         ieee80211_suspend_all(&sc->sc_ic);
6310
6311         if (do_stop) {
6312                 IWM_LOCK(sc);
6313                 iwm_stop(sc);
6314                 sc->sc_flags |= IWM_FLAG_SCANNING;
6315                 IWM_UNLOCK(sc);
6316         }
6317
6318         return (0);
6319 }
6320
6321 static int
6322 iwm_detach_local(struct iwm_softc *sc, int do_net80211)
6323 {
6324         struct iwm_fw_info *fw = &sc->sc_fw;
6325         device_t dev = sc->sc_dev;
6326         int i;
6327
6328         if (!sc->sc_attached)
6329                 return 0;
6330         sc->sc_attached = 0;
6331
6332         if (do_net80211)
6333                 ieee80211_draintask(&sc->sc_ic, &sc->sc_es_task);
6334
6335         callout_drain(&sc->sc_led_blink_to);
6336         callout_drain(&sc->sc_watchdog_to);
6337         iwm_stop_device(sc);
6338         if (do_net80211) {
6339                 IWM_LOCK(sc);
6340                 iwm_xmit_queue_drain(sc);
6341                 IWM_UNLOCK(sc);
6342                 ieee80211_ifdetach(&sc->sc_ic);
6343         }
6344
6345         iwm_phy_db_free(sc->sc_phy_db);
6346         sc->sc_phy_db = NULL;
6347
6348         iwm_free_nvm_data(sc->nvm_data);
6349
6350         /* Free descriptor rings */
6351         iwm_free_rx_ring(sc, &sc->rxq);
6352         for (i = 0; i < nitems(sc->txq); i++)
6353                 iwm_free_tx_ring(sc, &sc->txq[i]);
6354
6355         /* Free firmware */
6356         if (fw->fw_fp != NULL)
6357                 iwm_fw_info_free(fw);
6358
6359         /* Free scheduler */
6360         iwm_dma_contig_free(&sc->sched_dma);
6361         iwm_dma_contig_free(&sc->ict_dma);
6362         iwm_dma_contig_free(&sc->kw_dma);
6363         iwm_dma_contig_free(&sc->fw_dma);
6364
6365         iwm_free_fw_paging(sc);
6366
6367         /* Finished with the hardware - detach things */
6368         iwm_pci_detach(dev);
6369
6370         if (sc->sc_notif_wait != NULL) {
6371                 iwm_notification_wait_free(sc->sc_notif_wait);
6372                 sc->sc_notif_wait = NULL;
6373         }
6374
6375         IWM_LOCK_DESTROY(sc);
6376
6377         return (0);
6378 }
6379
6380 static int
6381 iwm_detach(device_t dev)
6382 {
6383         struct iwm_softc *sc = device_get_softc(dev);
6384
6385         return (iwm_detach_local(sc, 1));
6386 }
6387
6388 static device_method_t iwm_pci_methods[] = {
6389         /* Device interface */
6390         DEVMETHOD(device_probe,         iwm_probe),
6391         DEVMETHOD(device_attach,        iwm_attach),
6392         DEVMETHOD(device_detach,        iwm_detach),
6393         DEVMETHOD(device_suspend,       iwm_suspend),
6394         DEVMETHOD(device_resume,        iwm_resume),
6395
6396         DEVMETHOD_END
6397 };
6398
6399 static driver_t iwm_pci_driver = {
6400         "iwm",
6401         iwm_pci_methods,
6402         sizeof (struct iwm_softc)
6403 };
6404
6405 static devclass_t iwm_devclass;
6406
6407 DRIVER_MODULE(iwm, pci, iwm_pci_driver, iwm_devclass, NULL, NULL);
6408 MODULE_PNP_INFO("U16:device;P:#;T:vendor=0x8086", pci, iwm_pci_driver,
6409     iwm_devices, nitems(iwm_devices));
6410 MODULE_DEPEND(iwm, firmware, 1, 1, 1);
6411 MODULE_DEPEND(iwm, pci, 1, 1, 1);
6412 MODULE_DEPEND(iwm, wlan, 1, 1, 1);