]> CyberLeo.Net >> Repos - FreeBSD/FreeBSD.git/blob - sys/dev/ath/if_ath_rx_edma.c
Merge ^/head r358131 through r358178.
[FreeBSD/FreeBSD.git] / sys / dev / ath / if_ath_rx_edma.c
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2012 Adrian Chadd <adrian@FreeBSD.org>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer,
12  *    without modification.
13  * 2. Redistributions in binary form must reproduce at minimum a disclaimer
14  *    similar to the "NO WARRANTY" disclaimer below ("Disclaimer") and any
15  *    redistribution must be conditioned upon including a substantially
16  *    similar Disclaimer requirement for further binary redistribution.
17  *
18  * NO WARRANTY
19  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21  * LIMITED TO, THE IMPLIED WARRANTIES OF NONINFRINGEMENT, MERCHANTIBILITY
22  * AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
23  * THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR SPECIAL, EXEMPLARY,
24  * OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER
27  * IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
29  * THE POSSIBILITY OF SUCH DAMAGES.
30  */
31
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
34
35 /*
36  * Driver for the Atheros Wireless LAN controller.
37  *
38  * This software is derived from work of Atsushi Onoe; his contribution
39  * is greatly appreciated.
40  */
41
42 #include "opt_inet.h"
43 #include "opt_ath.h"
44 /*
45  * This is needed for register operations which are performed
46  * by the driver - eg, calls to ath_hal_gettsf32().
47  *
48  * It's also required for any AH_DEBUG checks in here, eg the
49  * module dependencies.
50  */
51 #include "opt_ah.h"
52 #include "opt_wlan.h"
53
54 #include <sys/param.h>
55 #include <sys/systm.h>
56 #include <sys/sysctl.h>
57 #include <sys/mbuf.h>
58 #include <sys/malloc.h>
59 #include <sys/lock.h>
60 #include <sys/mutex.h>
61 #include <sys/kernel.h>
62 #include <sys/socket.h>
63 #include <sys/sockio.h>
64 #include <sys/errno.h>
65 #include <sys/callout.h>
66 #include <sys/bus.h>
67 #include <sys/endian.h>
68 #include <sys/kthread.h>
69 #include <sys/taskqueue.h>
70 #include <sys/priv.h>
71 #include <sys/module.h>
72 #include <sys/ktr.h>
73 #include <sys/smp.h>    /* for mp_ncpus */
74
75 #include <machine/bus.h>
76
77 #include <net/if.h>
78 #include <net/if_var.h>
79 #include <net/if_dl.h>
80 #include <net/if_media.h>
81 #include <net/if_types.h>
82 #include <net/if_arp.h>
83 #include <net/ethernet.h>
84 #include <net/if_llc.h>
85
86 #include <net80211/ieee80211_var.h>
87 #include <net80211/ieee80211_regdomain.h>
88 #ifdef IEEE80211_SUPPORT_SUPERG
89 #include <net80211/ieee80211_superg.h>
90 #endif
91 #ifdef IEEE80211_SUPPORT_TDMA
92 #include <net80211/ieee80211_tdma.h>
93 #endif
94
95 #include <net/bpf.h>
96
97 #ifdef INET
98 #include <netinet/in.h>
99 #include <netinet/if_ether.h>
100 #endif
101
102 #include <dev/ath/if_athvar.h>
103 #include <dev/ath/ath_hal/ah_devid.h>           /* XXX for softled */
104 #include <dev/ath/ath_hal/ah_diagcodes.h>
105
106 #include <dev/ath/if_ath_debug.h>
107 #include <dev/ath/if_ath_misc.h>
108 #include <dev/ath/if_ath_tsf.h>
109 #include <dev/ath/if_ath_tx.h>
110 #include <dev/ath/if_ath_sysctl.h>
111 #include <dev/ath/if_ath_led.h>
112 #include <dev/ath/if_ath_keycache.h>
113 #include <dev/ath/if_ath_rx.h>
114 #include <dev/ath/if_ath_beacon.h>
115 #include <dev/ath/if_athdfs.h>
116 #include <dev/ath/if_ath_descdma.h>
117
118 #ifdef ATH_TX99_DIAG
119 #include <dev/ath/ath_tx99/ath_tx99.h>
120 #endif
121
122 #include <dev/ath/if_ath_rx_edma.h>
123
124 #ifdef  ATH_DEBUG_ALQ
125 #include <dev/ath/if_ath_alq.h>
126 #endif
127
128 /*
129  * some general macros
130   */
131 #define INCR(_l, _sz)           (_l) ++; (_l) &= ((_sz) - 1)
132 #define DECR(_l, _sz)           (_l) --; (_l) &= ((_sz) - 1)
133
134 MALLOC_DECLARE(M_ATHDEV);
135
136 /*
137  * XXX TODO:
138  *
139  * + Make sure the FIFO is correctly flushed and reinitialised
140  *   through a reset;
141  * + Verify multi-descriptor frames work!
142  * + There's a "memory use after free" which needs to be tracked down
143  *   and fixed ASAP.  I've seen this in the legacy path too, so it
144  *   may be a generic RX path issue.
145  */
146
147 /*
148  * XXX shuffle the function orders so these pre-declarations aren't
149  * required!
150  */
151 static  int ath_edma_rxfifo_alloc(struct ath_softc *sc, HAL_RX_QUEUE qtype,
152             int nbufs);
153 static  int ath_edma_rxfifo_flush(struct ath_softc *sc, HAL_RX_QUEUE qtype);
154 static  void ath_edma_rxbuf_free(struct ath_softc *sc, struct ath_buf *bf);
155 static  void ath_edma_recv_proc_queue(struct ath_softc *sc,
156             HAL_RX_QUEUE qtype, int dosched);
157 static  int ath_edma_recv_proc_deferred_queue(struct ath_softc *sc,
158             HAL_RX_QUEUE qtype, int dosched);
159
160 static void
161 ath_edma_stoprecv(struct ath_softc *sc, int dodelay)
162 {
163         struct ath_hal *ah = sc->sc_ah;
164
165         ATH_RX_LOCK(sc);
166
167         ath_hal_stoppcurecv(ah);
168         ath_hal_setrxfilter(ah, 0);
169
170         /*
171          *
172          */
173         if (ath_hal_stopdmarecv(ah) == AH_TRUE)
174                 sc->sc_rx_stopped = 1;
175
176         /*
177          * Give the various bus FIFOs (not EDMA descriptor FIFO)
178          * time to finish flushing out data.
179          */
180         DELAY(3000);
181
182         /* Flush RX pending for each queue */
183         /* XXX should generic-ify this */
184         if (sc->sc_rxedma[HAL_RX_QUEUE_HP].m_rxpending) {
185                 m_freem(sc->sc_rxedma[HAL_RX_QUEUE_HP].m_rxpending);
186                 sc->sc_rxedma[HAL_RX_QUEUE_HP].m_rxpending = NULL;
187         }
188
189         if (sc->sc_rxedma[HAL_RX_QUEUE_LP].m_rxpending) {
190                 m_freem(sc->sc_rxedma[HAL_RX_QUEUE_LP].m_rxpending);
191                 sc->sc_rxedma[HAL_RX_QUEUE_LP].m_rxpending = NULL;
192         }
193         ATH_RX_UNLOCK(sc);
194 }
195
196 /*
197  * Re-initialise the FIFO given the current buffer contents.
198  * Specifically, walk from head -> tail, pushing the FIFO contents
199  * back into the FIFO.
200  */
201 static void
202 ath_edma_reinit_fifo(struct ath_softc *sc, HAL_RX_QUEUE qtype)
203 {
204         struct ath_rx_edma *re = &sc->sc_rxedma[qtype];
205         struct ath_buf *bf;
206         int i, j;
207
208         ATH_RX_LOCK_ASSERT(sc);
209
210         i = re->m_fifo_head;
211         for (j = 0; j < re->m_fifo_depth; j++) {
212                 bf = re->m_fifo[i];
213                 DPRINTF(sc, ATH_DEBUG_EDMA_RX,
214                     "%s: Q%d: pos=%i, addr=0x%jx\n",
215                     __func__,
216                     qtype,
217                     i,
218                     (uintmax_t)bf->bf_daddr);
219                 ath_hal_putrxbuf(sc->sc_ah, bf->bf_daddr, qtype);
220                 INCR(i, re->m_fifolen);
221         }
222
223         /* Ensure this worked out right */
224         if (i != re->m_fifo_tail) {
225                 device_printf(sc->sc_dev, "%s: i (%d) != tail! (%d)\n",
226                     __func__,
227                     i,
228                     re->m_fifo_tail);
229         }
230 }
231
232 /*
233  * Start receive.
234  */
235 static int
236 ath_edma_startrecv(struct ath_softc *sc)
237 {
238         struct ath_hal *ah = sc->sc_ah;
239
240         ATH_RX_LOCK(sc);
241
242         /*
243          * Sanity check - are we being called whilst RX
244          * isn't stopped?  If so, we may end up pushing
245          * too many entries into the RX FIFO and
246          * badness occurs.
247          */
248
249         /* Enable RX FIFO */
250         ath_hal_rxena(ah);
251
252         /*
253          * In theory the hardware has been initialised, right?
254          */
255         if (sc->sc_rx_resetted == 1) {
256                 DPRINTF(sc, ATH_DEBUG_EDMA_RX,
257                     "%s: Re-initing HP FIFO\n", __func__);
258                 ath_edma_reinit_fifo(sc, HAL_RX_QUEUE_HP);
259                 DPRINTF(sc, ATH_DEBUG_EDMA_RX,
260                     "%s: Re-initing LP FIFO\n", __func__);
261                 ath_edma_reinit_fifo(sc, HAL_RX_QUEUE_LP);
262                 sc->sc_rx_resetted = 0;
263         } else {
264                 device_printf(sc->sc_dev,
265                     "%s: called without resetting chip?\n",
266                     __func__);
267         }
268
269         /* Add up to m_fifolen entries in each queue */
270         /*
271          * These must occur after the above write so the FIFO buffers
272          * are pushed/tracked in the same order as the hardware will
273          * process them.
274          *
275          * XXX TODO: is this really necessary? We should've stopped
276          * the hardware already and reinitialised it, so it's a no-op.
277          */
278         ath_edma_rxfifo_alloc(sc, HAL_RX_QUEUE_HP,
279             sc->sc_rxedma[HAL_RX_QUEUE_HP].m_fifolen);
280
281         ath_edma_rxfifo_alloc(sc, HAL_RX_QUEUE_LP,
282             sc->sc_rxedma[HAL_RX_QUEUE_LP].m_fifolen);
283
284         ath_mode_init(sc);
285         ath_hal_startpcurecv(ah, (!! sc->sc_scanning));
286
287         /*
288          * We're now doing RX DMA!
289          */
290         sc->sc_rx_stopped = 0;
291
292         ATH_RX_UNLOCK(sc);
293
294         return (0);
295 }
296
297 static void
298 ath_edma_recv_sched_queue(struct ath_softc *sc, HAL_RX_QUEUE qtype,
299     int dosched)
300 {
301
302         ATH_LOCK(sc);
303         ath_power_set_power_state(sc, HAL_PM_AWAKE);
304         ATH_UNLOCK(sc);
305
306         ath_edma_recv_proc_queue(sc, qtype, dosched);
307
308         ATH_LOCK(sc);
309         ath_power_restore_power_state(sc);
310         ATH_UNLOCK(sc);
311
312         taskqueue_enqueue(sc->sc_tq, &sc->sc_rxtask);
313 }
314
315 static void
316 ath_edma_recv_sched(struct ath_softc *sc, int dosched)
317 {
318
319         ATH_LOCK(sc);
320         ath_power_set_power_state(sc, HAL_PM_AWAKE);
321         ATH_UNLOCK(sc);
322
323         ath_edma_recv_proc_queue(sc, HAL_RX_QUEUE_HP, dosched);
324         ath_edma_recv_proc_queue(sc, HAL_RX_QUEUE_LP, dosched);
325
326         ATH_LOCK(sc);
327         ath_power_restore_power_state(sc);
328         ATH_UNLOCK(sc);
329
330         taskqueue_enqueue(sc->sc_tq, &sc->sc_rxtask);
331 }
332
333 static void
334 ath_edma_recv_flush(struct ath_softc *sc)
335 {
336
337         DPRINTF(sc, ATH_DEBUG_RECV, "%s: called\n", __func__);
338
339         ATH_PCU_LOCK(sc);
340         sc->sc_rxproc_cnt++;
341         ATH_PCU_UNLOCK(sc);
342
343         ATH_LOCK(sc);
344         ath_power_set_power_state(sc, HAL_PM_AWAKE);
345         ATH_UNLOCK(sc);
346
347         /*
348          * Flush any active frames from FIFO -> deferred list
349          */
350         ath_edma_recv_proc_queue(sc, HAL_RX_QUEUE_HP, 0);
351         ath_edma_recv_proc_queue(sc, HAL_RX_QUEUE_LP, 0);
352
353         /*
354          * Process what's in the deferred queue
355          */
356         /*
357          * XXX: If we read the tsf/channoise here and then pass it in,
358          * we could restore the power state before processing
359          * the deferred queue.
360          */
361         ath_edma_recv_proc_deferred_queue(sc, HAL_RX_QUEUE_HP, 0);
362         ath_edma_recv_proc_deferred_queue(sc, HAL_RX_QUEUE_LP, 0);
363
364         ATH_LOCK(sc);
365         ath_power_restore_power_state(sc);
366         ATH_UNLOCK(sc);
367
368         ATH_PCU_LOCK(sc);
369         sc->sc_rxproc_cnt--;
370         ATH_PCU_UNLOCK(sc);
371 }
372
373 /*
374  * Process frames from the current queue into the deferred queue.
375  */
376 static void
377 ath_edma_recv_proc_queue(struct ath_softc *sc, HAL_RX_QUEUE qtype,
378     int dosched)
379 {
380         struct ath_rx_edma *re = &sc->sc_rxedma[qtype];
381         struct ath_rx_status *rs;
382         struct ath_desc *ds;
383         struct ath_buf *bf;
384         struct mbuf *m;
385         struct ath_hal *ah = sc->sc_ah;
386         uint64_t tsf;
387         uint16_t nf;
388         int npkts = 0;
389
390         tsf = ath_hal_gettsf64(ah);
391         nf = ath_hal_getchannoise(ah, sc->sc_curchan);
392         sc->sc_stats.ast_rx_noise = nf;
393
394         ATH_RX_LOCK(sc);
395
396 #if 1
397         if (sc->sc_rx_resetted == 1) {
398                 /*
399                  * XXX We shouldn't ever be scheduled if
400                  * receive has been stopped - so complain
401                  * loudly!
402                  */
403                 device_printf(sc->sc_dev,
404                     "%s: sc_rx_resetted=1! Bad!\n",
405                     __func__);
406                 ATH_RX_UNLOCK(sc);
407                 return;
408         }
409 #endif
410
411         do {
412                 bf = re->m_fifo[re->m_fifo_head];
413                 /* This shouldn't occur! */
414                 if (bf == NULL) {
415                         device_printf(sc->sc_dev, "%s: Q%d: NULL bf?\n",
416                             __func__,
417                             qtype);
418                         break;
419                 }
420                 m = bf->bf_m;
421                 ds = bf->bf_desc;
422
423                 /*
424                  * Sync descriptor memory - this also syncs the buffer for us.
425                  * EDMA descriptors are in cached memory.
426                  */
427                 bus_dmamap_sync(sc->sc_dmat, bf->bf_dmamap,
428                     BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
429                 rs = &bf->bf_status.ds_rxstat;
430                 bf->bf_rxstatus = ath_hal_rxprocdesc(ah, ds, bf->bf_daddr,
431                     NULL, rs);
432                 if (bf->bf_rxstatus == HAL_EINPROGRESS)
433                         break;
434 #ifdef  ATH_DEBUG
435                 if (sc->sc_debug & ATH_DEBUG_RECV_DESC)
436                         ath_printrxbuf(sc, bf, 0, bf->bf_rxstatus == HAL_OK);
437 #endif /* ATH_DEBUG */
438 #ifdef  ATH_DEBUG_ALQ
439                 if (if_ath_alq_checkdebug(&sc->sc_alq, ATH_ALQ_EDMA_RXSTATUS))
440                         if_ath_alq_post(&sc->sc_alq, ATH_ALQ_EDMA_RXSTATUS,
441                             sc->sc_rx_statuslen, (char *) ds);
442 #endif /* ATH_DEBUG */
443
444                 /*
445                  * Completed descriptor.
446                  */
447                 DPRINTF(sc, ATH_DEBUG_EDMA_RX,
448                     "%s: Q%d: completed!\n", __func__, qtype);
449                 npkts++;
450
451                 /*
452                  * We've been synced already, so unmap.
453                  */
454                 bus_dmamap_unload(sc->sc_dmat, bf->bf_dmamap);
455
456                 /*
457                  * Remove the FIFO entry and place it on the completion
458                  * queue.
459                  */
460                 re->m_fifo[re->m_fifo_head] = NULL;
461                 TAILQ_INSERT_TAIL(&sc->sc_rx_rxlist[qtype], bf, bf_list);
462
463                 /* Bump the descriptor FIFO stats */
464                 INCR(re->m_fifo_head, re->m_fifolen);
465                 re->m_fifo_depth--;
466                 /* XXX check it doesn't fall below 0 */
467         } while (re->m_fifo_depth > 0);
468
469         /* Append some more fresh frames to the FIFO */
470         if (dosched)
471                 ath_edma_rxfifo_alloc(sc, qtype, re->m_fifolen);
472
473         ATH_RX_UNLOCK(sc);
474
475         /* rx signal state monitoring */
476         ath_hal_rxmonitor(ah, &sc->sc_halstats, sc->sc_curchan);
477
478         ATH_KTR(sc, ATH_KTR_INTERRUPTS, 1,
479             "ath edma rx proc: npkts=%d\n",
480             npkts);
481
482         return;
483 }
484
485 /*
486  * Flush the deferred queue.
487  *
488  * This destructively flushes the deferred queue - it doesn't
489  * call the wireless stack on each mbuf.
490  */
491 static void
492 ath_edma_flush_deferred_queue(struct ath_softc *sc)
493 {
494         struct ath_buf *bf;
495
496         ATH_RX_LOCK_ASSERT(sc);
497
498         /* Free in one set, inside the lock */
499         while (! TAILQ_EMPTY(&sc->sc_rx_rxlist[HAL_RX_QUEUE_LP])) {
500                 bf = TAILQ_FIRST(&sc->sc_rx_rxlist[HAL_RX_QUEUE_LP]);
501                 TAILQ_REMOVE(&sc->sc_rx_rxlist[HAL_RX_QUEUE_LP], bf, bf_list);
502                 /* Free the buffer/mbuf */
503                 ath_edma_rxbuf_free(sc, bf);
504         }
505         while (! TAILQ_EMPTY(&sc->sc_rx_rxlist[HAL_RX_QUEUE_HP])) {
506                 bf = TAILQ_FIRST(&sc->sc_rx_rxlist[HAL_RX_QUEUE_HP]);
507                 TAILQ_REMOVE(&sc->sc_rx_rxlist[HAL_RX_QUEUE_HP], bf, bf_list);
508                 /* Free the buffer/mbuf */
509                 ath_edma_rxbuf_free(sc, bf);
510         }
511 }
512
513 static int
514 ath_edma_recv_proc_deferred_queue(struct ath_softc *sc, HAL_RX_QUEUE qtype,
515     int dosched)
516 {
517         int ngood = 0;
518         uint64_t tsf;
519         struct ath_buf *bf, *next;
520         struct ath_rx_status *rs;
521         int16_t nf;
522         ath_bufhead rxlist;
523         struct mbuf *m;
524         struct epoch_tracker et;
525
526         TAILQ_INIT(&rxlist);
527
528         nf = ath_hal_getchannoise(sc->sc_ah, sc->sc_curchan);
529         /*
530          * XXX TODO: the NF/TSF should be stamped on the bufs themselves,
531          * otherwise we may end up adding in the wrong values if this
532          * is delayed too far..
533          */
534         tsf = ath_hal_gettsf64(sc->sc_ah);
535
536         /* Copy the list over */
537         ATH_RX_LOCK(sc);
538         TAILQ_CONCAT(&rxlist, &sc->sc_rx_rxlist[qtype], bf_list);
539         ATH_RX_UNLOCK(sc);
540
541         NET_EPOCH_ENTER(et);
542
543         /* Handle the completed descriptors */
544         /*
545          * XXX is this SAFE call needed? The ath_buf entries
546          * aren't modified by ath_rx_pkt, right?
547          */
548         TAILQ_FOREACH_SAFE(bf, &rxlist, bf_list, next) {
549                 /*
550                  * Skip the RX descriptor status - start at the data offset
551                  */
552                 m_adj(bf->bf_m, sc->sc_rx_statuslen);
553
554                 /* Handle the frame */
555
556                 rs = &bf->bf_status.ds_rxstat;
557                 m = bf->bf_m;
558                 bf->bf_m = NULL;
559                 if (ath_rx_pkt(sc, rs, bf->bf_rxstatus, tsf, nf, qtype, bf, m))
560                         ngood++;
561         }
562
563         if (ngood) {
564                 sc->sc_lastrx = tsf;
565         }
566         NET_EPOCH_EXIT(et);
567
568         ATH_KTR(sc, ATH_KTR_INTERRUPTS, 1,
569             "ath edma rx deferred proc: ngood=%d\n",
570             ngood);
571
572         /* Free in one set, inside the lock */
573         ATH_RX_LOCK(sc);
574         while (! TAILQ_EMPTY(&rxlist)) {
575                 bf = TAILQ_FIRST(&rxlist);
576                 TAILQ_REMOVE(&rxlist, bf, bf_list);
577                 /* Free the buffer/mbuf */
578                 ath_edma_rxbuf_free(sc, bf);
579         }
580         ATH_RX_UNLOCK(sc);
581
582         return (ngood);
583 }
584
585 static void
586 ath_edma_recv_tasklet(void *arg, int npending)
587 {
588         struct ath_softc *sc = (struct ath_softc *) arg;
589 #ifdef  IEEE80211_SUPPORT_SUPERG
590         struct ieee80211com *ic = &sc->sc_ic;
591 #endif
592
593         DPRINTF(sc, ATH_DEBUG_EDMA_RX, "%s: called; npending=%d\n",
594             __func__,
595             npending);
596
597         ATH_PCU_LOCK(sc);
598         if (sc->sc_inreset_cnt > 0) {
599                 device_printf(sc->sc_dev, "%s: sc_inreset_cnt > 0; skipping\n",
600                     __func__);
601                 ATH_PCU_UNLOCK(sc);
602                 return;
603         }
604         sc->sc_rxproc_cnt++;
605         ATH_PCU_UNLOCK(sc);
606
607         ATH_LOCK(sc);
608         ath_power_set_power_state(sc, HAL_PM_AWAKE);
609         ATH_UNLOCK(sc);
610
611         ath_edma_recv_proc_queue(sc, HAL_RX_QUEUE_HP, 1);
612         ath_edma_recv_proc_queue(sc, HAL_RX_QUEUE_LP, 1);
613
614         ath_edma_recv_proc_deferred_queue(sc, HAL_RX_QUEUE_HP, 1);
615         ath_edma_recv_proc_deferred_queue(sc, HAL_RX_QUEUE_LP, 1);
616
617         /*
618          * XXX: If we read the tsf/channoise here and then pass it in,
619          * we could restore the power state before processing
620          * the deferred queue.
621          */
622         ATH_LOCK(sc);
623         ath_power_restore_power_state(sc);
624         ATH_UNLOCK(sc);
625
626 #ifdef  IEEE80211_SUPPORT_SUPERG
627         ieee80211_ff_age_all(ic, 100);
628 #endif
629         if (ath_dfs_tasklet_needed(sc, sc->sc_curchan))
630                 taskqueue_enqueue(sc->sc_tq, &sc->sc_dfstask);
631
632         ATH_PCU_LOCK(sc);
633         sc->sc_rxproc_cnt--;
634         ATH_PCU_UNLOCK(sc);
635 }
636
637 /*
638  * Allocate an RX mbuf for the given ath_buf and initialise
639  * it for EDMA.
640  *
641  * + Allocate a 4KB mbuf;
642  * + Setup the DMA map for the given buffer;
643  * + Return that.
644  */
645 static int
646 ath_edma_rxbuf_init(struct ath_softc *sc, struct ath_buf *bf)
647 {
648
649         struct mbuf *m;
650         int error;
651         int len;
652
653         ATH_RX_LOCK_ASSERT(sc);
654
655         m = m_getm(NULL, sc->sc_edma_bufsize, M_NOWAIT, MT_DATA);
656         if (! m)
657                 return (ENOBUFS);               /* XXX ?*/
658
659         /* XXX warn/enforce alignment */
660
661         len = m->m_ext.ext_size;
662 #if 0
663         device_printf(sc->sc_dev, "%s: called: m=%p, size=%d, mtod=%p\n",
664             __func__,
665             m,
666             len,
667             mtod(m, char *));
668 #endif
669
670         m->m_pkthdr.len = m->m_len = m->m_ext.ext_size;
671
672         /*
673          * Populate ath_buf fields.
674          */
675         bf->bf_desc = mtod(m, struct ath_desc *);
676         bf->bf_lastds = bf->bf_desc;    /* XXX only really for TX? */
677         bf->bf_m = m;
678
679         /*
680          * Zero the descriptor and ensure it makes it out to the
681          * bounce buffer if one is required.
682          *
683          * XXX PREWRITE will copy the whole buffer; we only needed it
684          * to sync the first 32 DWORDS.  Oh well.
685          */
686         memset(bf->bf_desc, '\0', sc->sc_rx_statuslen);
687
688         /*
689          * Create DMA mapping.
690          */
691         error = bus_dmamap_load_mbuf_sg(sc->sc_dmat,
692             bf->bf_dmamap, m, bf->bf_segs, &bf->bf_nseg, BUS_DMA_NOWAIT);
693
694         if (error != 0) {
695                 device_printf(sc->sc_dev, "%s: failed; error=%d\n",
696                     __func__,
697                     error);
698                 m_freem(m);
699                 return (error);
700         }
701
702         /*
703          * Set daddr to the physical mapping page.
704          */
705         bf->bf_daddr = bf->bf_segs[0].ds_addr;
706
707         /*
708          * Prepare for the upcoming read.
709          *
710          * We need to both sync some data into the buffer (the zero'ed
711          * descriptor payload) and also prepare for the read that's going
712          * to occur.
713          */
714         bus_dmamap_sync(sc->sc_dmat, bf->bf_dmamap,
715             BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
716
717         /* Finish! */
718         return (0);
719 }
720
721 /*
722  * Allocate a RX buffer.
723  */
724 static struct ath_buf *
725 ath_edma_rxbuf_alloc(struct ath_softc *sc)
726 {
727         struct ath_buf *bf;
728         int error;
729
730         ATH_RX_LOCK_ASSERT(sc);
731
732         /* Allocate buffer */
733         bf = TAILQ_FIRST(&sc->sc_rxbuf);
734         /* XXX shouldn't happen upon startup? */
735         if (bf == NULL) {
736                 DPRINTF(sc, ATH_DEBUG_EDMA_RX, "%s: nothing on rxbuf?!\n",
737                     __func__);
738                 return (NULL);
739         }
740
741         /* Remove it from the free list */
742         TAILQ_REMOVE(&sc->sc_rxbuf, bf, bf_list);
743
744         /* Assign RX mbuf to it */
745         error = ath_edma_rxbuf_init(sc, bf);
746         if (error != 0) {
747                 device_printf(sc->sc_dev,
748                     "%s: bf=%p, rxbuf alloc failed! error=%d\n",
749                     __func__,
750                     bf,
751                     error);
752                 TAILQ_INSERT_TAIL(&sc->sc_rxbuf, bf, bf_list);
753                 return (NULL);
754         }
755
756         return (bf);
757 }
758
759 static void
760 ath_edma_rxbuf_free(struct ath_softc *sc, struct ath_buf *bf)
761 {
762
763         ATH_RX_LOCK_ASSERT(sc);
764
765         /*
766          * Only unload the frame if we haven't consumed
767          * the mbuf via ath_rx_pkt().
768          */
769         if (bf->bf_m) {
770                 bus_dmamap_unload(sc->sc_dmat, bf->bf_dmamap);
771                 m_freem(bf->bf_m);
772                 bf->bf_m = NULL;
773         }
774
775         /* XXX lock? */
776         TAILQ_INSERT_TAIL(&sc->sc_rxbuf, bf, bf_list);
777 }
778
779 /*
780  * Allocate up to 'n' entries and push them onto the hardware FIFO.
781  *
782  * Return how many entries were successfully pushed onto the
783  * FIFO.
784  */
785 static int
786 ath_edma_rxfifo_alloc(struct ath_softc *sc, HAL_RX_QUEUE qtype, int nbufs)
787 {
788         struct ath_rx_edma *re = &sc->sc_rxedma[qtype];
789         struct ath_buf *bf;
790         int i;
791
792         ATH_RX_LOCK_ASSERT(sc);
793
794         /*
795          * Allocate buffers until the FIFO is full or nbufs is reached.
796          */
797         for (i = 0; i < nbufs && re->m_fifo_depth < re->m_fifolen; i++) {
798                 /* Ensure the FIFO is already blank, complain loudly! */
799                 if (re->m_fifo[re->m_fifo_tail] != NULL) {
800                         device_printf(sc->sc_dev,
801                             "%s: Q%d: fifo[%d] != NULL (%p)\n",
802                             __func__,
803                             qtype,
804                             re->m_fifo_tail,
805                             re->m_fifo[re->m_fifo_tail]);
806
807                         /* Free the slot */
808                         ath_edma_rxbuf_free(sc, re->m_fifo[re->m_fifo_tail]);
809                         re->m_fifo_depth--;
810                         /* XXX check it's not < 0 */
811                         re->m_fifo[re->m_fifo_tail] = NULL;
812                 }
813
814                 bf = ath_edma_rxbuf_alloc(sc);
815                 /* XXX should ensure the FIFO is not NULL? */
816                 if (bf == NULL) {
817                         DPRINTF(sc, ATH_DEBUG_EDMA_RX,
818                             "%s: Q%d: alloc failed: i=%d, nbufs=%d?\n",
819                             __func__,
820                             qtype,
821                             i,
822                             nbufs);
823                         break;
824                 }
825
826                 re->m_fifo[re->m_fifo_tail] = bf;
827
828                 /* Write to the RX FIFO */
829                 DPRINTF(sc, ATH_DEBUG_EDMA_RX,
830                     "%s: Q%d: putrxbuf=%p (0x%jx)\n",
831                     __func__,
832                     qtype,
833                     bf->bf_desc,
834                     (uintmax_t) bf->bf_daddr);
835                 ath_hal_putrxbuf(sc->sc_ah, bf->bf_daddr, qtype);
836
837                 re->m_fifo_depth++;
838                 INCR(re->m_fifo_tail, re->m_fifolen);
839         }
840
841         /*
842          * Return how many were allocated.
843          */
844         DPRINTF(sc, ATH_DEBUG_EDMA_RX, "%s: Q%d: nbufs=%d, nalloced=%d\n",
845             __func__,
846             qtype,
847             nbufs,
848             i);
849         return (i);
850 }
851
852 static int
853 ath_edma_rxfifo_flush(struct ath_softc *sc, HAL_RX_QUEUE qtype)
854 {
855         struct ath_rx_edma *re = &sc->sc_rxedma[qtype];
856         int i;
857
858         ATH_RX_LOCK_ASSERT(sc);
859
860         for (i = 0; i < re->m_fifolen; i++) {
861                 if (re->m_fifo[i] != NULL) {
862 #ifdef  ATH_DEBUG
863                         struct ath_buf *bf = re->m_fifo[i];
864
865                         if (sc->sc_debug & ATH_DEBUG_RECV_DESC)
866                                 ath_printrxbuf(sc, bf, 0, HAL_OK);
867 #endif
868                         ath_edma_rxbuf_free(sc, re->m_fifo[i]);
869                         re->m_fifo[i] = NULL;
870                         re->m_fifo_depth--;
871                 }
872         }
873
874         if (re->m_rxpending != NULL) {
875                 m_freem(re->m_rxpending);
876                 re->m_rxpending = NULL;
877         }
878         re->m_fifo_head = re->m_fifo_tail = re->m_fifo_depth = 0;
879
880         return (0);
881 }
882
883 /*
884  * Setup the initial RX FIFO structure.
885  */
886 static int
887 ath_edma_setup_rxfifo(struct ath_softc *sc, HAL_RX_QUEUE qtype)
888 {
889         struct ath_rx_edma *re = &sc->sc_rxedma[qtype];
890
891         ATH_RX_LOCK_ASSERT(sc);
892
893         if (! ath_hal_getrxfifodepth(sc->sc_ah, qtype, &re->m_fifolen)) {
894                 device_printf(sc->sc_dev, "%s: qtype=%d, failed\n",
895                     __func__,
896                     qtype);
897                 return (-EINVAL);
898         }
899
900         if (bootverbose)
901                 device_printf(sc->sc_dev,
902                     "%s: type=%d, FIFO depth = %d entries\n",
903                     __func__,
904                     qtype,
905                     re->m_fifolen);
906
907         /* Allocate ath_buf FIFO array, pre-zero'ed */
908         re->m_fifo = malloc(sizeof(struct ath_buf *) * re->m_fifolen,
909             M_ATHDEV,
910             M_NOWAIT | M_ZERO);
911         if (re->m_fifo == NULL) {
912                 device_printf(sc->sc_dev, "%s: malloc failed\n",
913                     __func__);
914                 return (-ENOMEM);
915         }
916
917         /*
918          * Set initial "empty" state.
919          */
920         re->m_rxpending = NULL;
921         re->m_fifo_head = re->m_fifo_tail = re->m_fifo_depth = 0;
922
923         return (0);
924 }
925
926 static int
927 ath_edma_rxfifo_free(struct ath_softc *sc, HAL_RX_QUEUE qtype)
928 {
929         struct ath_rx_edma *re = &sc->sc_rxedma[qtype];
930
931         device_printf(sc->sc_dev, "%s: called; qtype=%d\n",
932             __func__,
933             qtype);
934
935         free(re->m_fifo, M_ATHDEV);
936
937         return (0);
938 }
939
940 static int
941 ath_edma_dma_rxsetup(struct ath_softc *sc)
942 {
943         int error;
944
945         /*
946          * Create RX DMA tag and buffers.
947          */
948         error = ath_descdma_setup_rx_edma(sc, &sc->sc_rxdma, &sc->sc_rxbuf,
949             "rx", ath_rxbuf, sc->sc_rx_statuslen);
950         if (error != 0)
951                 return error;
952
953         ATH_RX_LOCK(sc);
954         (void) ath_edma_setup_rxfifo(sc, HAL_RX_QUEUE_HP);
955         (void) ath_edma_setup_rxfifo(sc, HAL_RX_QUEUE_LP);
956         ATH_RX_UNLOCK(sc);
957
958         return (0);
959 }
960
961 static int
962 ath_edma_dma_rxteardown(struct ath_softc *sc)
963 {
964
965         ATH_RX_LOCK(sc);
966         ath_edma_flush_deferred_queue(sc);
967         ath_edma_rxfifo_flush(sc, HAL_RX_QUEUE_HP);
968         ath_edma_rxfifo_free(sc, HAL_RX_QUEUE_HP);
969
970         ath_edma_rxfifo_flush(sc, HAL_RX_QUEUE_LP);
971         ath_edma_rxfifo_free(sc, HAL_RX_QUEUE_LP);
972         ATH_RX_UNLOCK(sc);
973
974         /* Free RX ath_buf */
975         /* Free RX DMA tag */
976         if (sc->sc_rxdma.dd_desc_len != 0)
977                 ath_descdma_cleanup(sc, &sc->sc_rxdma, &sc->sc_rxbuf);
978
979         return (0);
980 }
981
982 void
983 ath_recv_setup_edma(struct ath_softc *sc)
984 {
985
986         /* Set buffer size to 4k */
987         sc->sc_edma_bufsize = 4096;
988
989         /* Fetch EDMA field and buffer sizes */
990         (void) ath_hal_getrxstatuslen(sc->sc_ah, &sc->sc_rx_statuslen);
991
992         /* Configure the hardware with the RX buffer size */
993         (void) ath_hal_setrxbufsize(sc->sc_ah, sc->sc_edma_bufsize -
994             sc->sc_rx_statuslen);
995
996         if (bootverbose) {
997                 device_printf(sc->sc_dev, "RX status length: %d\n",
998                     sc->sc_rx_statuslen);
999                 device_printf(sc->sc_dev, "RX buffer size: %d\n",
1000                     sc->sc_edma_bufsize);
1001         }
1002
1003         sc->sc_rx.recv_stop = ath_edma_stoprecv;
1004         sc->sc_rx.recv_start = ath_edma_startrecv;
1005         sc->sc_rx.recv_flush = ath_edma_recv_flush;
1006         sc->sc_rx.recv_tasklet = ath_edma_recv_tasklet;
1007         sc->sc_rx.recv_rxbuf_init = ath_edma_rxbuf_init;
1008
1009         sc->sc_rx.recv_setup = ath_edma_dma_rxsetup;
1010         sc->sc_rx.recv_teardown = ath_edma_dma_rxteardown;
1011
1012         sc->sc_rx.recv_sched = ath_edma_recv_sched;
1013         sc->sc_rx.recv_sched_queue = ath_edma_recv_sched_queue;
1014 }