dma.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677
  1. // SPDX-License-Identifier: ISC
  2. /*
  3. * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
  4. */
  5. #include <linux/dma-mapping.h>
  6. #include "mt76.h"
  7. #include "dma.h"
  8. static struct mt76_txwi_cache *
  9. mt76_alloc_txwi(struct mt76_dev *dev)
  10. {
  11. struct mt76_txwi_cache *t;
  12. dma_addr_t addr;
  13. u8 *txwi;
  14. int size;
  15. size = L1_CACHE_ALIGN(dev->drv->txwi_size + sizeof(*t));
  16. txwi = devm_kzalloc(dev->dev, size, GFP_ATOMIC);
  17. if (!txwi)
  18. return NULL;
  19. addr = dma_map_single(dev->dev, txwi, dev->drv->txwi_size,
  20. DMA_TO_DEVICE);
  21. t = (struct mt76_txwi_cache *)(txwi + dev->drv->txwi_size);
  22. t->dma_addr = addr;
  23. return t;
  24. }
  25. static struct mt76_txwi_cache *
  26. __mt76_get_txwi(struct mt76_dev *dev)
  27. {
  28. struct mt76_txwi_cache *t = NULL;
  29. spin_lock(&dev->lock);
  30. if (!list_empty(&dev->txwi_cache)) {
  31. t = list_first_entry(&dev->txwi_cache, struct mt76_txwi_cache,
  32. list);
  33. list_del(&t->list);
  34. }
  35. spin_unlock(&dev->lock);
  36. return t;
  37. }
  38. static struct mt76_txwi_cache *
  39. mt76_get_txwi(struct mt76_dev *dev)
  40. {
  41. struct mt76_txwi_cache *t = __mt76_get_txwi(dev);
  42. if (t)
  43. return t;
  44. return mt76_alloc_txwi(dev);
  45. }
  46. void
  47. mt76_put_txwi(struct mt76_dev *dev, struct mt76_txwi_cache *t)
  48. {
  49. if (!t)
  50. return;
  51. spin_lock(&dev->lock);
  52. list_add(&t->list, &dev->txwi_cache);
  53. spin_unlock(&dev->lock);
  54. }
  55. EXPORT_SYMBOL_GPL(mt76_put_txwi);
  56. static void
  57. mt76_free_pending_txwi(struct mt76_dev *dev)
  58. {
  59. struct mt76_txwi_cache *t;
  60. local_bh_disable();
  61. while ((t = __mt76_get_txwi(dev)) != NULL)
  62. dma_unmap_single(dev->dev, t->dma_addr, dev->drv->txwi_size,
  63. DMA_TO_DEVICE);
  64. local_bh_enable();
  65. }
  66. static int
  67. mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q,
  68. int idx, int n_desc, int bufsize,
  69. u32 ring_base)
  70. {
  71. int size;
  72. int i;
  73. spin_lock_init(&q->lock);
  74. spin_lock_init(&q->cleanup_lock);
  75. q->regs = dev->mmio.regs + ring_base + idx * MT_RING_SIZE;
  76. q->ndesc = n_desc;
  77. q->buf_size = bufsize;
  78. q->hw_idx = idx;
  79. size = q->ndesc * sizeof(struct mt76_desc);
  80. q->desc = dmam_alloc_coherent(dev->dev, size, &q->desc_dma, GFP_KERNEL);
  81. if (!q->desc)
  82. return -ENOMEM;
  83. size = q->ndesc * sizeof(*q->entry);
  84. q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL);
  85. if (!q->entry)
  86. return -ENOMEM;
  87. /* clear descriptors */
  88. for (i = 0; i < q->ndesc; i++)
  89. q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
  90. writel(q->desc_dma, &q->regs->desc_base);
  91. writel(0, &q->regs->cpu_idx);
  92. writel(0, &q->regs->dma_idx);
  93. writel(q->ndesc, &q->regs->ring_size);
  94. return 0;
  95. }
  96. static int
  97. mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q,
  98. struct mt76_queue_buf *buf, int nbufs, u32 info,
  99. struct sk_buff *skb, void *txwi)
  100. {
  101. struct mt76_queue_entry *entry;
  102. struct mt76_desc *desc;
  103. u32 ctrl;
  104. int i, idx = -1;
  105. if (txwi) {
  106. q->entry[q->head].txwi = DMA_DUMMY_DATA;
  107. q->entry[q->head].skip_buf0 = true;
  108. }
  109. for (i = 0; i < nbufs; i += 2, buf += 2) {
  110. u32 buf0 = buf[0].addr, buf1 = 0;
  111. idx = q->head;
  112. q->head = (q->head + 1) % q->ndesc;
  113. desc = &q->desc[idx];
  114. entry = &q->entry[idx];
  115. if (buf[0].skip_unmap)
  116. entry->skip_buf0 = true;
  117. entry->skip_buf1 = i == nbufs - 1;
  118. entry->dma_addr[0] = buf[0].addr;
  119. entry->dma_len[0] = buf[0].len;
  120. ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len);
  121. if (i < nbufs - 1) {
  122. entry->dma_addr[1] = buf[1].addr;
  123. entry->dma_len[1] = buf[1].len;
  124. buf1 = buf[1].addr;
  125. ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len);
  126. if (buf[1].skip_unmap)
  127. entry->skip_buf1 = true;
  128. }
  129. if (i == nbufs - 1)
  130. ctrl |= MT_DMA_CTL_LAST_SEC0;
  131. else if (i == nbufs - 2)
  132. ctrl |= MT_DMA_CTL_LAST_SEC1;
  133. WRITE_ONCE(desc->buf0, cpu_to_le32(buf0));
  134. WRITE_ONCE(desc->buf1, cpu_to_le32(buf1));
  135. WRITE_ONCE(desc->info, cpu_to_le32(info));
  136. WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl));
  137. q->queued++;
  138. }
  139. q->entry[idx].txwi = txwi;
  140. q->entry[idx].skb = skb;
  141. return idx;
  142. }
  143. static void
  144. mt76_dma_tx_cleanup_idx(struct mt76_dev *dev, struct mt76_queue *q, int idx,
  145. struct mt76_queue_entry *prev_e)
  146. {
  147. struct mt76_queue_entry *e = &q->entry[idx];
  148. if (!e->skip_buf0)
  149. dma_unmap_single(dev->dev, e->dma_addr[0], e->dma_len[0],
  150. DMA_TO_DEVICE);
  151. if (!e->skip_buf1)
  152. dma_unmap_single(dev->dev, e->dma_addr[1], e->dma_len[1],
  153. DMA_TO_DEVICE);
  154. if (e->txwi == DMA_DUMMY_DATA)
  155. e->txwi = NULL;
  156. if (e->skb == DMA_DUMMY_DATA)
  157. e->skb = NULL;
  158. *prev_e = *e;
  159. memset(e, 0, sizeof(*e));
  160. }
  161. static void
  162. mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q)
  163. {
  164. writel(q->desc_dma, &q->regs->desc_base);
  165. writel(q->ndesc, &q->regs->ring_size);
  166. q->head = readl(&q->regs->dma_idx);
  167. q->tail = q->head;
  168. }
  169. static void
  170. mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q)
  171. {
  172. wmb();
  173. writel(q->head, &q->regs->cpu_idx);
  174. }
  175. static void
  176. mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush)
  177. {
  178. struct mt76_queue_entry entry;
  179. int last;
  180. if (!q)
  181. return;
  182. spin_lock_bh(&q->cleanup_lock);
  183. if (flush)
  184. last = -1;
  185. else
  186. last = readl(&q->regs->dma_idx);
  187. while (q->queued > 0 && q->tail != last) {
  188. mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry);
  189. mt76_queue_tx_complete(dev, q, &entry);
  190. if (entry.txwi) {
  191. if (!(dev->drv->drv_flags & MT_DRV_TXWI_NO_FREE))
  192. mt76_put_txwi(dev, entry.txwi);
  193. }
  194. if (!flush && q->tail == last)
  195. last = readl(&q->regs->dma_idx);
  196. }
  197. spin_unlock_bh(&q->cleanup_lock);
  198. if (flush) {
  199. spin_lock_bh(&q->lock);
  200. mt76_dma_sync_idx(dev, q);
  201. mt76_dma_kick_queue(dev, q);
  202. spin_unlock_bh(&q->lock);
  203. }
  204. if (!q->queued)
  205. wake_up(&dev->tx_wait);
  206. }
  207. static void *
  208. mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx,
  209. int *len, u32 *info, bool *more)
  210. {
  211. struct mt76_queue_entry *e = &q->entry[idx];
  212. struct mt76_desc *desc = &q->desc[idx];
  213. dma_addr_t buf_addr;
  214. void *buf = e->buf;
  215. int buf_len = SKB_WITH_OVERHEAD(q->buf_size);
  216. buf_addr = e->dma_addr[0];
  217. if (len) {
  218. u32 ctl = le32_to_cpu(READ_ONCE(desc->ctrl));
  219. *len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctl);
  220. *more = !(ctl & MT_DMA_CTL_LAST_SEC0);
  221. }
  222. if (info)
  223. *info = le32_to_cpu(desc->info);
  224. dma_unmap_single(dev->dev, buf_addr, buf_len, DMA_FROM_DEVICE);
  225. e->buf = NULL;
  226. return buf;
  227. }
  228. static void *
  229. mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush,
  230. int *len, u32 *info, bool *more)
  231. {
  232. int idx = q->tail;
  233. *more = false;
  234. if (!q->queued)
  235. return NULL;
  236. if (flush)
  237. q->desc[idx].ctrl |= cpu_to_le32(MT_DMA_CTL_DMA_DONE);
  238. else if (!(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE)))
  239. return NULL;
  240. q->tail = (q->tail + 1) % q->ndesc;
  241. q->queued--;
  242. return mt76_dma_get_buf(dev, q, idx, len, info, more);
  243. }
  244. static int
  245. mt76_dma_tx_queue_skb_raw(struct mt76_dev *dev, struct mt76_queue *q,
  246. struct sk_buff *skb, u32 tx_info)
  247. {
  248. struct mt76_queue_buf buf;
  249. dma_addr_t addr;
  250. if (q->queued + 1 >= q->ndesc - 1)
  251. goto error;
  252. addr = dma_map_single(dev->dev, skb->data, skb->len,
  253. DMA_TO_DEVICE);
  254. if (unlikely(dma_mapping_error(dev->dev, addr)))
  255. goto error;
  256. buf.addr = addr;
  257. buf.len = skb->len;
  258. spin_lock_bh(&q->lock);
  259. mt76_dma_add_buf(dev, q, &buf, 1, tx_info, skb, NULL);
  260. mt76_dma_kick_queue(dev, q);
  261. spin_unlock_bh(&q->lock);
  262. return 0;
  263. error:
  264. dev_kfree_skb(skb);
  265. return -ENOMEM;
  266. }
  267. static int
  268. mt76_dma_tx_queue_skb(struct mt76_dev *dev, struct mt76_queue *q,
  269. struct sk_buff *skb, struct mt76_wcid *wcid,
  270. struct ieee80211_sta *sta)
  271. {
  272. struct mt76_tx_info tx_info = {
  273. .skb = skb,
  274. };
  275. struct ieee80211_hw *hw;
  276. int len, n = 0, ret = -ENOMEM;
  277. struct mt76_txwi_cache *t;
  278. struct sk_buff *iter;
  279. dma_addr_t addr;
  280. u8 *txwi;
  281. t = mt76_get_txwi(dev);
  282. if (!t) {
  283. hw = mt76_tx_status_get_hw(dev, skb);
  284. ieee80211_free_txskb(hw, skb);
  285. return -ENOMEM;
  286. }
  287. txwi = mt76_get_txwi_ptr(dev, t);
  288. skb->prev = skb->next = NULL;
  289. if (dev->drv->drv_flags & MT_DRV_TX_ALIGNED4_SKBS)
  290. mt76_insert_hdr_pad(skb);
  291. len = skb_headlen(skb);
  292. addr = dma_map_single(dev->dev, skb->data, len, DMA_TO_DEVICE);
  293. if (unlikely(dma_mapping_error(dev->dev, addr)))
  294. goto free;
  295. tx_info.buf[n].addr = t->dma_addr;
  296. tx_info.buf[n++].len = dev->drv->txwi_size;
  297. tx_info.buf[n].addr = addr;
  298. tx_info.buf[n++].len = len;
  299. skb_walk_frags(skb, iter) {
  300. if (n == ARRAY_SIZE(tx_info.buf))
  301. goto unmap;
  302. addr = dma_map_single(dev->dev, iter->data, iter->len,
  303. DMA_TO_DEVICE);
  304. if (unlikely(dma_mapping_error(dev->dev, addr)))
  305. goto unmap;
  306. tx_info.buf[n].addr = addr;
  307. tx_info.buf[n++].len = iter->len;
  308. }
  309. tx_info.nbuf = n;
  310. if (q->queued + (tx_info.nbuf + 1) / 2 >= q->ndesc - 1) {
  311. ret = -ENOMEM;
  312. goto unmap;
  313. }
  314. dma_sync_single_for_cpu(dev->dev, t->dma_addr, dev->drv->txwi_size,
  315. DMA_TO_DEVICE);
  316. ret = dev->drv->tx_prepare_skb(dev, txwi, q->qid, wcid, sta, &tx_info);
  317. dma_sync_single_for_device(dev->dev, t->dma_addr, dev->drv->txwi_size,
  318. DMA_TO_DEVICE);
  319. if (ret < 0)
  320. goto unmap;
  321. return mt76_dma_add_buf(dev, q, tx_info.buf, tx_info.nbuf,
  322. tx_info.info, tx_info.skb, t);
  323. unmap:
  324. for (n--; n > 0; n--)
  325. dma_unmap_single(dev->dev, tx_info.buf[n].addr,
  326. tx_info.buf[n].len, DMA_TO_DEVICE);
  327. free:
  328. #ifdef CONFIG_NL80211_TESTMODE
  329. /* fix tx_done accounting on queue overflow */
  330. if (mt76_is_testmode_skb(dev, skb, &hw)) {
  331. struct mt76_phy *phy = hw->priv;
  332. if (tx_info.skb == phy->test.tx_skb)
  333. phy->test.tx_done--;
  334. }
  335. #endif
  336. dev_kfree_skb(tx_info.skb);
  337. mt76_put_txwi(dev, t);
  338. return ret;
  339. }
  340. static int
  341. mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q)
  342. {
  343. dma_addr_t addr;
  344. void *buf;
  345. int frames = 0;
  346. int len = SKB_WITH_OVERHEAD(q->buf_size);
  347. int offset = q->buf_offset;
  348. spin_lock_bh(&q->lock);
  349. while (q->queued < q->ndesc - 1) {
  350. struct mt76_queue_buf qbuf;
  351. buf = page_frag_alloc(&q->rx_page, q->buf_size, GFP_ATOMIC);
  352. if (!buf)
  353. break;
  354. addr = dma_map_single(dev->dev, buf, len, DMA_FROM_DEVICE);
  355. if (unlikely(dma_mapping_error(dev->dev, addr))) {
  356. skb_free_frag(buf);
  357. break;
  358. }
  359. qbuf.addr = addr + offset;
  360. qbuf.len = len - offset;
  361. mt76_dma_add_buf(dev, q, &qbuf, 1, 0, buf, NULL);
  362. frames++;
  363. }
  364. if (frames)
  365. mt76_dma_kick_queue(dev, q);
  366. spin_unlock_bh(&q->lock);
  367. return frames;
  368. }
  369. static void
  370. mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
  371. {
  372. struct page *page;
  373. void *buf;
  374. bool more;
  375. spin_lock_bh(&q->lock);
  376. do {
  377. buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more);
  378. if (!buf)
  379. break;
  380. skb_free_frag(buf);
  381. } while (1);
  382. spin_unlock_bh(&q->lock);
  383. if (!q->rx_page.va)
  384. return;
  385. page = virt_to_page(q->rx_page.va);
  386. __page_frag_cache_drain(page, q->rx_page.pagecnt_bias);
  387. memset(&q->rx_page, 0, sizeof(q->rx_page));
  388. }
  389. static void
  390. mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)
  391. {
  392. struct mt76_queue *q = &dev->q_rx[qid];
  393. int i;
  394. for (i = 0; i < q->ndesc; i++)
  395. q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
  396. mt76_dma_rx_cleanup(dev, q);
  397. mt76_dma_sync_idx(dev, q);
  398. mt76_dma_rx_fill(dev, q);
  399. if (!q->rx_head)
  400. return;
  401. dev_kfree_skb(q->rx_head);
  402. q->rx_head = NULL;
  403. }
  404. static void
  405. mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data,
  406. int len, bool more)
  407. {
  408. struct sk_buff *skb = q->rx_head;
  409. struct skb_shared_info *shinfo = skb_shinfo(skb);
  410. int nr_frags = shinfo->nr_frags;
  411. if (nr_frags < ARRAY_SIZE(shinfo->frags)) {
  412. struct page *page = virt_to_head_page(data);
  413. int offset = data - page_address(page) + q->buf_offset;
  414. skb_add_rx_frag(skb, nr_frags, page, offset, len, q->buf_size);
  415. } else {
  416. skb_free_frag(data);
  417. }
  418. if (more)
  419. return;
  420. q->rx_head = NULL;
  421. if (nr_frags < ARRAY_SIZE(shinfo->frags))
  422. dev->drv->rx_skb(dev, q - dev->q_rx, skb);
  423. else
  424. dev_kfree_skb(skb);
  425. }
  426. static int
  427. mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget)
  428. {
  429. int len, data_len, done = 0;
  430. struct sk_buff *skb;
  431. unsigned char *data;
  432. bool more;
  433. while (done < budget) {
  434. u32 info;
  435. data = mt76_dma_dequeue(dev, q, false, &len, &info, &more);
  436. if (!data)
  437. break;
  438. if (q->rx_head)
  439. data_len = q->buf_size;
  440. else
  441. data_len = SKB_WITH_OVERHEAD(q->buf_size);
  442. if (data_len < len + q->buf_offset) {
  443. dev_kfree_skb(q->rx_head);
  444. q->rx_head = NULL;
  445. skb_free_frag(data);
  446. continue;
  447. }
  448. if (q->rx_head) {
  449. mt76_add_fragment(dev, q, data, len, more);
  450. continue;
  451. }
  452. skb = build_skb(data, q->buf_size);
  453. if (!skb) {
  454. skb_free_frag(data);
  455. continue;
  456. }
  457. skb_reserve(skb, q->buf_offset);
  458. if (q == &dev->q_rx[MT_RXQ_MCU]) {
  459. u32 *rxfce = (u32 *)skb->cb;
  460. *rxfce = info;
  461. }
  462. __skb_put(skb, len);
  463. done++;
  464. if (more) {
  465. q->rx_head = skb;
  466. continue;
  467. }
  468. dev->drv->rx_skb(dev, q - dev->q_rx, skb);
  469. }
  470. mt76_dma_rx_fill(dev, q);
  471. return done;
  472. }
  473. static int
  474. mt76_dma_rx_poll(struct napi_struct *napi, int budget)
  475. {
  476. struct mt76_dev *dev;
  477. int qid, done = 0, cur;
  478. dev = container_of(napi->dev, struct mt76_dev, napi_dev);
  479. qid = napi - dev->napi;
  480. rcu_read_lock();
  481. do {
  482. cur = mt76_dma_rx_process(dev, &dev->q_rx[qid], budget - done);
  483. mt76_rx_poll_complete(dev, qid, napi);
  484. done += cur;
  485. } while (cur && done < budget);
  486. rcu_read_unlock();
  487. if (done < budget && napi_complete(napi))
  488. dev->drv->rx_poll_complete(dev, qid);
  489. return done;
  490. }
  491. static int
  492. mt76_dma_init(struct mt76_dev *dev)
  493. {
  494. int i;
  495. init_dummy_netdev(&dev->napi_dev);
  496. mt76_for_each_q_rx(dev, i) {
  497. netif_threaded_napi_add(&dev->napi_dev, &dev->napi[i],
  498. mt76_dma_rx_poll, 64);
  499. mt76_dma_rx_fill(dev, &dev->q_rx[i]);
  500. napi_enable(&dev->napi[i]);
  501. }
  502. return 0;
  503. }
  504. static const struct mt76_queue_ops mt76_dma_ops = {
  505. .init = mt76_dma_init,
  506. .alloc = mt76_dma_alloc_queue,
  507. .tx_queue_skb_raw = mt76_dma_tx_queue_skb_raw,
  508. .tx_queue_skb = mt76_dma_tx_queue_skb,
  509. .tx_cleanup = mt76_dma_tx_cleanup,
  510. .rx_reset = mt76_dma_rx_reset,
  511. .kick = mt76_dma_kick_queue,
  512. };
  513. void mt76_dma_attach(struct mt76_dev *dev)
  514. {
  515. dev->queue_ops = &mt76_dma_ops;
  516. }
  517. EXPORT_SYMBOL_GPL(mt76_dma_attach);
  518. void mt76_dma_cleanup(struct mt76_dev *dev)
  519. {
  520. int i;
  521. mt76_worker_disable(&dev->tx_worker);
  522. netif_napi_del(&dev->tx_napi);
  523. for (i = 0; i < ARRAY_SIZE(dev->phy.q_tx); i++) {
  524. mt76_dma_tx_cleanup(dev, dev->phy.q_tx[i], true);
  525. if (dev->phy2)
  526. mt76_dma_tx_cleanup(dev, dev->phy2->q_tx[i], true);
  527. }
  528. for (i = 0; i < ARRAY_SIZE(dev->q_mcu); i++)
  529. mt76_dma_tx_cleanup(dev, dev->q_mcu[i], true);
  530. mt76_for_each_q_rx(dev, i) {
  531. netif_napi_del(&dev->napi[i]);
  532. mt76_dma_rx_cleanup(dev, &dev->q_rx[i]);
  533. }
  534. mt76_free_pending_txwi(dev);
  535. }
  536. EXPORT_SYMBOL_GPL(mt76_dma_cleanup);