dma.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692
  1. // SPDX-License-Identifier: ISC
  2. /*
  3. * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
  4. */
  5. #include <linux/dma-mapping.h>
  6. #include "mt76.h"
  7. #include "dma.h"
  8. static struct mt76_txwi_cache *
  9. mt76_alloc_txwi(struct mt76_dev *dev)
  10. {
  11. struct mt76_txwi_cache *t;
  12. dma_addr_t addr;
  13. u8 *txwi;
  14. int size;
  15. size = L1_CACHE_ALIGN(dev->drv->txwi_size + sizeof(*t));
  16. txwi = devm_kzalloc(dev->dev, size, GFP_ATOMIC);
  17. if (!txwi)
  18. return NULL;
  19. addr = dma_map_single(dev->dev, txwi, dev->drv->txwi_size,
  20. DMA_TO_DEVICE);
  21. t = (struct mt76_txwi_cache *)(txwi + dev->drv->txwi_size);
  22. t->dma_addr = addr;
  23. return t;
  24. }
  25. static struct mt76_txwi_cache *
  26. __mt76_get_txwi(struct mt76_dev *dev)
  27. {
  28. struct mt76_txwi_cache *t = NULL;
  29. spin_lock(&dev->lock);
  30. if (!list_empty(&dev->txwi_cache)) {
  31. t = list_first_entry(&dev->txwi_cache, struct mt76_txwi_cache,
  32. list);
  33. list_del(&t->list);
  34. }
  35. spin_unlock(&dev->lock);
  36. return t;
  37. }
  38. static struct mt76_txwi_cache *
  39. mt76_get_txwi(struct mt76_dev *dev)
  40. {
  41. struct mt76_txwi_cache *t = __mt76_get_txwi(dev);
  42. if (t)
  43. return t;
  44. return mt76_alloc_txwi(dev);
  45. }
  46. void
  47. mt76_put_txwi(struct mt76_dev *dev, struct mt76_txwi_cache *t)
  48. {
  49. if (!t)
  50. return;
  51. spin_lock(&dev->lock);
  52. list_add(&t->list, &dev->txwi_cache);
  53. spin_unlock(&dev->lock);
  54. }
  55. EXPORT_SYMBOL_GPL(mt76_put_txwi);
  56. static void
  57. mt76_free_pending_txwi(struct mt76_dev *dev)
  58. {
  59. struct mt76_txwi_cache *t;
  60. local_bh_disable();
  61. while ((t = __mt76_get_txwi(dev)) != NULL)
  62. dma_unmap_single(dev->dev, t->dma_addr, dev->drv->txwi_size,
  63. DMA_TO_DEVICE);
  64. local_bh_enable();
  65. }
  66. static void
  67. mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q)
  68. {
  69. writel(q->desc_dma, &q->regs->desc_base);
  70. writel(q->ndesc, &q->regs->ring_size);
  71. q->head = readl(&q->regs->dma_idx);
  72. q->tail = q->head;
  73. }
  74. static void
  75. mt76_dma_queue_reset(struct mt76_dev *dev, struct mt76_queue *q)
  76. {
  77. int i;
  78. if (!q)
  79. return;
  80. /* clear descriptors */
  81. for (i = 0; i < q->ndesc; i++)
  82. q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
  83. writel(0, &q->regs->cpu_idx);
  84. writel(0, &q->regs->dma_idx);
  85. mt76_dma_sync_idx(dev, q);
  86. }
  87. static int
  88. mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q,
  89. int idx, int n_desc, int bufsize,
  90. u32 ring_base)
  91. {
  92. int size;
  93. spin_lock_init(&q->lock);
  94. spin_lock_init(&q->cleanup_lock);
  95. q->regs = dev->mmio.regs + ring_base + idx * MT_RING_SIZE;
  96. q->ndesc = n_desc;
  97. q->buf_size = bufsize;
  98. q->hw_idx = idx;
  99. size = q->ndesc * sizeof(struct mt76_desc);
  100. q->desc = dmam_alloc_coherent(dev->dev, size, &q->desc_dma, GFP_KERNEL);
  101. if (!q->desc)
  102. return -ENOMEM;
  103. size = q->ndesc * sizeof(*q->entry);
  104. q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL);
  105. if (!q->entry)
  106. return -ENOMEM;
  107. mt76_dma_queue_reset(dev, q);
  108. return 0;
  109. }
  110. static int
  111. mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q,
  112. struct mt76_queue_buf *buf, int nbufs, u32 info,
  113. struct sk_buff *skb, void *txwi)
  114. {
  115. struct mt76_queue_entry *entry;
  116. struct mt76_desc *desc;
  117. u32 ctrl;
  118. int i, idx = -1;
  119. if (txwi) {
  120. q->entry[q->head].txwi = DMA_DUMMY_DATA;
  121. q->entry[q->head].skip_buf0 = true;
  122. }
  123. for (i = 0; i < nbufs; i += 2, buf += 2) {
  124. u32 buf0 = buf[0].addr, buf1 = 0;
  125. idx = q->head;
  126. q->head = (q->head + 1) % q->ndesc;
  127. desc = &q->desc[idx];
  128. entry = &q->entry[idx];
  129. if (buf[0].skip_unmap)
  130. entry->skip_buf0 = true;
  131. entry->skip_buf1 = i == nbufs - 1;
  132. entry->dma_addr[0] = buf[0].addr;
  133. entry->dma_len[0] = buf[0].len;
  134. ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len);
  135. if (i < nbufs - 1) {
  136. entry->dma_addr[1] = buf[1].addr;
  137. entry->dma_len[1] = buf[1].len;
  138. buf1 = buf[1].addr;
  139. ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len);
  140. if (buf[1].skip_unmap)
  141. entry->skip_buf1 = true;
  142. }
  143. if (i == nbufs - 1)
  144. ctrl |= MT_DMA_CTL_LAST_SEC0;
  145. else if (i == nbufs - 2)
  146. ctrl |= MT_DMA_CTL_LAST_SEC1;
  147. WRITE_ONCE(desc->buf0, cpu_to_le32(buf0));
  148. WRITE_ONCE(desc->buf1, cpu_to_le32(buf1));
  149. WRITE_ONCE(desc->info, cpu_to_le32(info));
  150. WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl));
  151. q->queued++;
  152. }
  153. q->entry[idx].txwi = txwi;
  154. q->entry[idx].skb = skb;
  155. return idx;
  156. }
  157. static void
  158. mt76_dma_tx_cleanup_idx(struct mt76_dev *dev, struct mt76_queue *q, int idx,
  159. struct mt76_queue_entry *prev_e)
  160. {
  161. struct mt76_queue_entry *e = &q->entry[idx];
  162. if (!e->skip_buf0)
  163. dma_unmap_single(dev->dev, e->dma_addr[0], e->dma_len[0],
  164. DMA_TO_DEVICE);
  165. if (!e->skip_buf1)
  166. dma_unmap_single(dev->dev, e->dma_addr[1], e->dma_len[1],
  167. DMA_TO_DEVICE);
  168. if (e->txwi == DMA_DUMMY_DATA)
  169. e->txwi = NULL;
  170. if (e->skb == DMA_DUMMY_DATA)
  171. e->skb = NULL;
  172. *prev_e = *e;
  173. memset(e, 0, sizeof(*e));
  174. }
  175. static void
  176. mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q)
  177. {
  178. wmb();
  179. writel(q->head, &q->regs->cpu_idx);
  180. }
  181. static void
  182. mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush)
  183. {
  184. struct mt76_queue_entry entry;
  185. int last;
  186. if (!q)
  187. return;
  188. spin_lock_bh(&q->cleanup_lock);
  189. if (flush)
  190. last = -1;
  191. else
  192. last = readl(&q->regs->dma_idx);
  193. while (q->queued > 0 && q->tail != last) {
  194. mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry);
  195. mt76_queue_tx_complete(dev, q, &entry);
  196. if (entry.txwi) {
  197. if (!(dev->drv->drv_flags & MT_DRV_TXWI_NO_FREE))
  198. mt76_put_txwi(dev, entry.txwi);
  199. }
  200. if (!flush && q->tail == last)
  201. last = readl(&q->regs->dma_idx);
  202. }
  203. spin_unlock_bh(&q->cleanup_lock);
  204. if (flush) {
  205. spin_lock_bh(&q->lock);
  206. mt76_dma_sync_idx(dev, q);
  207. mt76_dma_kick_queue(dev, q);
  208. spin_unlock_bh(&q->lock);
  209. }
  210. if (!q->queued)
  211. wake_up(&dev->tx_wait);
  212. }
  213. static void *
  214. mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx,
  215. int *len, u32 *info, bool *more)
  216. {
  217. struct mt76_queue_entry *e = &q->entry[idx];
  218. struct mt76_desc *desc = &q->desc[idx];
  219. dma_addr_t buf_addr;
  220. void *buf = e->buf;
  221. int buf_len = SKB_WITH_OVERHEAD(q->buf_size);
  222. buf_addr = e->dma_addr[0];
  223. if (len) {
  224. u32 ctl = le32_to_cpu(READ_ONCE(desc->ctrl));
  225. *len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctl);
  226. *more = !(ctl & MT_DMA_CTL_LAST_SEC0);
  227. }
  228. if (info)
  229. *info = le32_to_cpu(desc->info);
  230. dma_unmap_single(dev->dev, buf_addr, buf_len, DMA_FROM_DEVICE);
  231. e->buf = NULL;
  232. return buf;
  233. }
  234. static void *
  235. mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush,
  236. int *len, u32 *info, bool *more)
  237. {
  238. int idx = q->tail;
  239. *more = false;
  240. if (!q->queued)
  241. return NULL;
  242. if (flush)
  243. q->desc[idx].ctrl |= cpu_to_le32(MT_DMA_CTL_DMA_DONE);
  244. else if (!(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE)))
  245. return NULL;
  246. q->tail = (q->tail + 1) % q->ndesc;
  247. q->queued--;
  248. return mt76_dma_get_buf(dev, q, idx, len, info, more);
  249. }
  250. static int
  251. mt76_dma_tx_queue_skb_raw(struct mt76_dev *dev, struct mt76_queue *q,
  252. struct sk_buff *skb, u32 tx_info)
  253. {
  254. struct mt76_queue_buf buf = {};
  255. dma_addr_t addr;
  256. if (q->queued + 1 >= q->ndesc - 1)
  257. goto error;
  258. addr = dma_map_single(dev->dev, skb->data, skb->len,
  259. DMA_TO_DEVICE);
  260. if (unlikely(dma_mapping_error(dev->dev, addr)))
  261. goto error;
  262. buf.addr = addr;
  263. buf.len = skb->len;
  264. spin_lock_bh(&q->lock);
  265. mt76_dma_add_buf(dev, q, &buf, 1, tx_info, skb, NULL);
  266. mt76_dma_kick_queue(dev, q);
  267. spin_unlock_bh(&q->lock);
  268. return 0;
  269. error:
  270. dev_kfree_skb(skb);
  271. return -ENOMEM;
  272. }
  273. static int
  274. mt76_dma_tx_queue_skb(struct mt76_dev *dev, struct mt76_queue *q,
  275. struct sk_buff *skb, struct mt76_wcid *wcid,
  276. struct ieee80211_sta *sta)
  277. {
  278. struct mt76_tx_info tx_info = {
  279. .skb = skb,
  280. };
  281. struct ieee80211_hw *hw;
  282. int len, n = 0, ret = -ENOMEM;
  283. struct mt76_txwi_cache *t;
  284. struct sk_buff *iter;
  285. dma_addr_t addr;
  286. u8 *txwi;
  287. t = mt76_get_txwi(dev);
  288. if (!t) {
  289. hw = mt76_tx_status_get_hw(dev, skb);
  290. ieee80211_free_txskb(hw, skb);
  291. return -ENOMEM;
  292. }
  293. txwi = mt76_get_txwi_ptr(dev, t);
  294. skb->prev = skb->next = NULL;
  295. if (dev->drv->drv_flags & MT_DRV_TX_ALIGNED4_SKBS)
  296. mt76_insert_hdr_pad(skb);
  297. len = skb_headlen(skb);
  298. addr = dma_map_single(dev->dev, skb->data, len, DMA_TO_DEVICE);
  299. if (unlikely(dma_mapping_error(dev->dev, addr)))
  300. goto free;
  301. tx_info.buf[n].addr = t->dma_addr;
  302. tx_info.buf[n++].len = dev->drv->txwi_size;
  303. tx_info.buf[n].addr = addr;
  304. tx_info.buf[n++].len = len;
  305. skb_walk_frags(skb, iter) {
  306. if (n == ARRAY_SIZE(tx_info.buf))
  307. goto unmap;
  308. addr = dma_map_single(dev->dev, iter->data, iter->len,
  309. DMA_TO_DEVICE);
  310. if (unlikely(dma_mapping_error(dev->dev, addr)))
  311. goto unmap;
  312. tx_info.buf[n].addr = addr;
  313. tx_info.buf[n++].len = iter->len;
  314. }
  315. tx_info.nbuf = n;
  316. if (q->queued + (tx_info.nbuf + 1) / 2 >= q->ndesc - 1) {
  317. ret = -ENOMEM;
  318. goto unmap;
  319. }
  320. dma_sync_single_for_cpu(dev->dev, t->dma_addr, dev->drv->txwi_size,
  321. DMA_TO_DEVICE);
  322. ret = dev->drv->tx_prepare_skb(dev, txwi, q->qid, wcid, sta, &tx_info);
  323. dma_sync_single_for_device(dev->dev, t->dma_addr, dev->drv->txwi_size,
  324. DMA_TO_DEVICE);
  325. if (ret < 0)
  326. goto unmap;
  327. return mt76_dma_add_buf(dev, q, tx_info.buf, tx_info.nbuf,
  328. tx_info.info, tx_info.skb, t);
  329. unmap:
  330. for (n--; n > 0; n--)
  331. dma_unmap_single(dev->dev, tx_info.buf[n].addr,
  332. tx_info.buf[n].len, DMA_TO_DEVICE);
  333. free:
  334. #ifdef CONFIG_NL80211_TESTMODE
  335. /* fix tx_done accounting on queue overflow */
  336. if (mt76_is_testmode_skb(dev, skb, &hw)) {
  337. struct mt76_phy *phy = hw->priv;
  338. if (tx_info.skb == phy->test.tx_skb)
  339. phy->test.tx_done--;
  340. }
  341. #endif
  342. dev_kfree_skb(tx_info.skb);
  343. mt76_put_txwi(dev, t);
  344. return ret;
  345. }
  346. static int
  347. mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q)
  348. {
  349. dma_addr_t addr;
  350. void *buf;
  351. int frames = 0;
  352. int len = SKB_WITH_OVERHEAD(q->buf_size);
  353. int offset = q->buf_offset;
  354. spin_lock_bh(&q->lock);
  355. while (q->queued < q->ndesc - 1) {
  356. struct mt76_queue_buf qbuf;
  357. buf = page_frag_alloc(&q->rx_page, q->buf_size, GFP_ATOMIC);
  358. if (!buf)
  359. break;
  360. addr = dma_map_single(dev->dev, buf, len, DMA_FROM_DEVICE);
  361. if (unlikely(dma_mapping_error(dev->dev, addr))) {
  362. skb_free_frag(buf);
  363. break;
  364. }
  365. qbuf.addr = addr + offset;
  366. qbuf.len = len - offset;
  367. mt76_dma_add_buf(dev, q, &qbuf, 1, 0, buf, NULL);
  368. frames++;
  369. }
  370. if (frames)
  371. mt76_dma_kick_queue(dev, q);
  372. spin_unlock_bh(&q->lock);
  373. return frames;
  374. }
  375. static void
  376. mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
  377. {
  378. struct page *page;
  379. void *buf;
  380. bool more;
  381. spin_lock_bh(&q->lock);
  382. do {
  383. buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more);
  384. if (!buf)
  385. break;
  386. skb_free_frag(buf);
  387. } while (1);
  388. spin_unlock_bh(&q->lock);
  389. if (!q->rx_page.va)
  390. return;
  391. page = virt_to_page(q->rx_page.va);
  392. __page_frag_cache_drain(page, q->rx_page.pagecnt_bias);
  393. memset(&q->rx_page, 0, sizeof(q->rx_page));
  394. }
  395. static void
  396. mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)
  397. {
  398. struct mt76_queue *q = &dev->q_rx[qid];
  399. int i;
  400. for (i = 0; i < q->ndesc; i++)
  401. q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
  402. mt76_dma_rx_cleanup(dev, q);
  403. mt76_dma_sync_idx(dev, q);
  404. mt76_dma_rx_fill(dev, q);
  405. if (!q->rx_head)
  406. return;
  407. dev_kfree_skb(q->rx_head);
  408. q->rx_head = NULL;
  409. }
  410. static void
  411. mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data,
  412. int len, bool more)
  413. {
  414. struct sk_buff *skb = q->rx_head;
  415. struct skb_shared_info *shinfo = skb_shinfo(skb);
  416. int nr_frags = shinfo->nr_frags;
  417. if (nr_frags < ARRAY_SIZE(shinfo->frags)) {
  418. struct page *page = virt_to_head_page(data);
  419. int offset = data - page_address(page) + q->buf_offset;
  420. skb_add_rx_frag(skb, nr_frags, page, offset, len, q->buf_size);
  421. } else {
  422. skb_free_frag(data);
  423. }
  424. if (more)
  425. return;
  426. q->rx_head = NULL;
  427. if (nr_frags < ARRAY_SIZE(shinfo->frags))
  428. dev->drv->rx_skb(dev, q - dev->q_rx, skb);
  429. else
  430. dev_kfree_skb(skb);
  431. }
  432. static int
  433. mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget)
  434. {
  435. int len, data_len, done = 0;
  436. struct sk_buff *skb;
  437. unsigned char *data;
  438. bool more;
  439. while (done < budget) {
  440. u32 info;
  441. data = mt76_dma_dequeue(dev, q, false, &len, &info, &more);
  442. if (!data)
  443. break;
  444. if (q->rx_head)
  445. data_len = q->buf_size;
  446. else
  447. data_len = SKB_WITH_OVERHEAD(q->buf_size);
  448. if (data_len < len + q->buf_offset) {
  449. dev_kfree_skb(q->rx_head);
  450. q->rx_head = NULL;
  451. skb_free_frag(data);
  452. continue;
  453. }
  454. if (q->rx_head) {
  455. mt76_add_fragment(dev, q, data, len, more);
  456. continue;
  457. }
  458. skb = build_skb(data, q->buf_size);
  459. if (!skb) {
  460. skb_free_frag(data);
  461. continue;
  462. }
  463. skb_reserve(skb, q->buf_offset);
  464. if (q == &dev->q_rx[MT_RXQ_MCU]) {
  465. u32 *rxfce = (u32 *)skb->cb;
  466. *rxfce = info;
  467. }
  468. __skb_put(skb, len);
  469. done++;
  470. if (more) {
  471. q->rx_head = skb;
  472. continue;
  473. }
  474. dev->drv->rx_skb(dev, q - dev->q_rx, skb);
  475. }
  476. mt76_dma_rx_fill(dev, q);
  477. return done;
  478. }
  479. int mt76_dma_rx_poll(struct napi_struct *napi, int budget)
  480. {
  481. struct mt76_dev *dev;
  482. int qid, done = 0, cur;
  483. dev = container_of(napi->dev, struct mt76_dev, napi_dev);
  484. qid = napi - dev->napi;
  485. rcu_read_lock();
  486. do {
  487. cur = mt76_dma_rx_process(dev, &dev->q_rx[qid], budget - done);
  488. mt76_rx_poll_complete(dev, qid, napi);
  489. done += cur;
  490. } while (cur && done < budget);
  491. rcu_read_unlock();
  492. if (done < budget && napi_complete(napi))
  493. dev->drv->rx_poll_complete(dev, qid);
  494. return done;
  495. }
  496. EXPORT_SYMBOL_GPL(mt76_dma_rx_poll);
  497. static int
  498. mt76_dma_init(struct mt76_dev *dev,
  499. int (*poll)(struct napi_struct *napi, int budget))
  500. {
  501. int i;
  502. init_dummy_netdev(&dev->napi_dev);
  503. init_dummy_netdev(&dev->tx_napi_dev);
  504. snprintf(dev->napi_dev.name, sizeof(dev->napi_dev.name), "%s",
  505. wiphy_name(dev->hw->wiphy));
  506. dev->napi_dev.threaded = 1;
  507. mt76_for_each_q_rx(dev, i) {
  508. netif_napi_add(&dev->napi_dev, &dev->napi[i], poll, 64);
  509. mt76_dma_rx_fill(dev, &dev->q_rx[i]);
  510. napi_enable(&dev->napi[i]);
  511. }
  512. return 0;
  513. }
  514. static const struct mt76_queue_ops mt76_dma_ops = {
  515. .init = mt76_dma_init,
  516. .alloc = mt76_dma_alloc_queue,
  517. .reset_q = mt76_dma_queue_reset,
  518. .tx_queue_skb_raw = mt76_dma_tx_queue_skb_raw,
  519. .tx_queue_skb = mt76_dma_tx_queue_skb,
  520. .tx_cleanup = mt76_dma_tx_cleanup,
  521. .rx_cleanup = mt76_dma_rx_cleanup,
  522. .rx_reset = mt76_dma_rx_reset,
  523. .kick = mt76_dma_kick_queue,
  524. };
  525. void mt76_dma_attach(struct mt76_dev *dev)
  526. {
  527. dev->queue_ops = &mt76_dma_ops;
  528. }
  529. EXPORT_SYMBOL_GPL(mt76_dma_attach);
  530. void mt76_dma_cleanup(struct mt76_dev *dev)
  531. {
  532. int i;
  533. mt76_worker_disable(&dev->tx_worker);
  534. netif_napi_del(&dev->tx_napi);
  535. for (i = 0; i < ARRAY_SIZE(dev->phy.q_tx); i++) {
  536. mt76_dma_tx_cleanup(dev, dev->phy.q_tx[i], true);
  537. if (dev->phy2)
  538. mt76_dma_tx_cleanup(dev, dev->phy2->q_tx[i], true);
  539. }
  540. for (i = 0; i < ARRAY_SIZE(dev->q_mcu); i++)
  541. mt76_dma_tx_cleanup(dev, dev->q_mcu[i], true);
  542. mt76_for_each_q_rx(dev, i) {
  543. netif_napi_del(&dev->napi[i]);
  544. mt76_dma_rx_cleanup(dev, &dev->q_rx[i]);
  545. }
  546. mt76_free_pending_txwi(dev);
  547. }
  548. EXPORT_SYMBOL_GPL(mt76_dma_cleanup);