dma.c 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699
  1. // SPDX-License-Identifier: ISC
  2. /*
  3. * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
  4. */
  5. #include <linux/dma-mapping.h>
  6. #include "mt76.h"
  7. #include "dma.h"
  8. static struct mt76_txwi_cache *
  9. mt76_alloc_txwi(struct mt76_dev *dev)
  10. {
  11. struct mt76_txwi_cache *t;
  12. dma_addr_t addr;
  13. u8 *txwi;
  14. int size;
  15. size = L1_CACHE_ALIGN(dev->drv->txwi_size + sizeof(*t));
  16. txwi = devm_kzalloc(dev->dev, size, GFP_ATOMIC);
  17. if (!txwi)
  18. return NULL;
  19. addr = dma_map_single(dev->dev, txwi, dev->drv->txwi_size,
  20. DMA_TO_DEVICE);
  21. t = (struct mt76_txwi_cache *)(txwi + dev->drv->txwi_size);
  22. t->dma_addr = addr;
  23. return t;
  24. }
  25. static struct mt76_txwi_cache *
  26. __mt76_get_txwi(struct mt76_dev *dev)
  27. {
  28. struct mt76_txwi_cache *t = NULL;
  29. spin_lock(&dev->lock);
  30. if (!list_empty(&dev->txwi_cache)) {
  31. t = list_first_entry(&dev->txwi_cache, struct mt76_txwi_cache,
  32. list);
  33. list_del(&t->list);
  34. }
  35. spin_unlock(&dev->lock);
  36. return t;
  37. }
  38. static struct mt76_txwi_cache *
  39. mt76_get_txwi(struct mt76_dev *dev)
  40. {
  41. struct mt76_txwi_cache *t = __mt76_get_txwi(dev);
  42. if (t)
  43. return t;
  44. return mt76_alloc_txwi(dev);
  45. }
  46. void
  47. mt76_put_txwi(struct mt76_dev *dev, struct mt76_txwi_cache *t)
  48. {
  49. if (!t)
  50. return;
  51. spin_lock(&dev->lock);
  52. list_add(&t->list, &dev->txwi_cache);
  53. spin_unlock(&dev->lock);
  54. }
  55. EXPORT_SYMBOL_GPL(mt76_put_txwi);
  56. static void
  57. mt76_free_pending_txwi(struct mt76_dev *dev)
  58. {
  59. struct mt76_txwi_cache *t;
  60. local_bh_disable();
  61. while ((t = __mt76_get_txwi(dev)) != NULL)
  62. dma_unmap_single(dev->dev, t->dma_addr, dev->drv->txwi_size,
  63. DMA_TO_DEVICE);
  64. local_bh_enable();
  65. }
  66. static void
  67. mt76_dma_sync_idx(struct mt76_dev *dev, struct mt76_queue *q)
  68. {
  69. writel(q->desc_dma, &q->regs->desc_base);
  70. writel(q->ndesc, &q->regs->ring_size);
  71. q->head = readl(&q->regs->dma_idx);
  72. q->tail = q->head;
  73. }
  74. static void
  75. mt76_dma_queue_reset(struct mt76_dev *dev, struct mt76_queue *q)
  76. {
  77. int i;
  78. if (!q)
  79. return;
  80. /* clear descriptors */
  81. for (i = 0; i < q->ndesc; i++)
  82. q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
  83. writel(0, &q->regs->cpu_idx);
  84. writel(0, &q->regs->dma_idx);
  85. mt76_dma_sync_idx(dev, q);
  86. }
  87. static int
  88. mt76_dma_alloc_queue(struct mt76_dev *dev, struct mt76_queue *q,
  89. int idx, int n_desc, int bufsize,
  90. u32 ring_base)
  91. {
  92. int size;
  93. spin_lock_init(&q->lock);
  94. spin_lock_init(&q->cleanup_lock);
  95. q->regs = dev->mmio.regs + ring_base + idx * MT_RING_SIZE;
  96. q->ndesc = n_desc;
  97. q->buf_size = bufsize;
  98. q->hw_idx = idx;
  99. size = q->ndesc * sizeof(struct mt76_desc);
  100. q->desc = dmam_alloc_coherent(dev->dev, size, &q->desc_dma, GFP_KERNEL);
  101. if (!q->desc)
  102. return -ENOMEM;
  103. size = q->ndesc * sizeof(*q->entry);
  104. q->entry = devm_kzalloc(dev->dev, size, GFP_KERNEL);
  105. if (!q->entry)
  106. return -ENOMEM;
  107. mt76_dma_queue_reset(dev, q);
  108. return 0;
  109. }
  110. static int
  111. mt76_dma_add_buf(struct mt76_dev *dev, struct mt76_queue *q,
  112. struct mt76_queue_buf *buf, int nbufs, u32 info,
  113. struct sk_buff *skb, void *txwi)
  114. {
  115. struct mt76_queue_entry *entry;
  116. struct mt76_desc *desc;
  117. u32 ctrl;
  118. int i, idx = -1;
  119. if (txwi) {
  120. q->entry[q->head].txwi = DMA_DUMMY_DATA;
  121. q->entry[q->head].skip_buf0 = true;
  122. }
  123. for (i = 0; i < nbufs; i += 2, buf += 2) {
  124. u32 buf0 = buf[0].addr, buf1 = 0;
  125. idx = q->head;
  126. q->head = (q->head + 1) % q->ndesc;
  127. desc = &q->desc[idx];
  128. entry = &q->entry[idx];
  129. if (buf[0].skip_unmap)
  130. entry->skip_buf0 = true;
  131. entry->skip_buf1 = i == nbufs - 1;
  132. entry->dma_addr[0] = buf[0].addr;
  133. entry->dma_len[0] = buf[0].len;
  134. ctrl = FIELD_PREP(MT_DMA_CTL_SD_LEN0, buf[0].len);
  135. if (i < nbufs - 1) {
  136. entry->dma_addr[1] = buf[1].addr;
  137. entry->dma_len[1] = buf[1].len;
  138. buf1 = buf[1].addr;
  139. ctrl |= FIELD_PREP(MT_DMA_CTL_SD_LEN1, buf[1].len);
  140. if (buf[1].skip_unmap)
  141. entry->skip_buf1 = true;
  142. }
  143. if (i == nbufs - 1)
  144. ctrl |= MT_DMA_CTL_LAST_SEC0;
  145. else if (i == nbufs - 2)
  146. ctrl |= MT_DMA_CTL_LAST_SEC1;
  147. WRITE_ONCE(desc->buf0, cpu_to_le32(buf0));
  148. WRITE_ONCE(desc->buf1, cpu_to_le32(buf1));
  149. WRITE_ONCE(desc->info, cpu_to_le32(info));
  150. WRITE_ONCE(desc->ctrl, cpu_to_le32(ctrl));
  151. q->queued++;
  152. }
  153. q->entry[idx].txwi = txwi;
  154. q->entry[idx].skb = skb;
  155. q->entry[idx].wcid = 0xffff;
  156. return idx;
  157. }
  158. static void
  159. mt76_dma_tx_cleanup_idx(struct mt76_dev *dev, struct mt76_queue *q, int idx,
  160. struct mt76_queue_entry *prev_e)
  161. {
  162. struct mt76_queue_entry *e = &q->entry[idx];
  163. if (!e->skip_buf0)
  164. dma_unmap_single(dev->dev, e->dma_addr[0], e->dma_len[0],
  165. DMA_TO_DEVICE);
  166. if (!e->skip_buf1)
  167. dma_unmap_single(dev->dev, e->dma_addr[1], e->dma_len[1],
  168. DMA_TO_DEVICE);
  169. if (e->txwi == DMA_DUMMY_DATA)
  170. e->txwi = NULL;
  171. if (e->skb == DMA_DUMMY_DATA)
  172. e->skb = NULL;
  173. *prev_e = *e;
  174. memset(e, 0, sizeof(*e));
  175. }
  176. static void
  177. mt76_dma_kick_queue(struct mt76_dev *dev, struct mt76_queue *q)
  178. {
  179. wmb();
  180. writel(q->head, &q->regs->cpu_idx);
  181. }
  182. static void
  183. mt76_dma_tx_cleanup(struct mt76_dev *dev, struct mt76_queue *q, bool flush)
  184. {
  185. struct mt76_queue_entry entry;
  186. int last;
  187. if (!q)
  188. return;
  189. spin_lock_bh(&q->cleanup_lock);
  190. if (flush)
  191. last = -1;
  192. else
  193. last = readl(&q->regs->dma_idx);
  194. while (q->queued > 0 && q->tail != last) {
  195. mt76_dma_tx_cleanup_idx(dev, q, q->tail, &entry);
  196. mt76_queue_tx_complete(dev, q, &entry);
  197. if (entry.txwi) {
  198. if (!(dev->drv->drv_flags & MT_DRV_TXWI_NO_FREE))
  199. mt76_put_txwi(dev, entry.txwi);
  200. }
  201. if (!flush && q->tail == last)
  202. last = readl(&q->regs->dma_idx);
  203. }
  204. spin_unlock_bh(&q->cleanup_lock);
  205. if (flush) {
  206. spin_lock_bh(&q->lock);
  207. mt76_dma_sync_idx(dev, q);
  208. mt76_dma_kick_queue(dev, q);
  209. spin_unlock_bh(&q->lock);
  210. }
  211. if (!q->queued)
  212. wake_up(&dev->tx_wait);
  213. }
  214. static void *
  215. mt76_dma_get_buf(struct mt76_dev *dev, struct mt76_queue *q, int idx,
  216. int *len, u32 *info, bool *more)
  217. {
  218. struct mt76_queue_entry *e = &q->entry[idx];
  219. struct mt76_desc *desc = &q->desc[idx];
  220. dma_addr_t buf_addr;
  221. void *buf = e->buf;
  222. int buf_len = SKB_WITH_OVERHEAD(q->buf_size);
  223. buf_addr = e->dma_addr[0];
  224. if (len) {
  225. u32 ctl = le32_to_cpu(READ_ONCE(desc->ctrl));
  226. *len = FIELD_GET(MT_DMA_CTL_SD_LEN0, ctl);
  227. *more = !(ctl & MT_DMA_CTL_LAST_SEC0);
  228. }
  229. if (info)
  230. *info = le32_to_cpu(desc->info);
  231. dma_unmap_single(dev->dev, buf_addr, buf_len, DMA_FROM_DEVICE);
  232. e->buf = NULL;
  233. return buf;
  234. }
  235. static void *
  236. mt76_dma_dequeue(struct mt76_dev *dev, struct mt76_queue *q, bool flush,
  237. int *len, u32 *info, bool *more)
  238. {
  239. int idx = q->tail;
  240. *more = false;
  241. if (!q->queued)
  242. return NULL;
  243. if (flush)
  244. q->desc[idx].ctrl |= cpu_to_le32(MT_DMA_CTL_DMA_DONE);
  245. else if (!(q->desc[idx].ctrl & cpu_to_le32(MT_DMA_CTL_DMA_DONE)))
  246. return NULL;
  247. q->tail = (q->tail + 1) % q->ndesc;
  248. q->queued--;
  249. return mt76_dma_get_buf(dev, q, idx, len, info, more);
  250. }
  251. static int
  252. mt76_dma_tx_queue_skb_raw(struct mt76_dev *dev, struct mt76_queue *q,
  253. struct sk_buff *skb, u32 tx_info)
  254. {
  255. struct mt76_queue_buf buf = {};
  256. dma_addr_t addr;
  257. if (q->queued + 1 >= q->ndesc - 1)
  258. goto error;
  259. addr = dma_map_single(dev->dev, skb->data, skb->len,
  260. DMA_TO_DEVICE);
  261. if (unlikely(dma_mapping_error(dev->dev, addr)))
  262. goto error;
  263. buf.addr = addr;
  264. buf.len = skb->len;
  265. spin_lock_bh(&q->lock);
  266. mt76_dma_add_buf(dev, q, &buf, 1, tx_info, skb, NULL);
  267. mt76_dma_kick_queue(dev, q);
  268. spin_unlock_bh(&q->lock);
  269. return 0;
  270. error:
  271. dev_kfree_skb(skb);
  272. return -ENOMEM;
  273. }
  274. static int
  275. mt76_dma_tx_queue_skb(struct mt76_dev *dev, struct mt76_queue *q,
  276. struct sk_buff *skb, struct mt76_wcid *wcid,
  277. struct ieee80211_sta *sta)
  278. {
  279. struct ieee80211_tx_status status = {
  280. .sta = sta,
  281. };
  282. struct mt76_tx_info tx_info = {
  283. .skb = skb,
  284. };
  285. struct ieee80211_hw *hw;
  286. int len, n = 0, ret = -ENOMEM;
  287. struct mt76_txwi_cache *t;
  288. struct sk_buff *iter;
  289. dma_addr_t addr;
  290. u8 *txwi;
  291. t = mt76_get_txwi(dev);
  292. if (!t)
  293. goto free_skb;
  294. txwi = mt76_get_txwi_ptr(dev, t);
  295. skb->prev = skb->next = NULL;
  296. if (dev->drv->drv_flags & MT_DRV_TX_ALIGNED4_SKBS)
  297. mt76_insert_hdr_pad(skb);
  298. len = skb_headlen(skb);
  299. addr = dma_map_single(dev->dev, skb->data, len, DMA_TO_DEVICE);
  300. if (unlikely(dma_mapping_error(dev->dev, addr)))
  301. goto free;
  302. tx_info.buf[n].addr = t->dma_addr;
  303. tx_info.buf[n++].len = dev->drv->txwi_size;
  304. tx_info.buf[n].addr = addr;
  305. tx_info.buf[n++].len = len;
  306. skb_walk_frags(skb, iter) {
  307. if (n == ARRAY_SIZE(tx_info.buf))
  308. goto unmap;
  309. addr = dma_map_single(dev->dev, iter->data, iter->len,
  310. DMA_TO_DEVICE);
  311. if (unlikely(dma_mapping_error(dev->dev, addr)))
  312. goto unmap;
  313. tx_info.buf[n].addr = addr;
  314. tx_info.buf[n++].len = iter->len;
  315. }
  316. tx_info.nbuf = n;
  317. if (q->queued + (tx_info.nbuf + 1) / 2 >= q->ndesc - 1) {
  318. ret = -ENOMEM;
  319. goto unmap;
  320. }
  321. dma_sync_single_for_cpu(dev->dev, t->dma_addr, dev->drv->txwi_size,
  322. DMA_TO_DEVICE);
  323. ret = dev->drv->tx_prepare_skb(dev, txwi, q->qid, wcid, sta, &tx_info);
  324. dma_sync_single_for_device(dev->dev, t->dma_addr, dev->drv->txwi_size,
  325. DMA_TO_DEVICE);
  326. if (ret < 0)
  327. goto unmap;
  328. return mt76_dma_add_buf(dev, q, tx_info.buf, tx_info.nbuf,
  329. tx_info.info, tx_info.skb, t);
  330. unmap:
  331. for (n--; n > 0; n--)
  332. dma_unmap_single(dev->dev, tx_info.buf[n].addr,
  333. tx_info.buf[n].len, DMA_TO_DEVICE);
  334. free:
  335. #ifdef CONFIG_NL80211_TESTMODE
  336. /* fix tx_done accounting on queue overflow */
  337. if (mt76_is_testmode_skb(dev, skb, &hw)) {
  338. struct mt76_phy *phy = hw->priv;
  339. if (tx_info.skb == phy->test.tx_skb)
  340. phy->test.tx_done--;
  341. }
  342. #endif
  343. mt76_put_txwi(dev, t);
  344. free_skb:
  345. status.skb = tx_info.skb;
  346. hw = mt76_tx_status_get_hw(dev, tx_info.skb);
  347. ieee80211_tx_status_ext(hw, &status);
  348. return ret;
  349. }
  350. static int
  351. mt76_dma_rx_fill(struct mt76_dev *dev, struct mt76_queue *q)
  352. {
  353. dma_addr_t addr;
  354. void *buf;
  355. int frames = 0;
  356. int len = SKB_WITH_OVERHEAD(q->buf_size);
  357. int offset = q->buf_offset;
  358. spin_lock_bh(&q->lock);
  359. while (q->queued < q->ndesc - 1) {
  360. struct mt76_queue_buf qbuf;
  361. buf = page_frag_alloc(&q->rx_page, q->buf_size, GFP_ATOMIC);
  362. if (!buf)
  363. break;
  364. addr = dma_map_single(dev->dev, buf, len, DMA_FROM_DEVICE);
  365. if (unlikely(dma_mapping_error(dev->dev, addr))) {
  366. skb_free_frag(buf);
  367. break;
  368. }
  369. qbuf.addr = addr + offset;
  370. qbuf.len = len - offset;
  371. mt76_dma_add_buf(dev, q, &qbuf, 1, 0, buf, NULL);
  372. frames++;
  373. }
  374. if (frames)
  375. mt76_dma_kick_queue(dev, q);
  376. spin_unlock_bh(&q->lock);
  377. return frames;
  378. }
  379. static void
  380. mt76_dma_rx_cleanup(struct mt76_dev *dev, struct mt76_queue *q)
  381. {
  382. struct page *page;
  383. void *buf;
  384. bool more;
  385. spin_lock_bh(&q->lock);
  386. do {
  387. buf = mt76_dma_dequeue(dev, q, true, NULL, NULL, &more);
  388. if (!buf)
  389. break;
  390. skb_free_frag(buf);
  391. } while (1);
  392. spin_unlock_bh(&q->lock);
  393. if (!q->rx_page.va)
  394. return;
  395. page = virt_to_page(q->rx_page.va);
  396. __page_frag_cache_drain(page, q->rx_page.pagecnt_bias);
  397. memset(&q->rx_page, 0, sizeof(q->rx_page));
  398. }
  399. static void
  400. mt76_dma_rx_reset(struct mt76_dev *dev, enum mt76_rxq_id qid)
  401. {
  402. struct mt76_queue *q = &dev->q_rx[qid];
  403. int i;
  404. for (i = 0; i < q->ndesc; i++)
  405. q->desc[i].ctrl = cpu_to_le32(MT_DMA_CTL_DMA_DONE);
  406. mt76_dma_rx_cleanup(dev, q);
  407. mt76_dma_sync_idx(dev, q);
  408. mt76_dma_rx_fill(dev, q);
  409. if (!q->rx_head)
  410. return;
  411. dev_kfree_skb(q->rx_head);
  412. q->rx_head = NULL;
  413. }
  414. static void
  415. mt76_add_fragment(struct mt76_dev *dev, struct mt76_queue *q, void *data,
  416. int len, bool more)
  417. {
  418. struct sk_buff *skb = q->rx_head;
  419. struct skb_shared_info *shinfo = skb_shinfo(skb);
  420. int nr_frags = shinfo->nr_frags;
  421. if (nr_frags < ARRAY_SIZE(shinfo->frags)) {
  422. struct page *page = virt_to_head_page(data);
  423. int offset = data - page_address(page) + q->buf_offset;
  424. skb_add_rx_frag(skb, nr_frags, page, offset, len, q->buf_size);
  425. } else {
  426. skb_free_frag(data);
  427. }
  428. if (more)
  429. return;
  430. q->rx_head = NULL;
  431. if (nr_frags < ARRAY_SIZE(shinfo->frags))
  432. dev->drv->rx_skb(dev, q - dev->q_rx, skb);
  433. else
  434. dev_kfree_skb(skb);
  435. }
  436. static int
  437. mt76_dma_rx_process(struct mt76_dev *dev, struct mt76_queue *q, int budget)
  438. {
  439. int len, data_len, done = 0;
  440. struct sk_buff *skb;
  441. unsigned char *data;
  442. bool more;
  443. while (done < budget) {
  444. u32 info;
  445. data = mt76_dma_dequeue(dev, q, false, &len, &info, &more);
  446. if (!data)
  447. break;
  448. if (q->rx_head)
  449. data_len = q->buf_size;
  450. else
  451. data_len = SKB_WITH_OVERHEAD(q->buf_size);
  452. if (data_len < len + q->buf_offset) {
  453. dev_kfree_skb(q->rx_head);
  454. q->rx_head = NULL;
  455. skb_free_frag(data);
  456. continue;
  457. }
  458. if (q->rx_head) {
  459. mt76_add_fragment(dev, q, data, len, more);
  460. continue;
  461. }
  462. skb = build_skb(data, q->buf_size);
  463. if (!skb) {
  464. skb_free_frag(data);
  465. continue;
  466. }
  467. skb_reserve(skb, q->buf_offset);
  468. if (q == &dev->q_rx[MT_RXQ_MCU]) {
  469. u32 *rxfce = (u32 *)skb->cb;
  470. *rxfce = info;
  471. }
  472. __skb_put(skb, len);
  473. done++;
  474. if (more) {
  475. q->rx_head = skb;
  476. continue;
  477. }
  478. dev->drv->rx_skb(dev, q - dev->q_rx, skb);
  479. }
  480. mt76_dma_rx_fill(dev, q);
  481. return done;
  482. }
  483. int mt76_dma_rx_poll(struct napi_struct *napi, int budget)
  484. {
  485. struct mt76_dev *dev;
  486. int qid, done = 0, cur;
  487. dev = container_of(napi->dev, struct mt76_dev, napi_dev);
  488. qid = napi - dev->napi;
  489. rcu_read_lock();
  490. do {
  491. cur = mt76_dma_rx_process(dev, &dev->q_rx[qid], budget - done);
  492. mt76_rx_poll_complete(dev, qid, napi);
  493. done += cur;
  494. } while (cur && done < budget);
  495. rcu_read_unlock();
  496. if (done < budget && napi_complete(napi))
  497. dev->drv->rx_poll_complete(dev, qid);
  498. return done;
  499. }
  500. EXPORT_SYMBOL_GPL(mt76_dma_rx_poll);
  501. static int
  502. mt76_dma_init(struct mt76_dev *dev,
  503. int (*poll)(struct napi_struct *napi, int budget))
  504. {
  505. int i;
  506. init_dummy_netdev(&dev->napi_dev);
  507. init_dummy_netdev(&dev->tx_napi_dev);
  508. snprintf(dev->napi_dev.name, sizeof(dev->napi_dev.name), "%s",
  509. wiphy_name(dev->hw->wiphy));
  510. dev->napi_dev.threaded = 1;
  511. mt76_for_each_q_rx(dev, i) {
  512. netif_napi_add(&dev->napi_dev, &dev->napi[i], poll, 64);
  513. mt76_dma_rx_fill(dev, &dev->q_rx[i]);
  514. napi_enable(&dev->napi[i]);
  515. }
  516. return 0;
  517. }
  518. static const struct mt76_queue_ops mt76_dma_ops = {
  519. .init = mt76_dma_init,
  520. .alloc = mt76_dma_alloc_queue,
  521. .reset_q = mt76_dma_queue_reset,
  522. .tx_queue_skb_raw = mt76_dma_tx_queue_skb_raw,
  523. .tx_queue_skb = mt76_dma_tx_queue_skb,
  524. .tx_cleanup = mt76_dma_tx_cleanup,
  525. .rx_cleanup = mt76_dma_rx_cleanup,
  526. .rx_reset = mt76_dma_rx_reset,
  527. .kick = mt76_dma_kick_queue,
  528. };
  529. void mt76_dma_attach(struct mt76_dev *dev)
  530. {
  531. dev->queue_ops = &mt76_dma_ops;
  532. }
  533. EXPORT_SYMBOL_GPL(mt76_dma_attach);
  534. void mt76_dma_cleanup(struct mt76_dev *dev)
  535. {
  536. int i;
  537. mt76_worker_disable(&dev->tx_worker);
  538. netif_napi_del(&dev->tx_napi);
  539. for (i = 0; i < ARRAY_SIZE(dev->phy.q_tx); i++) {
  540. mt76_dma_tx_cleanup(dev, dev->phy.q_tx[i], true);
  541. if (dev->phy2)
  542. mt76_dma_tx_cleanup(dev, dev->phy2->q_tx[i], true);
  543. }
  544. for (i = 0; i < ARRAY_SIZE(dev->q_mcu); i++)
  545. mt76_dma_tx_cleanup(dev, dev->q_mcu[i], true);
  546. mt76_for_each_q_rx(dev, i) {
  547. netif_napi_del(&dev->napi[i]);
  548. mt76_dma_rx_cleanup(dev, &dev->q_rx[i]);
  549. }
  550. mt76_free_pending_txwi(dev);
  551. }
  552. EXPORT_SYMBOL_GPL(mt76_dma_cleanup);