tx.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641
  1. // SPDX-License-Identifier: ISC
  2. /*
  3. * Copyright (C) 2016 Felix Fietkau <nbd@nbd.name>
  4. */
  5. #include "mt76.h"
  6. static int
  7. mt76_txq_get_qid(struct ieee80211_txq *txq)
  8. {
  9. if (!txq->sta)
  10. return MT_TXQ_BE;
  11. return txq->ac;
  12. }
  13. void
  14. mt76_tx_check_agg_ssn(struct ieee80211_sta *sta, struct sk_buff *skb)
  15. {
  16. struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
  17. struct ieee80211_txq *txq;
  18. struct mt76_txq *mtxq;
  19. u8 tid;
  20. if (!sta || !ieee80211_is_data_qos(hdr->frame_control) ||
  21. !ieee80211_is_data_present(hdr->frame_control))
  22. return;
  23. tid = skb->priority & IEEE80211_QOS_CTL_TAG1D_MASK;
  24. txq = sta->txq[tid];
  25. mtxq = (struct mt76_txq *)txq->drv_priv;
  26. if (!mtxq->aggr)
  27. return;
  28. mtxq->agg_ssn = le16_to_cpu(hdr->seq_ctrl) + 0x10;
  29. }
  30. EXPORT_SYMBOL_GPL(mt76_tx_check_agg_ssn);
  31. void
  32. mt76_tx_status_lock(struct mt76_dev *dev, struct sk_buff_head *list)
  33. __acquires(&dev->status_list.lock)
  34. {
  35. __skb_queue_head_init(list);
  36. spin_lock_bh(&dev->status_list.lock);
  37. }
  38. EXPORT_SYMBOL_GPL(mt76_tx_status_lock);
  39. void
  40. mt76_tx_status_unlock(struct mt76_dev *dev, struct sk_buff_head *list)
  41. __releases(&dev->status_list.lock)
  42. {
  43. struct ieee80211_hw *hw;
  44. struct sk_buff *skb;
  45. spin_unlock_bh(&dev->status_list.lock);
  46. while ((skb = __skb_dequeue(list)) != NULL) {
  47. hw = mt76_tx_status_get_hw(dev, skb);
  48. ieee80211_tx_status(hw, skb);
  49. }
  50. }
  51. EXPORT_SYMBOL_GPL(mt76_tx_status_unlock);
  52. static void
  53. __mt76_tx_status_skb_done(struct mt76_dev *dev, struct sk_buff *skb, u8 flags,
  54. struct sk_buff_head *list)
  55. {
  56. struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
  57. struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb);
  58. u8 done = MT_TX_CB_DMA_DONE | MT_TX_CB_TXS_DONE;
  59. flags |= cb->flags;
  60. cb->flags = flags;
  61. if ((flags & done) != done)
  62. return;
  63. __skb_unlink(skb, &dev->status_list);
  64. /* Tx status can be unreliable. if it fails, mark the frame as ACKed */
  65. if (flags & MT_TX_CB_TXS_FAILED) {
  66. ieee80211_tx_info_clear_status(info);
  67. info->status.rates[0].idx = -1;
  68. info->flags |= IEEE80211_TX_STAT_ACK;
  69. }
  70. __skb_queue_tail(list, skb);
  71. }
  72. void
  73. mt76_tx_status_skb_done(struct mt76_dev *dev, struct sk_buff *skb,
  74. struct sk_buff_head *list)
  75. {
  76. __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_TXS_DONE, list);
  77. }
  78. EXPORT_SYMBOL_GPL(mt76_tx_status_skb_done);
  79. int
  80. mt76_tx_status_skb_add(struct mt76_dev *dev, struct mt76_wcid *wcid,
  81. struct sk_buff *skb)
  82. {
  83. struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
  84. struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb);
  85. int pid;
  86. if (!wcid)
  87. return MT_PACKET_ID_NO_ACK;
  88. if (info->flags & IEEE80211_TX_CTL_NO_ACK)
  89. return MT_PACKET_ID_NO_ACK;
  90. if (!(info->flags & (IEEE80211_TX_CTL_REQ_TX_STATUS |
  91. IEEE80211_TX_CTL_RATE_CTRL_PROBE)))
  92. return MT_PACKET_ID_NO_SKB;
  93. spin_lock_bh(&dev->status_list.lock);
  94. memset(cb, 0, sizeof(*cb));
  95. wcid->packet_id = (wcid->packet_id + 1) & MT_PACKET_ID_MASK;
  96. if (wcid->packet_id == MT_PACKET_ID_NO_ACK ||
  97. wcid->packet_id == MT_PACKET_ID_NO_SKB)
  98. wcid->packet_id = MT_PACKET_ID_FIRST;
  99. pid = wcid->packet_id;
  100. cb->wcid = wcid->idx;
  101. cb->pktid = pid;
  102. cb->jiffies = jiffies;
  103. __skb_queue_tail(&dev->status_list, skb);
  104. spin_unlock_bh(&dev->status_list.lock);
  105. return pid;
  106. }
  107. EXPORT_SYMBOL_GPL(mt76_tx_status_skb_add);
  108. struct sk_buff *
  109. mt76_tx_status_skb_get(struct mt76_dev *dev, struct mt76_wcid *wcid, int pktid,
  110. struct sk_buff_head *list)
  111. {
  112. struct sk_buff *skb, *tmp;
  113. skb_queue_walk_safe(&dev->status_list, skb, tmp) {
  114. struct mt76_tx_cb *cb = mt76_tx_skb_cb(skb);
  115. if (wcid && cb->wcid != wcid->idx)
  116. continue;
  117. if (cb->pktid == pktid)
  118. return skb;
  119. if (pktid >= 0 && !time_after(jiffies, cb->jiffies +
  120. MT_TX_STATUS_SKB_TIMEOUT))
  121. continue;
  122. __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_TXS_FAILED |
  123. MT_TX_CB_TXS_DONE, list);
  124. }
  125. return NULL;
  126. }
  127. EXPORT_SYMBOL_GPL(mt76_tx_status_skb_get);
  128. void
  129. mt76_tx_status_check(struct mt76_dev *dev, struct mt76_wcid *wcid, bool flush)
  130. {
  131. struct sk_buff_head list;
  132. mt76_tx_status_lock(dev, &list);
  133. mt76_tx_status_skb_get(dev, wcid, flush ? -1 : 0, &list);
  134. mt76_tx_status_unlock(dev, &list);
  135. }
  136. EXPORT_SYMBOL_GPL(mt76_tx_status_check);
  137. static void
  138. mt76_tx_check_non_aql(struct mt76_dev *dev, u16 wcid_idx, struct sk_buff *skb)
  139. {
  140. struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
  141. struct mt76_wcid *wcid;
  142. int pending;
  143. if (info->tx_time_est)
  144. return;
  145. if (wcid_idx >= ARRAY_SIZE(dev->wcid))
  146. return;
  147. rcu_read_lock();
  148. wcid = rcu_dereference(dev->wcid[wcid_idx]);
  149. if (wcid) {
  150. pending = atomic_dec_return(&wcid->non_aql_packets);
  151. if (pending < 0)
  152. atomic_cmpxchg(&wcid->non_aql_packets, pending, 0);
  153. }
  154. rcu_read_unlock();
  155. }
  156. void mt76_tx_complete_skb(struct mt76_dev *dev, u16 wcid_idx, struct sk_buff *skb)
  157. {
  158. struct ieee80211_hw *hw;
  159. struct sk_buff_head list;
  160. mt76_tx_check_non_aql(dev, wcid_idx, skb);
  161. #ifdef CONFIG_NL80211_TESTMODE
  162. if (mt76_is_testmode_skb(dev, skb, &hw)) {
  163. struct mt76_phy *phy = hw->priv;
  164. if (skb == phy->test.tx_skb)
  165. phy->test.tx_done++;
  166. if (phy->test.tx_queued == phy->test.tx_done)
  167. wake_up(&dev->tx_wait);
  168. ieee80211_free_txskb(hw, skb);
  169. return;
  170. }
  171. #endif
  172. if (!skb->prev) {
  173. hw = mt76_tx_status_get_hw(dev, skb);
  174. ieee80211_free_txskb(hw, skb);
  175. return;
  176. }
  177. mt76_tx_status_lock(dev, &list);
  178. __mt76_tx_status_skb_done(dev, skb, MT_TX_CB_DMA_DONE, &list);
  179. mt76_tx_status_unlock(dev, &list);
  180. }
  181. EXPORT_SYMBOL_GPL(mt76_tx_complete_skb);
  182. static int
  183. __mt76_tx_queue_skb(struct mt76_phy *phy, int qid, struct sk_buff *skb,
  184. struct mt76_wcid *wcid, struct ieee80211_sta *sta,
  185. bool *stop)
  186. {
  187. struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
  188. struct mt76_queue *q = phy->q_tx[qid];
  189. struct mt76_dev *dev = phy->dev;
  190. bool non_aql;
  191. int pending;
  192. int idx;
  193. non_aql = !info->tx_time_est;
  194. idx = dev->queue_ops->tx_queue_skb(dev, q, skb, wcid, sta);
  195. if (idx < 0 || !sta || !non_aql)
  196. return idx;
  197. wcid = (struct mt76_wcid *)sta->drv_priv;
  198. q->entry[idx].wcid = wcid->idx;
  199. pending = atomic_inc_return(&wcid->non_aql_packets);
  200. if (stop && pending >= MT_MAX_NON_AQL_PKT)
  201. *stop = true;
  202. return idx;
  203. }
  204. void
  205. mt76_tx(struct mt76_phy *phy, struct ieee80211_sta *sta,
  206. struct mt76_wcid *wcid, struct sk_buff *skb)
  207. {
  208. struct mt76_dev *dev = phy->dev;
  209. struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
  210. struct ieee80211_hdr *hdr = (struct ieee80211_hdr *)skb->data;
  211. struct mt76_queue *q;
  212. int qid = skb_get_queue_mapping(skb);
  213. bool ext_phy = phy != &dev->phy;
  214. if (mt76_testmode_enabled(phy)) {
  215. ieee80211_free_txskb(phy->hw, skb);
  216. return;
  217. }
  218. if (WARN_ON(qid >= MT_TXQ_PSD)) {
  219. qid = MT_TXQ_BE;
  220. skb_set_queue_mapping(skb, qid);
  221. }
  222. if ((dev->drv->drv_flags & MT_DRV_HW_MGMT_TXQ) &&
  223. !(info->flags & IEEE80211_TX_CTL_HW_80211_ENCAP) &&
  224. !ieee80211_is_data(hdr->frame_control) &&
  225. !ieee80211_is_bufferable_mmpdu(hdr->frame_control)) {
  226. qid = MT_TXQ_PSD;
  227. skb_set_queue_mapping(skb, qid);
  228. }
  229. if (!(wcid->tx_info & MT_WCID_TX_INFO_SET))
  230. ieee80211_get_tx_rates(info->control.vif, sta, skb,
  231. info->control.rates, 1);
  232. if (ext_phy)
  233. info->hw_queue |= MT_TX_HW_QUEUE_EXT_PHY;
  234. q = phy->q_tx[qid];
  235. spin_lock_bh(&q->lock);
  236. __mt76_tx_queue_skb(phy, qid, skb, wcid, sta, NULL);
  237. dev->queue_ops->kick(dev, q);
  238. spin_unlock_bh(&q->lock);
  239. }
  240. EXPORT_SYMBOL_GPL(mt76_tx);
  241. static struct sk_buff *
  242. mt76_txq_dequeue(struct mt76_phy *phy, struct mt76_txq *mtxq)
  243. {
  244. struct ieee80211_txq *txq = mtxq_to_txq(mtxq);
  245. struct ieee80211_tx_info *info;
  246. bool ext_phy = phy != &phy->dev->phy;
  247. struct sk_buff *skb;
  248. skb = ieee80211_tx_dequeue(phy->hw, txq);
  249. if (!skb)
  250. return NULL;
  251. info = IEEE80211_SKB_CB(skb);
  252. if (ext_phy)
  253. info->hw_queue |= MT_TX_HW_QUEUE_EXT_PHY;
  254. return skb;
  255. }
  256. static void
  257. mt76_queue_ps_skb(struct mt76_phy *phy, struct ieee80211_sta *sta,
  258. struct sk_buff *skb, bool last)
  259. {
  260. struct mt76_wcid *wcid = (struct mt76_wcid *)sta->drv_priv;
  261. struct ieee80211_tx_info *info = IEEE80211_SKB_CB(skb);
  262. info->control.flags |= IEEE80211_TX_CTRL_PS_RESPONSE;
  263. if (last)
  264. info->flags |= IEEE80211_TX_STATUS_EOSP |
  265. IEEE80211_TX_CTL_REQ_TX_STATUS;
  266. mt76_skb_set_moredata(skb, !last);
  267. __mt76_tx_queue_skb(phy, MT_TXQ_PSD, skb, wcid, sta, NULL);
  268. }
  269. void
  270. mt76_release_buffered_frames(struct ieee80211_hw *hw, struct ieee80211_sta *sta,
  271. u16 tids, int nframes,
  272. enum ieee80211_frame_release_type reason,
  273. bool more_data)
  274. {
  275. struct mt76_phy *phy = hw->priv;
  276. struct mt76_dev *dev = phy->dev;
  277. struct sk_buff *last_skb = NULL;
  278. struct mt76_queue *hwq = phy->q_tx[MT_TXQ_PSD];
  279. int i;
  280. spin_lock_bh(&hwq->lock);
  281. for (i = 0; tids && nframes; i++, tids >>= 1) {
  282. struct ieee80211_txq *txq = sta->txq[i];
  283. struct mt76_txq *mtxq = (struct mt76_txq *)txq->drv_priv;
  284. struct sk_buff *skb;
  285. if (!(tids & 1))
  286. continue;
  287. do {
  288. skb = mt76_txq_dequeue(phy, mtxq);
  289. if (!skb)
  290. break;
  291. nframes--;
  292. if (last_skb)
  293. mt76_queue_ps_skb(phy, sta, last_skb, false);
  294. last_skb = skb;
  295. } while (nframes);
  296. }
  297. if (last_skb) {
  298. mt76_queue_ps_skb(phy, sta, last_skb, true);
  299. dev->queue_ops->kick(dev, hwq);
  300. } else {
  301. ieee80211_sta_eosp(sta);
  302. }
  303. spin_unlock_bh(&hwq->lock);
  304. }
  305. EXPORT_SYMBOL_GPL(mt76_release_buffered_frames);
  306. static bool
  307. mt76_txq_stopped(struct mt76_queue *q)
  308. {
  309. return q->stopped || q->blocked ||
  310. q->queued + MT_TXQ_FREE_THR >= q->ndesc;
  311. }
  312. static int
  313. mt76_txq_send_burst(struct mt76_phy *phy, struct mt76_queue *q,
  314. struct mt76_txq *mtxq)
  315. {
  316. struct mt76_dev *dev = phy->dev;
  317. struct ieee80211_txq *txq = mtxq_to_txq(mtxq);
  318. enum mt76_txq_id qid = mt76_txq_get_qid(txq);
  319. struct mt76_wcid *wcid = mtxq->wcid;
  320. struct ieee80211_tx_info *info;
  321. struct sk_buff *skb;
  322. int n_frames = 1;
  323. bool stop = false;
  324. int idx;
  325. if (test_bit(MT_WCID_FLAG_PS, &wcid->flags))
  326. return 0;
  327. if (atomic_read(&wcid->non_aql_packets) >= MT_MAX_NON_AQL_PKT)
  328. return 0;
  329. skb = mt76_txq_dequeue(phy, mtxq);
  330. if (!skb)
  331. return 0;
  332. info = IEEE80211_SKB_CB(skb);
  333. if (!(wcid->tx_info & MT_WCID_TX_INFO_SET))
  334. ieee80211_get_tx_rates(txq->vif, txq->sta, skb,
  335. info->control.rates, 1);
  336. idx = __mt76_tx_queue_skb(phy, qid, skb, wcid, txq->sta, &stop);
  337. if (idx < 0)
  338. return idx;
  339. do {
  340. if (test_bit(MT76_STATE_PM, &phy->state) ||
  341. test_bit(MT76_RESET, &phy->state))
  342. return -EBUSY;
  343. if (stop || mt76_txq_stopped(q))
  344. break;
  345. skb = mt76_txq_dequeue(phy, mtxq);
  346. if (!skb)
  347. break;
  348. info = IEEE80211_SKB_CB(skb);
  349. if (!(wcid->tx_info & MT_WCID_TX_INFO_SET))
  350. ieee80211_get_tx_rates(txq->vif, txq->sta, skb,
  351. info->control.rates, 1);
  352. idx = __mt76_tx_queue_skb(phy, qid, skb, wcid, txq->sta, &stop);
  353. if (idx < 0)
  354. break;
  355. n_frames++;
  356. } while (1);
  357. dev->queue_ops->kick(dev, q);
  358. return n_frames;
  359. }
  360. static int
  361. mt76_txq_schedule_list(struct mt76_phy *phy, enum mt76_txq_id qid)
  362. {
  363. struct mt76_queue *q = phy->q_tx[qid];
  364. struct mt76_dev *dev = phy->dev;
  365. struct ieee80211_txq *txq;
  366. struct mt76_txq *mtxq;
  367. struct mt76_wcid *wcid;
  368. int ret = 0;
  369. while (1) {
  370. if (test_bit(MT76_STATE_PM, &phy->state) ||
  371. test_bit(MT76_RESET, &phy->state)) {
  372. ret = -EBUSY;
  373. break;
  374. }
  375. if (dev->queue_ops->tx_cleanup &&
  376. q->queued + 2 * MT_TXQ_FREE_THR >= q->ndesc) {
  377. dev->queue_ops->tx_cleanup(dev, q, false);
  378. }
  379. txq = ieee80211_next_txq(phy->hw, qid);
  380. if (!txq)
  381. break;
  382. mtxq = (struct mt76_txq *)txq->drv_priv;
  383. wcid = mtxq->wcid;
  384. if (wcid && test_bit(MT_WCID_FLAG_PS, &wcid->flags))
  385. continue;
  386. spin_lock_bh(&q->lock);
  387. if (mtxq->send_bar && mtxq->aggr) {
  388. struct ieee80211_txq *txq = mtxq_to_txq(mtxq);
  389. struct ieee80211_sta *sta = txq->sta;
  390. struct ieee80211_vif *vif = txq->vif;
  391. u16 agg_ssn = mtxq->agg_ssn;
  392. u8 tid = txq->tid;
  393. mtxq->send_bar = false;
  394. spin_unlock_bh(&q->lock);
  395. ieee80211_send_bar(vif, sta->addr, tid, agg_ssn);
  396. spin_lock_bh(&q->lock);
  397. }
  398. if (!mt76_txq_stopped(q))
  399. ret += mt76_txq_send_burst(phy, q, mtxq);
  400. spin_unlock_bh(&q->lock);
  401. ieee80211_return_txq(phy->hw, txq, false);
  402. }
  403. return ret;
  404. }
  405. void mt76_txq_schedule(struct mt76_phy *phy, enum mt76_txq_id qid)
  406. {
  407. int len;
  408. if (qid >= 4)
  409. return;
  410. rcu_read_lock();
  411. do {
  412. ieee80211_txq_schedule_start(phy->hw, qid);
  413. len = mt76_txq_schedule_list(phy, qid);
  414. ieee80211_txq_schedule_end(phy->hw, qid);
  415. } while (len > 0);
  416. rcu_read_unlock();
  417. }
  418. EXPORT_SYMBOL_GPL(mt76_txq_schedule);
  419. void mt76_txq_schedule_all(struct mt76_phy *phy)
  420. {
  421. int i;
  422. for (i = 0; i <= MT_TXQ_BK; i++)
  423. mt76_txq_schedule(phy, i);
  424. }
  425. EXPORT_SYMBOL_GPL(mt76_txq_schedule_all);
  426. void mt76_tx_worker(struct mt76_worker *w)
  427. {
  428. struct mt76_dev *dev = container_of(w, struct mt76_dev, tx_worker);
  429. mt76_txq_schedule_all(&dev->phy);
  430. if (dev->phy2)
  431. mt76_txq_schedule_all(dev->phy2);
  432. #ifdef CONFIG_NL80211_TESTMODE
  433. if (dev->phy.test.tx_pending)
  434. mt76_testmode_tx_pending(&dev->phy);
  435. if (dev->phy2 && dev->phy2->test.tx_pending)
  436. mt76_testmode_tx_pending(dev->phy2);
  437. #endif
  438. }
  439. void mt76_stop_tx_queues(struct mt76_phy *phy, struct ieee80211_sta *sta,
  440. bool send_bar)
  441. {
  442. int i;
  443. for (i = 0; i < ARRAY_SIZE(sta->txq); i++) {
  444. struct ieee80211_txq *txq = sta->txq[i];
  445. struct mt76_queue *hwq;
  446. struct mt76_txq *mtxq;
  447. if (!txq)
  448. continue;
  449. hwq = phy->q_tx[mt76_txq_get_qid(txq)];
  450. mtxq = (struct mt76_txq *)txq->drv_priv;
  451. spin_lock_bh(&hwq->lock);
  452. mtxq->send_bar = mtxq->aggr && send_bar;
  453. spin_unlock_bh(&hwq->lock);
  454. }
  455. }
  456. EXPORT_SYMBOL_GPL(mt76_stop_tx_queues);
  457. void mt76_wake_tx_queue(struct ieee80211_hw *hw, struct ieee80211_txq *txq)
  458. {
  459. struct mt76_phy *phy = hw->priv;
  460. struct mt76_dev *dev = phy->dev;
  461. if (!test_bit(MT76_STATE_RUNNING, &phy->state))
  462. return;
  463. mt76_worker_schedule(&dev->tx_worker);
  464. }
  465. EXPORT_SYMBOL_GPL(mt76_wake_tx_queue);
  466. u8 mt76_ac_to_hwq(u8 ac)
  467. {
  468. static const u8 wmm_queue_map[] = {
  469. [IEEE80211_AC_BE] = 0,
  470. [IEEE80211_AC_BK] = 1,
  471. [IEEE80211_AC_VI] = 2,
  472. [IEEE80211_AC_VO] = 3,
  473. };
  474. if (WARN_ON(ac >= IEEE80211_NUM_ACS))
  475. return 0;
  476. return wmm_queue_map[ac];
  477. }
  478. EXPORT_SYMBOL_GPL(mt76_ac_to_hwq);
  479. int mt76_skb_adjust_pad(struct sk_buff *skb, int pad)
  480. {
  481. struct sk_buff *iter, *last = skb;
  482. /* First packet of a A-MSDU burst keeps track of the whole burst
  483. * length, need to update length of it and the last packet.
  484. */
  485. skb_walk_frags(skb, iter) {
  486. last = iter;
  487. if (!iter->next) {
  488. skb->data_len += pad;
  489. skb->len += pad;
  490. break;
  491. }
  492. }
  493. if (skb_pad(last, pad))
  494. return -ENOMEM;
  495. __skb_put(last, pad);
  496. return 0;
  497. }
  498. EXPORT_SYMBOL_GPL(mt76_skb_adjust_pad);
  499. void mt76_queue_tx_complete(struct mt76_dev *dev, struct mt76_queue *q,
  500. struct mt76_queue_entry *e)
  501. {
  502. if (e->skb)
  503. dev->drv->tx_complete_skb(dev, e);
  504. spin_lock_bh(&q->lock);
  505. q->tail = (q->tail + 1) % q->ndesc;
  506. q->queued--;
  507. spin_unlock_bh(&q->lock);
  508. }
  509. EXPORT_SYMBOL_GPL(mt76_queue_tx_complete);