blob: be841718eb49db6f85e009b82f6a7da671ea2dc9 [file] [log] [blame]
Ioana Ciornei0bb29b22018-07-31 12:02:47 -05001// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002/* Copyright 2014-2016 Freescale Semiconductor Inc.
3 * Copyright 2016-2017 NXP
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05004 */
5#include <linux/init.h>
6#include <linux/module.h>
7#include <linux/platform_device.h>
8#include <linux/etherdevice.h>
9#include <linux/of_net.h>
10#include <linux/interrupt.h>
11#include <linux/msi.h>
12#include <linux/kthread.h>
Ioana Radulescu08eb2392017-05-24 07:13:27 -050013#include <linux/iommu.h>
Ioana Radulescu859f9982018-04-26 18:23:47 +080014#include <linux/net_tstamp.h>
Bogdan Purcareata6bd067c2018-02-05 08:07:42 -060015#include <linux/fsl/mc.h>
Ioana Ciocoi Radulescu7e273a82018-11-26 16:27:29 +000016#include <linux/bpf.h>
17#include <linux/bpf_trace.h>
Ioana Radulescu859f9982018-04-26 18:23:47 +080018#include <net/sock.h>
19
Ioana Radulescu6e2387e2017-04-28 04:50:29 -050020#include "dpaa2-eth.h"
21
Ioana Radulescu56361872017-04-28 04:50:32 -050022/* CREATE_TRACE_POINTS only needs to be defined once. Other dpa files
23 * using trace events only need to #include <trace/events/sched.h>
24 */
25#define CREATE_TRACE_POINTS
26#include "dpaa2-eth-trace.h"
27
Ioana Radulescu6e2387e2017-04-28 04:50:29 -050028MODULE_LICENSE("Dual BSD/GPL");
29MODULE_AUTHOR("Freescale Semiconductor, Inc");
30MODULE_DESCRIPTION("Freescale DPAA2 Ethernet Driver");
31
Ioana Radulescu08eb2392017-05-24 07:13:27 -050032static void *dpaa2_iova_to_virt(struct iommu_domain *domain,
33 dma_addr_t iova_addr)
34{
35 phys_addr_t phys_addr;
36
37 phys_addr = domain ? iommu_iova_to_phys(domain, iova_addr) : iova_addr;
38
39 return phys_to_virt(phys_addr);
40}
41
Ioana Radulescu6e2387e2017-04-28 04:50:29 -050042static void validate_rx_csum(struct dpaa2_eth_priv *priv,
43 u32 fd_status,
44 struct sk_buff *skb)
45{
46 skb_checksum_none_assert(skb);
47
48 /* HW checksum validation is disabled, nothing to do here */
49 if (!(priv->net_dev->features & NETIF_F_RXCSUM))
50 return;
51
52 /* Read checksum validation bits */
53 if (!((fd_status & DPAA2_FAS_L3CV) &&
54 (fd_status & DPAA2_FAS_L4CV)))
55 return;
56
57 /* Inform the stack there's no need to compute L3/L4 csum anymore */
58 skb->ip_summed = CHECKSUM_UNNECESSARY;
59}
60
61/* Free a received FD.
62 * Not to be used for Tx conf FDs or on any other paths.
63 */
64static void free_rx_fd(struct dpaa2_eth_priv *priv,
65 const struct dpaa2_fd *fd,
66 void *vaddr)
67{
68 struct device *dev = priv->net_dev->dev.parent;
69 dma_addr_t addr = dpaa2_fd_get_addr(fd);
70 u8 fd_format = dpaa2_fd_get_format(fd);
71 struct dpaa2_sg_entry *sgt;
72 void *sg_vaddr;
73 int i;
74
75 /* If single buffer frame, just free the data buffer */
76 if (fd_format == dpaa2_fd_single)
77 goto free_buf;
78 else if (fd_format != dpaa2_fd_sg)
79 /* We don't support any other format */
80 return;
81
Ioana Radulescu729d79b2017-10-11 08:29:48 -050082 /* For S/G frames, we first need to free all SG entries
83 * except the first one, which was taken care of already
84 */
Ioana Radulescu6e2387e2017-04-28 04:50:29 -050085 sgt = vaddr + dpaa2_fd_get_offset(fd);
Ioana Radulescu729d79b2017-10-11 08:29:48 -050086 for (i = 1; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) {
Ioana Radulescu6e2387e2017-04-28 04:50:29 -050087 addr = dpaa2_sg_get_addr(&sgt[i]);
Ioana Radulescu08eb2392017-05-24 07:13:27 -050088 sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -050089 dma_unmap_single(dev, addr, DPAA2_ETH_RX_BUF_SIZE,
Ioana Ciocoi Radulescu18c2e772018-11-26 16:27:32 +000090 DMA_BIDIRECTIONAL);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -050091
Ioana Radulescu6e2387e2017-04-28 04:50:29 -050092 skb_free_frag(sg_vaddr);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -050093 if (dpaa2_sg_is_final(&sgt[i]))
94 break;
95 }
96
97free_buf:
98 skb_free_frag(vaddr);
99}
100
101/* Build a linear skb based on a single-buffer frame descriptor */
Ioana Ciorneifdb6ca92018-10-12 16:27:35 +0000102static struct sk_buff *build_linear_skb(struct dpaa2_eth_channel *ch,
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500103 const struct dpaa2_fd *fd,
104 void *fd_vaddr)
105{
106 struct sk_buff *skb = NULL;
107 u16 fd_offset = dpaa2_fd_get_offset(fd);
108 u32 fd_length = dpaa2_fd_get_len(fd);
109
Ioana Radulescucbb3ea42017-10-11 08:29:44 -0500110 ch->buf_count--;
111
Bogdan Purcareata4b2d9fe2017-10-29 08:20:43 +0000112 skb = build_skb(fd_vaddr, DPAA2_ETH_SKB_SIZE);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500113 if (unlikely(!skb))
114 return NULL;
115
116 skb_reserve(skb, fd_offset);
117 skb_put(skb, fd_length);
118
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500119 return skb;
120}
121
122/* Build a non linear (fragmented) skb based on a S/G table */
123static struct sk_buff *build_frag_skb(struct dpaa2_eth_priv *priv,
124 struct dpaa2_eth_channel *ch,
125 struct dpaa2_sg_entry *sgt)
126{
127 struct sk_buff *skb = NULL;
128 struct device *dev = priv->net_dev->dev.parent;
129 void *sg_vaddr;
130 dma_addr_t sg_addr;
131 u16 sg_offset;
132 u32 sg_length;
133 struct page *page, *head_page;
134 int page_offset;
135 int i;
136
137 for (i = 0; i < DPAA2_ETH_MAX_SG_ENTRIES; i++) {
138 struct dpaa2_sg_entry *sge = &sgt[i];
139
140 /* NOTE: We only support SG entries in dpaa2_sg_single format,
141 * but this is the only format we may receive from HW anyway
142 */
143
144 /* Get the address and length from the S/G entry */
145 sg_addr = dpaa2_sg_get_addr(sge);
Ioana Radulescu08eb2392017-05-24 07:13:27 -0500146 sg_vaddr = dpaa2_iova_to_virt(priv->iommu_domain, sg_addr);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500147 dma_unmap_single(dev, sg_addr, DPAA2_ETH_RX_BUF_SIZE,
Ioana Ciocoi Radulescu18c2e772018-11-26 16:27:32 +0000148 DMA_BIDIRECTIONAL);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500149
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500150 sg_length = dpaa2_sg_get_len(sge);
151
152 if (i == 0) {
153 /* We build the skb around the first data buffer */
Bogdan Purcareata4b2d9fe2017-10-29 08:20:43 +0000154 skb = build_skb(sg_vaddr, DPAA2_ETH_SKB_SIZE);
Ioana Radulescucbb3ea42017-10-11 08:29:44 -0500155 if (unlikely(!skb)) {
Ioana Radulescu729d79b2017-10-11 08:29:48 -0500156 /* Free the first SG entry now, since we already
157 * unmapped it and obtained the virtual address
158 */
159 skb_free_frag(sg_vaddr);
160
Ioana Radulescucbb3ea42017-10-11 08:29:44 -0500161 /* We still need to subtract the buffers used
162 * by this FD from our software counter
163 */
164 while (!dpaa2_sg_is_final(&sgt[i]) &&
165 i < DPAA2_ETH_MAX_SG_ENTRIES)
166 i++;
167 break;
168 }
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500169
170 sg_offset = dpaa2_sg_get_offset(sge);
171 skb_reserve(skb, sg_offset);
172 skb_put(skb, sg_length);
173 } else {
174 /* Rest of the data buffers are stored as skb frags */
175 page = virt_to_page(sg_vaddr);
176 head_page = virt_to_head_page(sg_vaddr);
177
178 /* Offset in page (which may be compound).
179 * Data in subsequent SG entries is stored from the
180 * beginning of the buffer, so we don't need to add the
181 * sg_offset.
182 */
183 page_offset = ((unsigned long)sg_vaddr &
184 (PAGE_SIZE - 1)) +
185 (page_address(page) - page_address(head_page));
186
187 skb_add_rx_frag(skb, i - 1, head_page, page_offset,
188 sg_length, DPAA2_ETH_RX_BUF_SIZE);
189 }
190
191 if (dpaa2_sg_is_final(sge))
192 break;
193 }
194
Ioana Radulescub63baf72017-10-11 08:29:45 -0500195 WARN_ONCE(i == DPAA2_ETH_MAX_SG_ENTRIES, "Final bit not set in SGT");
196
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500197 /* Count all data buffers + SG table buffer */
198 ch->buf_count -= i + 2;
199
200 return skb;
201}
202
Ioana Ciocoi Radulescu569375f2018-11-26 16:27:31 +0000203/* Free buffers acquired from the buffer pool or which were meant to
204 * be released in the pool
205 */
206static void free_bufs(struct dpaa2_eth_priv *priv, u64 *buf_array, int count)
207{
208 struct device *dev = priv->net_dev->dev.parent;
209 void *vaddr;
210 int i;
211
212 for (i = 0; i < count; i++) {
213 vaddr = dpaa2_iova_to_virt(priv->iommu_domain, buf_array[i]);
214 dma_unmap_single(dev, buf_array[i], DPAA2_ETH_RX_BUF_SIZE,
Ioana Ciocoi Radulescu18c2e772018-11-26 16:27:32 +0000215 DMA_BIDIRECTIONAL);
Ioana Ciocoi Radulescu569375f2018-11-26 16:27:31 +0000216 skb_free_frag(vaddr);
217 }
218}
219
Ioana Ciocoi Radulescu5d39dc22018-11-26 16:27:31 +0000220static void xdp_release_buf(struct dpaa2_eth_priv *priv,
221 struct dpaa2_eth_channel *ch,
222 dma_addr_t addr)
223{
224 int err;
225
226 ch->xdp.drop_bufs[ch->xdp.drop_cnt++] = addr;
227 if (ch->xdp.drop_cnt < DPAA2_ETH_BUFS_PER_CMD)
228 return;
229
230 while ((err = dpaa2_io_service_release(ch->dpio, priv->bpid,
231 ch->xdp.drop_bufs,
232 ch->xdp.drop_cnt)) == -EBUSY)
233 cpu_relax();
234
235 if (err) {
236 free_bufs(priv, ch->xdp.drop_bufs, ch->xdp.drop_cnt);
237 ch->buf_count -= ch->xdp.drop_cnt;
238 }
239
240 ch->xdp.drop_cnt = 0;
241}
242
Ioana Ciocoi Radulescu99e43522018-11-26 16:27:33 +0000243static int xdp_enqueue(struct dpaa2_eth_priv *priv, struct dpaa2_fd *fd,
244 void *buf_start, u16 queue_id)
245{
246 struct dpaa2_eth_fq *fq;
247 struct dpaa2_faead *faead;
248 u32 ctrl, frc;
249 int i, err;
250
251 /* Mark the egress frame hardware annotation area as valid */
252 frc = dpaa2_fd_get_frc(fd);
253 dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV);
254 dpaa2_fd_set_ctrl(fd, DPAA2_FD_CTRL_ASAL);
255
256 /* Instruct hardware to release the FD buffer directly into
257 * the buffer pool once transmission is completed, instead of
258 * sending a Tx confirmation frame to us
259 */
260 ctrl = DPAA2_FAEAD_A4V | DPAA2_FAEAD_A2V | DPAA2_FAEAD_EBDDV;
261 faead = dpaa2_get_faead(buf_start, false);
262 faead->ctrl = cpu_to_le32(ctrl);
263 faead->conf_fqid = 0;
264
265 fq = &priv->fq[queue_id];
266 for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) {
267 err = dpaa2_io_service_enqueue_qd(fq->channel->dpio,
268 priv->tx_qdid, 0,
269 fq->tx_qdbin, fd);
270 if (err != -EBUSY)
271 break;
272 }
273
274 return err;
275}
276
Ioana Ciocoi Radulescu7e273a82018-11-26 16:27:29 +0000277static u32 run_xdp(struct dpaa2_eth_priv *priv,
278 struct dpaa2_eth_channel *ch,
Ioana Ciocoi Radulescu99e43522018-11-26 16:27:33 +0000279 struct dpaa2_eth_fq *rx_fq,
Ioana Ciocoi Radulescu7e273a82018-11-26 16:27:29 +0000280 struct dpaa2_fd *fd, void *vaddr)
281{
Ioana Ciocoi Radulescu5d39dc22018-11-26 16:27:31 +0000282 dma_addr_t addr = dpaa2_fd_get_addr(fd);
Ioana Ciocoi Radulescu99e43522018-11-26 16:27:33 +0000283 struct rtnl_link_stats64 *percpu_stats;
Ioana Ciocoi Radulescu7e273a82018-11-26 16:27:29 +0000284 struct bpf_prog *xdp_prog;
285 struct xdp_buff xdp;
286 u32 xdp_act = XDP_PASS;
Ioana Ciocoi Radulescu99e43522018-11-26 16:27:33 +0000287 int err;
288
289 percpu_stats = this_cpu_ptr(priv->percpu_stats);
Ioana Ciocoi Radulescu7e273a82018-11-26 16:27:29 +0000290
291 rcu_read_lock();
292
293 xdp_prog = READ_ONCE(ch->xdp.prog);
294 if (!xdp_prog)
295 goto out;
296
297 xdp.data = vaddr + dpaa2_fd_get_offset(fd);
298 xdp.data_end = xdp.data + dpaa2_fd_get_len(fd);
Ioana Ciocoi Radulescu7b1eea12018-11-26 16:27:30 +0000299 xdp.data_hard_start = xdp.data - XDP_PACKET_HEADROOM;
Ioana Ciocoi Radulescu7e273a82018-11-26 16:27:29 +0000300 xdp_set_data_meta_invalid(&xdp);
301
302 xdp_act = bpf_prog_run_xdp(xdp_prog, &xdp);
303
Ioana Ciocoi Radulescu7b1eea12018-11-26 16:27:30 +0000304 /* xdp.data pointer may have changed */
305 dpaa2_fd_set_offset(fd, xdp.data - vaddr);
306 dpaa2_fd_set_len(fd, xdp.data_end - xdp.data);
307
Ioana Ciocoi Radulescu7e273a82018-11-26 16:27:29 +0000308 switch (xdp_act) {
309 case XDP_PASS:
310 break;
Ioana Ciocoi Radulescu99e43522018-11-26 16:27:33 +0000311 case XDP_TX:
312 err = xdp_enqueue(priv, fd, vaddr, rx_fq->flowid);
313 if (err) {
314 xdp_release_buf(priv, ch, addr);
315 percpu_stats->tx_errors++;
Ioana Ciocoi Radulescua4a7b762018-11-26 16:27:34 +0000316 ch->stats.xdp_tx_err++;
Ioana Ciocoi Radulescu99e43522018-11-26 16:27:33 +0000317 } else {
318 percpu_stats->tx_packets++;
319 percpu_stats->tx_bytes += dpaa2_fd_get_len(fd);
Ioana Ciocoi Radulescua4a7b762018-11-26 16:27:34 +0000320 ch->stats.xdp_tx++;
Ioana Ciocoi Radulescu99e43522018-11-26 16:27:33 +0000321 }
322 break;
Ioana Ciocoi Radulescu7e273a82018-11-26 16:27:29 +0000323 default:
324 bpf_warn_invalid_xdp_action(xdp_act);
325 case XDP_ABORTED:
326 trace_xdp_exception(priv->net_dev, xdp_prog, xdp_act);
327 case XDP_DROP:
Ioana Ciocoi Radulescu5d39dc22018-11-26 16:27:31 +0000328 xdp_release_buf(priv, ch, addr);
Ioana Ciocoi Radulescua4a7b762018-11-26 16:27:34 +0000329 ch->stats.xdp_drop++;
Ioana Ciocoi Radulescu7e273a82018-11-26 16:27:29 +0000330 break;
331 }
332
333out:
334 rcu_read_unlock();
335 return xdp_act;
336}
337
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500338/* Main Rx frame processing routine */
339static void dpaa2_eth_rx(struct dpaa2_eth_priv *priv,
340 struct dpaa2_eth_channel *ch,
341 const struct dpaa2_fd *fd,
Ioana Ciocoi Radulescudbcdf722018-11-14 11:48:35 +0000342 struct dpaa2_eth_fq *fq)
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500343{
344 dma_addr_t addr = dpaa2_fd_get_addr(fd);
345 u8 fd_format = dpaa2_fd_get_format(fd);
346 void *vaddr;
347 struct sk_buff *skb;
348 struct rtnl_link_stats64 *percpu_stats;
Ioana Radulescu85047ab2017-04-28 04:50:31 -0500349 struct dpaa2_eth_drv_stats *percpu_extras;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500350 struct device *dev = priv->net_dev->dev.parent;
351 struct dpaa2_fas *fas;
Ioana Radulescud695e762017-06-06 10:00:35 -0500352 void *buf_data;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500353 u32 status = 0;
Ioana Ciocoi Radulescu7e273a82018-11-26 16:27:29 +0000354 u32 xdp_act;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500355
Ioana Radulescu56361872017-04-28 04:50:32 -0500356 /* Tracing point */
357 trace_dpaa2_rx_fd(priv->net_dev, fd);
358
Ioana Radulescu08eb2392017-05-24 07:13:27 -0500359 vaddr = dpaa2_iova_to_virt(priv->iommu_domain, addr);
Ioana Ciocoi Radulescu5d39dc22018-11-26 16:27:31 +0000360 dma_sync_single_for_cpu(dev, addr, DPAA2_ETH_RX_BUF_SIZE,
Ioana Ciocoi Radulescu18c2e772018-11-26 16:27:32 +0000361 DMA_BIDIRECTIONAL);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500362
Ioana Radulescu54ce8912017-12-08 06:47:53 -0600363 fas = dpaa2_get_fas(vaddr, false);
Ioana Radulescud695e762017-06-06 10:00:35 -0500364 prefetch(fas);
365 buf_data = vaddr + dpaa2_fd_get_offset(fd);
366 prefetch(buf_data);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500367
368 percpu_stats = this_cpu_ptr(priv->percpu_stats);
Ioana Radulescu85047ab2017-04-28 04:50:31 -0500369 percpu_extras = this_cpu_ptr(priv->percpu_extras);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500370
371 if (fd_format == dpaa2_fd_single) {
Ioana Ciocoi Radulescu99e43522018-11-26 16:27:33 +0000372 xdp_act = run_xdp(priv, ch, fq, (struct dpaa2_fd *)fd, vaddr);
Ioana Ciocoi Radulescu7e273a82018-11-26 16:27:29 +0000373 if (xdp_act != XDP_PASS) {
374 percpu_stats->rx_packets++;
375 percpu_stats->rx_bytes += dpaa2_fd_get_len(fd);
376 return;
377 }
378
Ioana Ciocoi Radulescu5d39dc22018-11-26 16:27:31 +0000379 dma_unmap_single(dev, addr, DPAA2_ETH_RX_BUF_SIZE,
Ioana Ciocoi Radulescu18c2e772018-11-26 16:27:32 +0000380 DMA_BIDIRECTIONAL);
Ioana Ciorneifdb6ca92018-10-12 16:27:35 +0000381 skb = build_linear_skb(ch, fd, vaddr);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500382 } else if (fd_format == dpaa2_fd_sg) {
Ioana Ciocoi Radulescu7e273a82018-11-26 16:27:29 +0000383 WARN_ON(priv->xdp_prog);
384
Ioana Ciocoi Radulescu5d39dc22018-11-26 16:27:31 +0000385 dma_unmap_single(dev, addr, DPAA2_ETH_RX_BUF_SIZE,
Ioana Ciocoi Radulescu18c2e772018-11-26 16:27:32 +0000386 DMA_BIDIRECTIONAL);
Ioana Radulescud695e762017-06-06 10:00:35 -0500387 skb = build_frag_skb(priv, ch, buf_data);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500388 skb_free_frag(vaddr);
Ioana Radulescu85047ab2017-04-28 04:50:31 -0500389 percpu_extras->rx_sg_frames++;
390 percpu_extras->rx_sg_bytes += dpaa2_fd_get_len(fd);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500391 } else {
392 /* We don't support any other format */
393 goto err_frame_format;
394 }
395
396 if (unlikely(!skb))
397 goto err_build_skb;
398
399 prefetch(skb->data);
400
Ioana Radulescu859f9982018-04-26 18:23:47 +0800401 /* Get the timestamp value */
402 if (priv->rx_tstamp) {
403 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
404 __le64 *ts = dpaa2_get_ts(vaddr, false);
405 u64 ns;
406
407 memset(shhwtstamps, 0, sizeof(*shhwtstamps));
408
409 ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts);
410 shhwtstamps->hwtstamp = ns_to_ktime(ns);
411 }
412
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500413 /* Check if we need to validate the L4 csum */
414 if (likely(dpaa2_fd_get_frc(fd) & DPAA2_FD_FRC_FASV)) {
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500415 status = le32_to_cpu(fas->status);
416 validate_rx_csum(priv, status, skb);
417 }
418
419 skb->protocol = eth_type_trans(skb, priv->net_dev);
Ioana Ciocoi Radulescudbcdf722018-11-14 11:48:35 +0000420 skb_record_rx_queue(skb, fq->flowid);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500421
422 percpu_stats->rx_packets++;
423 percpu_stats->rx_bytes += dpaa2_fd_get_len(fd);
424
Ioana Ciocoi Radulescudbcdf722018-11-14 11:48:35 +0000425 napi_gro_receive(&ch->napi, skb);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500426
427 return;
428
429err_build_skb:
430 free_rx_fd(priv, fd, vaddr);
431err_frame_format:
432 percpu_stats->rx_dropped++;
433}
434
435/* Consume all frames pull-dequeued into the store. This is the simplest way to
436 * make sure we don't accidentally issue another volatile dequeue which would
437 * overwrite (leak) frames already in the store.
438 *
439 * Observance of NAPI budget is not our concern, leaving that to the caller.
440 */
Ioana Ciocoi Radulescu68049a52018-10-08 14:16:31 +0000441static int consume_frames(struct dpaa2_eth_channel *ch,
Ioana Ciocoi Radulescu569dac62018-11-14 11:48:36 +0000442 struct dpaa2_eth_fq **src)
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500443{
444 struct dpaa2_eth_priv *priv = ch->priv;
Ioana Ciocoi Radulescu68049a52018-10-08 14:16:31 +0000445 struct dpaa2_eth_fq *fq = NULL;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500446 struct dpaa2_dq *dq;
447 const struct dpaa2_fd *fd;
448 int cleaned = 0;
449 int is_last;
450
451 do {
452 dq = dpaa2_io_store_next(ch->store, &is_last);
453 if (unlikely(!dq)) {
454 /* If we're here, we *must* have placed a
455 * volatile dequeue comnmand, so keep reading through
456 * the store until we get some sort of valid response
457 * token (either a valid frame or an "empty dequeue")
458 */
459 continue;
460 }
461
462 fd = dpaa2_dq_fd(dq);
Ioana Radulescu75c583a2018-02-26 10:28:06 -0600463 fq = (struct dpaa2_eth_fq *)(uintptr_t)dpaa2_dq_fqd_ctx(dq);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500464
Ioana Ciocoi Radulescudbcdf722018-11-14 11:48:35 +0000465 fq->consume(priv, ch, fd, fq);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500466 cleaned++;
467 } while (!is_last);
468
Ioana Ciocoi Radulescu68049a52018-10-08 14:16:31 +0000469 if (!cleaned)
470 return 0;
471
472 fq->stats.frames += cleaned;
Ioana Ciocoi Radulescu68049a52018-10-08 14:16:31 +0000473
474 /* A dequeue operation only pulls frames from a single queue
Ioana Ciocoi Radulescu569dac62018-11-14 11:48:36 +0000475 * into the store. Return the frame queue as an out param.
Ioana Ciocoi Radulescu68049a52018-10-08 14:16:31 +0000476 */
Ioana Ciocoi Radulescu569dac62018-11-14 11:48:36 +0000477 if (src)
478 *src = fq;
Ioana Ciocoi Radulescu68049a52018-10-08 14:16:31 +0000479
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500480 return cleaned;
481}
482
Ioana Radulescu859f9982018-04-26 18:23:47 +0800483/* Configure the egress frame annotation for timestamp update */
484static void enable_tx_tstamp(struct dpaa2_fd *fd, void *buf_start)
485{
486 struct dpaa2_faead *faead;
487 u32 ctrl, frc;
488
489 /* Mark the egress frame annotation area as valid */
490 frc = dpaa2_fd_get_frc(fd);
491 dpaa2_fd_set_frc(fd, frc | DPAA2_FD_FRC_FAEADV);
492
493 /* Set hardware annotation size */
494 ctrl = dpaa2_fd_get_ctrl(fd);
495 dpaa2_fd_set_ctrl(fd, ctrl | DPAA2_FD_CTRL_ASAL);
496
497 /* enable UPD (update prepanded data) bit in FAEAD field of
498 * hardware frame annotation area
499 */
500 ctrl = DPAA2_FAEAD_A2V | DPAA2_FAEAD_UPDV | DPAA2_FAEAD_UPD;
501 faead = dpaa2_get_faead(buf_start, true);
502 faead->ctrl = cpu_to_le32(ctrl);
503}
504
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500505/* Create a frame descriptor based on a fragmented skb */
506static int build_sg_fd(struct dpaa2_eth_priv *priv,
507 struct sk_buff *skb,
508 struct dpaa2_fd *fd)
509{
510 struct device *dev = priv->net_dev->dev.parent;
511 void *sgt_buf = NULL;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500512 dma_addr_t addr;
513 int nr_frags = skb_shinfo(skb)->nr_frags;
514 struct dpaa2_sg_entry *sgt;
515 int i, err;
516 int sgt_buf_size;
517 struct scatterlist *scl, *crt_scl;
518 int num_sg;
519 int num_dma_bufs;
520 struct dpaa2_eth_swa *swa;
521
522 /* Create and map scatterlist.
523 * We don't advertise NETIF_F_FRAGLIST, so skb_to_sgvec() will not have
524 * to go beyond nr_frags+1.
525 * Note: We don't support chained scatterlists
526 */
527 if (unlikely(PAGE_SIZE / sizeof(struct scatterlist) < nr_frags + 1))
528 return -EINVAL;
529
530 scl = kcalloc(nr_frags + 1, sizeof(struct scatterlist), GFP_ATOMIC);
531 if (unlikely(!scl))
532 return -ENOMEM;
533
534 sg_init_table(scl, nr_frags + 1);
535 num_sg = skb_to_sgvec(skb, scl, 0, skb->len);
Ioana Radulescu1e5fa9e2017-05-24 07:13:28 -0500536 num_dma_bufs = dma_map_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500537 if (unlikely(!num_dma_bufs)) {
538 err = -ENOMEM;
539 goto dma_map_sg_failed;
540 }
541
542 /* Prepare the HW SGT structure */
543 sgt_buf_size = priv->tx_data_offset +
Ioana Radulescufa722c02018-03-23 08:44:12 -0500544 sizeof(struct dpaa2_sg_entry) * num_dma_bufs;
Ioana Radulescu6a9bbe52018-03-14 15:04:51 -0500545 sgt_buf = netdev_alloc_frag(sgt_buf_size + DPAA2_ETH_TX_BUF_ALIGN);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500546 if (unlikely(!sgt_buf)) {
547 err = -ENOMEM;
548 goto sgt_buf_alloc_failed;
549 }
550 sgt_buf = PTR_ALIGN(sgt_buf, DPAA2_ETH_TX_BUF_ALIGN);
Ioana Radulescu6a9bbe52018-03-14 15:04:51 -0500551 memset(sgt_buf, 0, sgt_buf_size);
552
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500553 sgt = (struct dpaa2_sg_entry *)(sgt_buf + priv->tx_data_offset);
554
555 /* Fill in the HW SGT structure.
556 *
557 * sgt_buf is zeroed out, so the following fields are implicit
558 * in all sgt entries:
559 * - offset is 0
560 * - format is 'dpaa2_sg_single'
561 */
562 for_each_sg(scl, crt_scl, num_dma_bufs, i) {
563 dpaa2_sg_set_addr(&sgt[i], sg_dma_address(crt_scl));
564 dpaa2_sg_set_len(&sgt[i], sg_dma_len(crt_scl));
565 }
566 dpaa2_sg_set_final(&sgt[i - 1], true);
567
568 /* Store the skb backpointer in the SGT buffer.
569 * Fit the scatterlist and the number of buffers alongside the
570 * skb backpointer in the software annotation area. We'll need
571 * all of them on Tx Conf.
572 */
573 swa = (struct dpaa2_eth_swa *)sgt_buf;
574 swa->skb = skb;
575 swa->scl = scl;
576 swa->num_sg = num_sg;
Ioana Radulescub2718e62018-03-23 08:44:11 -0500577 swa->sgt_size = sgt_buf_size;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500578
579 /* Separately map the SGT buffer */
Ioana Radulescu1e5fa9e2017-05-24 07:13:28 -0500580 addr = dma_map_single(dev, sgt_buf, sgt_buf_size, DMA_BIDIRECTIONAL);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500581 if (unlikely(dma_mapping_error(dev, addr))) {
582 err = -ENOMEM;
583 goto dma_map_single_failed;
584 }
585 dpaa2_fd_set_offset(fd, priv->tx_data_offset);
586 dpaa2_fd_set_format(fd, dpaa2_fd_sg);
587 dpaa2_fd_set_addr(fd, addr);
588 dpaa2_fd_set_len(fd, skb->len);
Ioana Radulescub948c8c2018-10-12 16:27:40 +0000589 dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500590
Ioana Radulescu859f9982018-04-26 18:23:47 +0800591 if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)
592 enable_tx_tstamp(fd, sgt_buf);
593
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500594 return 0;
595
596dma_map_single_failed:
Ioana Radulescu6a9bbe52018-03-14 15:04:51 -0500597 skb_free_frag(sgt_buf);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500598sgt_buf_alloc_failed:
Ioana Radulescu1e5fa9e2017-05-24 07:13:28 -0500599 dma_unmap_sg(dev, scl, num_sg, DMA_BIDIRECTIONAL);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500600dma_map_sg_failed:
601 kfree(scl);
602 return err;
603}
604
605/* Create a frame descriptor based on a linear skb */
606static int build_single_fd(struct dpaa2_eth_priv *priv,
607 struct sk_buff *skb,
608 struct dpaa2_fd *fd)
609{
610 struct device *dev = priv->net_dev->dev.parent;
Ioana Radulescuc1636852017-12-08 06:47:58 -0600611 u8 *buffer_start, *aligned_start;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500612 struct sk_buff **skbh;
613 dma_addr_t addr;
614
Ioana Radulescuc1636852017-12-08 06:47:58 -0600615 buffer_start = skb->data - dpaa2_eth_needed_headroom(priv, skb);
616
617 /* If there's enough room to align the FD address, do it.
618 * It will help hardware optimize accesses.
619 */
620 aligned_start = PTR_ALIGN(buffer_start - DPAA2_ETH_TX_BUF_ALIGN,
621 DPAA2_ETH_TX_BUF_ALIGN);
622 if (aligned_start >= skb->head)
623 buffer_start = aligned_start;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500624
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500625 /* Store a backpointer to the skb at the beginning of the buffer
626 * (in the private data area) such that we can release it
627 * on Tx confirm
628 */
629 skbh = (struct sk_buff **)buffer_start;
630 *skbh = skb;
631
632 addr = dma_map_single(dev, buffer_start,
633 skb_tail_pointer(skb) - buffer_start,
Ioana Radulescu1e5fa9e2017-05-24 07:13:28 -0500634 DMA_BIDIRECTIONAL);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500635 if (unlikely(dma_mapping_error(dev, addr)))
636 return -ENOMEM;
637
638 dpaa2_fd_set_addr(fd, addr);
639 dpaa2_fd_set_offset(fd, (u16)(skb->data - buffer_start));
640 dpaa2_fd_set_len(fd, skb->len);
641 dpaa2_fd_set_format(fd, dpaa2_fd_single);
Ioana Radulescub948c8c2018-10-12 16:27:40 +0000642 dpaa2_fd_set_ctrl(fd, FD_CTRL_PTA);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500643
Ioana Radulescu859f9982018-04-26 18:23:47 +0800644 if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)
645 enable_tx_tstamp(fd, buffer_start);
646
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500647 return 0;
648}
649
650/* FD freeing routine on the Tx path
651 *
652 * DMA-unmap and free FD and possibly SGT buffer allocated on Tx. The skb
653 * back-pointed to is also freed.
654 * This can be called either from dpaa2_eth_tx_conf() or on the error path of
655 * dpaa2_eth_tx().
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500656 */
657static void free_tx_fd(const struct dpaa2_eth_priv *priv,
Ioana Radulescu2b7c86e2017-12-08 06:47:56 -0600658 const struct dpaa2_fd *fd)
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500659{
660 struct device *dev = priv->net_dev->dev.parent;
661 dma_addr_t fd_addr;
662 struct sk_buff **skbh, *skb;
663 unsigned char *buffer_start;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500664 struct dpaa2_eth_swa *swa;
665 u8 fd_format = dpaa2_fd_get_format(fd);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500666
667 fd_addr = dpaa2_fd_get_addr(fd);
Ioana Radulescu08eb2392017-05-24 07:13:27 -0500668 skbh = dpaa2_iova_to_virt(priv->iommu_domain, fd_addr);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500669
670 if (fd_format == dpaa2_fd_single) {
671 skb = *skbh;
672 buffer_start = (unsigned char *)skbh;
673 /* Accessing the skb buffer is safe before dma unmap, because
674 * we didn't map the actual skb shell.
675 */
676 dma_unmap_single(dev, fd_addr,
677 skb_tail_pointer(skb) - buffer_start,
Ioana Radulescu1e5fa9e2017-05-24 07:13:28 -0500678 DMA_BIDIRECTIONAL);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500679 } else if (fd_format == dpaa2_fd_sg) {
680 swa = (struct dpaa2_eth_swa *)skbh;
681 skb = swa->skb;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500682
683 /* Unmap the scatterlist */
Ioana Radulescub2718e62018-03-23 08:44:11 -0500684 dma_unmap_sg(dev, swa->scl, swa->num_sg, DMA_BIDIRECTIONAL);
685 kfree(swa->scl);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500686
687 /* Unmap the SGT buffer */
Ioana Radulescub2718e62018-03-23 08:44:11 -0500688 dma_unmap_single(dev, fd_addr, swa->sgt_size,
689 DMA_BIDIRECTIONAL);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500690 } else {
Ioana Radulescu2b7c86e2017-12-08 06:47:56 -0600691 netdev_dbg(priv->net_dev, "Invalid FD format\n");
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500692 return;
693 }
694
Ioana Radulescu859f9982018-04-26 18:23:47 +0800695 /* Get the timestamp value */
696 if (priv->tx_tstamp && skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) {
697 struct skb_shared_hwtstamps shhwtstamps;
698 __le64 *ts = dpaa2_get_ts(skbh, true);
699 u64 ns;
700
701 memset(&shhwtstamps, 0, sizeof(shhwtstamps));
702
703 ns = DPAA2_PTP_CLK_PERIOD_NS * le64_to_cpup(ts);
704 shhwtstamps.hwtstamp = ns_to_ktime(ns);
705 skb_tstamp_tx(skb, &shhwtstamps);
706 }
707
Ioana Radulescu6a9bbe52018-03-14 15:04:51 -0500708 /* Free SGT buffer allocated on tx */
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500709 if (fd_format != dpaa2_fd_single)
Ioana Radulescu6a9bbe52018-03-14 15:04:51 -0500710 skb_free_frag(skbh);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500711
712 /* Move on with skb release */
713 dev_kfree_skb(skb);
714}
715
Ioana Radulescuc433db42017-06-06 10:00:26 -0500716static netdev_tx_t dpaa2_eth_tx(struct sk_buff *skb, struct net_device *net_dev)
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500717{
718 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
719 struct dpaa2_fd fd;
720 struct rtnl_link_stats64 *percpu_stats;
Ioana Radulescu85047ab2017-04-28 04:50:31 -0500721 struct dpaa2_eth_drv_stats *percpu_extras;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500722 struct dpaa2_eth_fq *fq;
Ioana Ciocoi Radulescu569dac62018-11-14 11:48:36 +0000723 struct netdev_queue *nq;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500724 u16 queue_mapping;
Ioana Radulescu18c21462017-12-08 06:47:57 -0600725 unsigned int needed_headroom;
Ioana Ciocoi Radulescu569dac62018-11-14 11:48:36 +0000726 u32 fd_len;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500727 int err, i;
728
729 percpu_stats = this_cpu_ptr(priv->percpu_stats);
Ioana Radulescu85047ab2017-04-28 04:50:31 -0500730 percpu_extras = this_cpu_ptr(priv->percpu_extras);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500731
Ioana Radulescu18c21462017-12-08 06:47:57 -0600732 needed_headroom = dpaa2_eth_needed_headroom(priv, skb);
733 if (skb_headroom(skb) < needed_headroom) {
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500734 struct sk_buff *ns;
735
Ioana Radulescu18c21462017-12-08 06:47:57 -0600736 ns = skb_realloc_headroom(skb, needed_headroom);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500737 if (unlikely(!ns)) {
738 percpu_stats->tx_dropped++;
739 goto err_alloc_headroom;
740 }
Ioana Radulescu6662b5e2017-12-08 06:47:55 -0600741 percpu_extras->tx_reallocs++;
Ioana Radulescu859f9982018-04-26 18:23:47 +0800742
743 if (skb->sk)
744 skb_set_owner_w(ns, skb->sk);
745
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500746 dev_kfree_skb(skb);
747 skb = ns;
748 }
749
750 /* We'll be holding a back-reference to the skb until Tx Confirmation;
751 * we don't want that overwritten by a concurrent Tx with a cloned skb.
752 */
753 skb = skb_unshare(skb, GFP_ATOMIC);
754 if (unlikely(!skb)) {
755 /* skb_unshare() has already freed the skb */
756 percpu_stats->tx_dropped++;
757 return NETDEV_TX_OK;
758 }
759
760 /* Setup the FD fields */
761 memset(&fd, 0, sizeof(fd));
762
Ioana Radulescu85047ab2017-04-28 04:50:31 -0500763 if (skb_is_nonlinear(skb)) {
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500764 err = build_sg_fd(priv, skb, &fd);
Ioana Radulescu85047ab2017-04-28 04:50:31 -0500765 percpu_extras->tx_sg_frames++;
766 percpu_extras->tx_sg_bytes += skb->len;
767 } else {
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500768 err = build_single_fd(priv, skb, &fd);
Ioana Radulescu85047ab2017-04-28 04:50:31 -0500769 }
770
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500771 if (unlikely(err)) {
772 percpu_stats->tx_dropped++;
773 goto err_build_fd;
774 }
775
Ioana Radulescu56361872017-04-28 04:50:32 -0500776 /* Tracing point */
777 trace_dpaa2_tx_fd(net_dev, &fd);
778
Ioana Radulescu537336c2017-12-21 06:33:20 -0600779 /* TxConf FQ selection relies on queue id from the stack.
780 * In case of a forwarded frame from another DPNI interface, we choose
781 * a queue affined to the same core that processed the Rx frame
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500782 */
Ioana Radulescu537336c2017-12-21 06:33:20 -0600783 queue_mapping = skb_get_queue_mapping(skb);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500784 fq = &priv->fq[queue_mapping];
785 for (i = 0; i < DPAA2_ETH_ENQUEUE_RETRIES; i++) {
Ioana Radulescu7ec05962018-01-05 05:04:32 -0600786 err = dpaa2_io_service_enqueue_qd(fq->channel->dpio,
787 priv->tx_qdid, 0,
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500788 fq->tx_qdbin, &fd);
789 if (err != -EBUSY)
790 break;
791 }
Ioana Radulescu85047ab2017-04-28 04:50:31 -0500792 percpu_extras->tx_portal_busy += i;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500793 if (unlikely(err < 0)) {
794 percpu_stats->tx_errors++;
795 /* Clean up everything, including freeing the skb */
Ioana Radulescu2b7c86e2017-12-08 06:47:56 -0600796 free_tx_fd(priv, &fd);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500797 } else {
Ioana Ciocoi Radulescu569dac62018-11-14 11:48:36 +0000798 fd_len = dpaa2_fd_get_len(&fd);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500799 percpu_stats->tx_packets++;
Ioana Ciocoi Radulescu569dac62018-11-14 11:48:36 +0000800 percpu_stats->tx_bytes += fd_len;
801
802 nq = netdev_get_tx_queue(net_dev, queue_mapping);
803 netdev_tx_sent_queue(nq, fd_len);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500804 }
805
806 return NETDEV_TX_OK;
807
808err_build_fd:
809err_alloc_headroom:
810 dev_kfree_skb(skb);
811
812 return NETDEV_TX_OK;
813}
814
815/* Tx confirmation frame processing routine */
816static void dpaa2_eth_tx_conf(struct dpaa2_eth_priv *priv,
Ioana Ciorneib00c8982018-10-12 16:27:38 +0000817 struct dpaa2_eth_channel *ch __always_unused,
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500818 const struct dpaa2_fd *fd,
Ioana Ciocoi Radulescu569dac62018-11-14 11:48:36 +0000819 struct dpaa2_eth_fq *fq)
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500820{
821 struct rtnl_link_stats64 *percpu_stats;
Ioana Radulescu85047ab2017-04-28 04:50:31 -0500822 struct dpaa2_eth_drv_stats *percpu_extras;
Ioana Ciocoi Radulescu569dac62018-11-14 11:48:36 +0000823 u32 fd_len = dpaa2_fd_get_len(fd);
Ioana Radulescu39163c02017-06-06 10:00:39 -0500824 u32 fd_errors;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500825
Ioana Radulescu56361872017-04-28 04:50:32 -0500826 /* Tracing point */
827 trace_dpaa2_tx_conf_fd(priv->net_dev, fd);
828
Ioana Radulescu85047ab2017-04-28 04:50:31 -0500829 percpu_extras = this_cpu_ptr(priv->percpu_extras);
830 percpu_extras->tx_conf_frames++;
Ioana Ciocoi Radulescu569dac62018-11-14 11:48:36 +0000831 percpu_extras->tx_conf_bytes += fd_len;
832
833 fq->dq_frames++;
834 fq->dq_bytes += fd_len;
Ioana Radulescu85047ab2017-04-28 04:50:31 -0500835
Ioana Radulescu39163c02017-06-06 10:00:39 -0500836 /* Check frame errors in the FD field */
837 fd_errors = dpaa2_fd_get_ctrl(fd) & DPAA2_FD_TX_ERR_MASK;
Ioana Radulescu2b7c86e2017-12-08 06:47:56 -0600838 free_tx_fd(priv, fd);
Ioana Radulescu39163c02017-06-06 10:00:39 -0500839
840 if (likely(!fd_errors))
841 return;
842
Ioana Radulescu2b7c86e2017-12-08 06:47:56 -0600843 if (net_ratelimit())
844 netdev_dbg(priv->net_dev, "TX frame FD error: 0x%08x\n",
845 fd_errors);
846
Ioana Radulescu39163c02017-06-06 10:00:39 -0500847 percpu_stats = this_cpu_ptr(priv->percpu_stats);
848 /* Tx-conf logically pertains to the egress path. */
849 percpu_stats->tx_errors++;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500850}
851
852static int set_rx_csum(struct dpaa2_eth_priv *priv, bool enable)
853{
854 int err;
855
856 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token,
857 DPNI_OFF_RX_L3_CSUM, enable);
858 if (err) {
859 netdev_err(priv->net_dev,
860 "dpni_set_offload(RX_L3_CSUM) failed\n");
861 return err;
862 }
863
864 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token,
865 DPNI_OFF_RX_L4_CSUM, enable);
866 if (err) {
867 netdev_err(priv->net_dev,
868 "dpni_set_offload(RX_L4_CSUM) failed\n");
869 return err;
870 }
871
872 return 0;
873}
874
875static int set_tx_csum(struct dpaa2_eth_priv *priv, bool enable)
876{
877 int err;
878
879 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token,
880 DPNI_OFF_TX_L3_CSUM, enable);
881 if (err) {
882 netdev_err(priv->net_dev, "dpni_set_offload(TX_L3_CSUM) failed\n");
883 return err;
884 }
885
886 err = dpni_set_offload(priv->mc_io, 0, priv->mc_token,
887 DPNI_OFF_TX_L4_CSUM, enable);
888 if (err) {
889 netdev_err(priv->net_dev, "dpni_set_offload(TX_L4_CSUM) failed\n");
890 return err;
891 }
892
893 return 0;
894}
895
896/* Perform a single release command to add buffers
897 * to the specified buffer pool
898 */
Ioana Radulescu7ec05962018-01-05 05:04:32 -0600899static int add_bufs(struct dpaa2_eth_priv *priv,
900 struct dpaa2_eth_channel *ch, u16 bpid)
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500901{
902 struct device *dev = priv->net_dev->dev.parent;
903 u64 buf_array[DPAA2_ETH_BUFS_PER_CMD];
904 void *buf;
905 dma_addr_t addr;
Ioana Radulescu87eb55e2017-10-11 08:29:43 -0500906 int i, err;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500907
908 for (i = 0; i < DPAA2_ETH_BUFS_PER_CMD; i++) {
909 /* Allocate buffer visible to WRIOP + skb shared info +
910 * alignment padding
911 */
Bogdan Purcareata8a4fd872017-10-29 08:20:42 +0000912 buf = napi_alloc_frag(dpaa2_eth_buf_raw_size(priv));
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500913 if (unlikely(!buf))
914 goto err_alloc;
915
Bogdan Purcareata8a4fd872017-10-29 08:20:42 +0000916 buf = PTR_ALIGN(buf, priv->rx_buf_align);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500917
918 addr = dma_map_single(dev, buf, DPAA2_ETH_RX_BUF_SIZE,
Ioana Ciocoi Radulescu18c2e772018-11-26 16:27:32 +0000919 DMA_BIDIRECTIONAL);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500920 if (unlikely(dma_mapping_error(dev, addr)))
921 goto err_map;
922
923 buf_array[i] = addr;
Ioana Radulescu56361872017-04-28 04:50:32 -0500924
925 /* tracing point */
926 trace_dpaa2_eth_buf_seed(priv->net_dev,
Bogdan Purcareata8a4fd872017-10-29 08:20:42 +0000927 buf, dpaa2_eth_buf_raw_size(priv),
Ioana Radulescu56361872017-04-28 04:50:32 -0500928 addr, DPAA2_ETH_RX_BUF_SIZE,
929 bpid);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500930 }
931
932release_bufs:
Ioana Radulescu87eb55e2017-10-11 08:29:43 -0500933 /* In case the portal is busy, retry until successful */
Ioana Radulescu7ec05962018-01-05 05:04:32 -0600934 while ((err = dpaa2_io_service_release(ch->dpio, bpid,
Ioana Radulescu87eb55e2017-10-11 08:29:43 -0500935 buf_array, i)) == -EBUSY)
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500936 cpu_relax();
Ioana Radulescu87eb55e2017-10-11 08:29:43 -0500937
938 /* If release command failed, clean up and bail out;
939 * not much else we can do about it
940 */
941 if (err) {
942 free_bufs(priv, buf_array, i);
943 return 0;
944 }
945
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500946 return i;
947
948err_map:
949 skb_free_frag(buf);
950err_alloc:
Ioana Radulescu87eb55e2017-10-11 08:29:43 -0500951 /* If we managed to allocate at least some buffers,
952 * release them to hardware
953 */
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500954 if (i)
955 goto release_bufs;
956
957 return 0;
958}
959
960static int seed_pool(struct dpaa2_eth_priv *priv, u16 bpid)
961{
962 int i, j;
963 int new_count;
964
965 /* This is the lazy seeding of Rx buffer pools.
966 * dpaa2_add_bufs() is also used on the Rx hotpath and calls
967 * napi_alloc_frag(). The trouble with that is that it in turn ends up
968 * calling this_cpu_ptr(), which mandates execution in atomic context.
969 * Rather than splitting up the code, do a one-off preempt disable.
970 */
971 preempt_disable();
972 for (j = 0; j < priv->num_channels; j++) {
973 for (i = 0; i < DPAA2_ETH_NUM_BUFS;
974 i += DPAA2_ETH_BUFS_PER_CMD) {
Ioana Radulescu7ec05962018-01-05 05:04:32 -0600975 new_count = add_bufs(priv, priv->channel[j], bpid);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500976 priv->channel[j]->buf_count += new_count;
977
978 if (new_count < DPAA2_ETH_BUFS_PER_CMD) {
979 preempt_enable();
980 return -ENOMEM;
981 }
982 }
983 }
984 preempt_enable();
985
986 return 0;
987}
988
989/**
990 * Drain the specified number of buffers from the DPNI's private buffer pool.
991 * @count must not exceeed DPAA2_ETH_BUFS_PER_CMD
992 */
993static void drain_bufs(struct dpaa2_eth_priv *priv, int count)
994{
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500995 u64 buf_array[DPAA2_ETH_BUFS_PER_CMD];
Ioana Radulescu87eb55e2017-10-11 08:29:43 -0500996 int ret;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -0500997
998 do {
Ioana Radulescu05fa39c2017-06-06 10:00:37 -0500999 ret = dpaa2_io_service_acquire(NULL, priv->bpid,
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001000 buf_array, count);
1001 if (ret < 0) {
1002 netdev_err(priv->net_dev, "dpaa2_io_service_acquire() failed\n");
1003 return;
1004 }
Ioana Radulescu87eb55e2017-10-11 08:29:43 -05001005 free_bufs(priv, buf_array, ret);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001006 } while (ret);
1007}
1008
1009static void drain_pool(struct dpaa2_eth_priv *priv)
1010{
1011 int i;
1012
1013 drain_bufs(priv, DPAA2_ETH_BUFS_PER_CMD);
1014 drain_bufs(priv, 1);
1015
1016 for (i = 0; i < priv->num_channels; i++)
1017 priv->channel[i]->buf_count = 0;
1018}
1019
1020/* Function is called from softirq context only, so we don't need to guard
1021 * the access to percpu count
1022 */
1023static int refill_pool(struct dpaa2_eth_priv *priv,
1024 struct dpaa2_eth_channel *ch,
1025 u16 bpid)
1026{
1027 int new_count;
1028
1029 if (likely(ch->buf_count >= DPAA2_ETH_REFILL_THRESH))
1030 return 0;
1031
1032 do {
Ioana Radulescu7ec05962018-01-05 05:04:32 -06001033 new_count = add_bufs(priv, ch, bpid);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001034 if (unlikely(!new_count)) {
1035 /* Out of memory; abort for now, we'll try later on */
1036 break;
1037 }
1038 ch->buf_count += new_count;
1039 } while (ch->buf_count < DPAA2_ETH_NUM_BUFS);
1040
1041 if (unlikely(ch->buf_count < DPAA2_ETH_NUM_BUFS))
1042 return -ENOMEM;
1043
1044 return 0;
1045}
1046
1047static int pull_channel(struct dpaa2_eth_channel *ch)
1048{
1049 int err;
Ioana Radulescu85047ab2017-04-28 04:50:31 -05001050 int dequeues = -1;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001051
1052 /* Retry while portal is busy */
1053 do {
Ioana Radulescu7ec05962018-01-05 05:04:32 -06001054 err = dpaa2_io_service_pull_channel(ch->dpio, ch->ch_id,
1055 ch->store);
Ioana Radulescu85047ab2017-04-28 04:50:31 -05001056 dequeues++;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001057 cpu_relax();
1058 } while (err == -EBUSY);
1059
Ioana Radulescu85047ab2017-04-28 04:50:31 -05001060 ch->stats.dequeue_portal_busy += dequeues;
1061 if (unlikely(err))
1062 ch->stats.pull_err++;
1063
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001064 return err;
1065}
1066
1067/* NAPI poll routine
1068 *
1069 * Frames are dequeued from the QMan channel associated with this NAPI context.
1070 * Rx, Tx confirmation and (if configured) Rx error frames all count
1071 * towards the NAPI budget.
1072 */
1073static int dpaa2_eth_poll(struct napi_struct *napi, int budget)
1074{
1075 struct dpaa2_eth_channel *ch;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001076 struct dpaa2_eth_priv *priv;
Ioana Ciocoi Radulescu68049a52018-10-08 14:16:31 +00001077 int rx_cleaned = 0, txconf_cleaned = 0;
Ioana Ciocoi Radulescu569dac62018-11-14 11:48:36 +00001078 struct dpaa2_eth_fq *fq, *txc_fq = NULL;
1079 struct netdev_queue *nq;
1080 int store_cleaned, work_done;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001081 int err;
1082
1083 ch = container_of(napi, struct dpaa2_eth_channel, napi);
1084 priv = ch->priv;
1085
Ioana Ciocoi Radulescu68049a52018-10-08 14:16:31 +00001086 do {
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001087 err = pull_channel(ch);
1088 if (unlikely(err))
1089 break;
1090
1091 /* Refill pool if appropriate */
Ioana Radulescu05fa39c2017-06-06 10:00:37 -05001092 refill_pool(priv, ch, priv->bpid);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001093
Ioana Ciocoi Radulescu569dac62018-11-14 11:48:36 +00001094 store_cleaned = consume_frames(ch, &fq);
1095 if (!store_cleaned)
1096 break;
1097 if (fq->type == DPAA2_RX_FQ) {
Ioana Ciocoi Radulescu68049a52018-10-08 14:16:31 +00001098 rx_cleaned += store_cleaned;
Ioana Ciocoi Radulescu569dac62018-11-14 11:48:36 +00001099 } else {
Ioana Ciocoi Radulescu68049a52018-10-08 14:16:31 +00001100 txconf_cleaned += store_cleaned;
Ioana Ciocoi Radulescu569dac62018-11-14 11:48:36 +00001101 /* We have a single Tx conf FQ on this channel */
1102 txc_fq = fq;
1103 }
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001104
Ioana Ciocoi Radulescu68049a52018-10-08 14:16:31 +00001105 /* If we either consumed the whole NAPI budget with Rx frames
1106 * or we reached the Tx confirmations threshold, we're done.
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001107 */
Ioana Ciocoi Radulescu68049a52018-10-08 14:16:31 +00001108 if (rx_cleaned >= budget ||
Ioana Ciocoi Radulescu569dac62018-11-14 11:48:36 +00001109 txconf_cleaned >= DPAA2_ETH_TXCONF_PER_NAPI) {
1110 work_done = budget;
1111 goto out;
1112 }
Ioana Ciocoi Radulescu68049a52018-10-08 14:16:31 +00001113 } while (store_cleaned);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001114
Ioana Ciocoi Radulescu68049a52018-10-08 14:16:31 +00001115 /* We didn't consume the entire budget, so finish napi and
1116 * re-enable data availability notifications
1117 */
1118 napi_complete_done(napi, rx_cleaned);
1119 do {
1120 err = dpaa2_io_service_rearm(ch->dpio, &ch->nctx);
1121 cpu_relax();
1122 } while (err == -EBUSY);
1123 WARN_ONCE(err, "CDAN notifications rearm failed on core %d",
1124 ch->nctx.desired_cpu);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001125
Ioana Ciocoi Radulescu569dac62018-11-14 11:48:36 +00001126 work_done = max(rx_cleaned, 1);
1127
1128out:
1129 if (txc_fq) {
1130 nq = netdev_get_tx_queue(priv->net_dev, txc_fq->flowid);
1131 netdev_tx_completed_queue(nq, txc_fq->dq_frames,
1132 txc_fq->dq_bytes);
1133 txc_fq->dq_frames = 0;
1134 txc_fq->dq_bytes = 0;
1135 }
1136
1137 return work_done;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001138}
1139
1140static void enable_ch_napi(struct dpaa2_eth_priv *priv)
1141{
1142 struct dpaa2_eth_channel *ch;
1143 int i;
1144
1145 for (i = 0; i < priv->num_channels; i++) {
1146 ch = priv->channel[i];
1147 napi_enable(&ch->napi);
1148 }
1149}
1150
1151static void disable_ch_napi(struct dpaa2_eth_priv *priv)
1152{
1153 struct dpaa2_eth_channel *ch;
1154 int i;
1155
1156 for (i = 0; i < priv->num_channels; i++) {
1157 ch = priv->channel[i];
1158 napi_disable(&ch->napi);
1159 }
1160}
1161
1162static int link_state_update(struct dpaa2_eth_priv *priv)
1163{
Ioana Ciornei85b7a342018-10-12 16:27:33 +00001164 struct dpni_link_state state = {0};
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001165 int err;
1166
1167 err = dpni_get_link_state(priv->mc_io, 0, priv->mc_token, &state);
1168 if (unlikely(err)) {
1169 netdev_err(priv->net_dev,
1170 "dpni_get_link_state() failed\n");
1171 return err;
1172 }
1173
1174 /* Chech link state; speed / duplex changes are not treated yet */
1175 if (priv->link_state.up == state.up)
1176 return 0;
1177
1178 priv->link_state = state;
1179 if (state.up) {
1180 netif_carrier_on(priv->net_dev);
1181 netif_tx_start_all_queues(priv->net_dev);
1182 } else {
1183 netif_tx_stop_all_queues(priv->net_dev);
1184 netif_carrier_off(priv->net_dev);
1185 }
1186
Ioana Radulescu77160af2017-06-06 10:00:28 -05001187 netdev_info(priv->net_dev, "Link Event: state %s\n",
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001188 state.up ? "up" : "down");
1189
1190 return 0;
1191}
1192
1193static int dpaa2_eth_open(struct net_device *net_dev)
1194{
1195 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1196 int err;
1197
Ioana Radulescu05fa39c2017-06-06 10:00:37 -05001198 err = seed_pool(priv, priv->bpid);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001199 if (err) {
1200 /* Not much to do; the buffer pool, though not filled up,
1201 * may still contain some buffers which would enable us
1202 * to limp on.
1203 */
1204 netdev_err(net_dev, "Buffer seeding failed for DPBP %d (bpid=%d)\n",
Ioana Radulescu05fa39c2017-06-06 10:00:37 -05001205 priv->dpbp_dev->obj_desc.id, priv->bpid);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001206 }
1207
1208 /* We'll only start the txqs when the link is actually ready; make sure
1209 * we don't race against the link up notification, which may come
1210 * immediately after dpni_enable();
1211 */
1212 netif_tx_stop_all_queues(net_dev);
1213 enable_ch_napi(priv);
1214 /* Also, explicitly set carrier off, otherwise netif_carrier_ok() will
1215 * return true and cause 'ip link show' to report the LOWER_UP flag,
1216 * even though the link notification wasn't even received.
1217 */
1218 netif_carrier_off(net_dev);
1219
1220 err = dpni_enable(priv->mc_io, 0, priv->mc_token);
1221 if (err < 0) {
1222 netdev_err(net_dev, "dpni_enable() failed\n");
1223 goto enable_err;
1224 }
1225
1226 /* If the DPMAC object has already processed the link up interrupt,
1227 * we have to learn the link state ourselves.
1228 */
1229 err = link_state_update(priv);
1230 if (err < 0) {
1231 netdev_err(net_dev, "Can't update link state\n");
1232 goto link_state_err;
1233 }
1234
1235 return 0;
1236
1237link_state_err:
1238enable_err:
1239 disable_ch_napi(priv);
1240 drain_pool(priv);
1241 return err;
1242}
1243
1244/* The DPIO store must be empty when we call this,
1245 * at the end of every NAPI cycle.
1246 */
Ioana Ciorneifdb6ca92018-10-12 16:27:35 +00001247static u32 drain_channel(struct dpaa2_eth_channel *ch)
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001248{
1249 u32 drained = 0, total = 0;
1250
1251 do {
1252 pull_channel(ch);
Ioana Ciocoi Radulescu68049a52018-10-08 14:16:31 +00001253 drained = consume_frames(ch, NULL);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001254 total += drained;
1255 } while (drained);
1256
1257 return total;
1258}
1259
1260static u32 drain_ingress_frames(struct dpaa2_eth_priv *priv)
1261{
1262 struct dpaa2_eth_channel *ch;
1263 int i;
1264 u32 drained = 0;
1265
1266 for (i = 0; i < priv->num_channels; i++) {
1267 ch = priv->channel[i];
Ioana Ciorneifdb6ca92018-10-12 16:27:35 +00001268 drained += drain_channel(ch);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001269 }
1270
1271 return drained;
1272}
1273
1274static int dpaa2_eth_stop(struct net_device *net_dev)
1275{
1276 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
Ioana Ciornei85b7a342018-10-12 16:27:33 +00001277 int dpni_enabled = 0;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001278 int retries = 10;
1279 u32 drained;
1280
1281 netif_tx_stop_all_queues(net_dev);
1282 netif_carrier_off(net_dev);
1283
1284 /* Loop while dpni_disable() attempts to drain the egress FQs
1285 * and confirm them back to us.
1286 */
1287 do {
1288 dpni_disable(priv->mc_io, 0, priv->mc_token);
1289 dpni_is_enabled(priv->mc_io, 0, priv->mc_token, &dpni_enabled);
1290 if (dpni_enabled)
1291 /* Allow the hardware some slack */
1292 msleep(100);
1293 } while (dpni_enabled && --retries);
1294 if (!retries) {
1295 netdev_warn(net_dev, "Retry count exceeded disabling DPNI\n");
1296 /* Must go on and disable NAPI nonetheless, so we don't crash at
1297 * the next "ifconfig up"
1298 */
1299 }
1300
1301 /* Wait for NAPI to complete on every core and disable it.
1302 * In particular, this will also prevent NAPI from being rescheduled if
1303 * a new CDAN is serviced, effectively discarding the CDAN. We therefore
1304 * don't even need to disarm the channels, except perhaps for the case
1305 * of a huge coalescing value.
1306 */
1307 disable_ch_napi(priv);
1308
1309 /* Manually drain the Rx and TxConf queues */
1310 drained = drain_ingress_frames(priv);
1311 if (drained)
1312 netdev_dbg(net_dev, "Drained %d frames.\n", drained);
1313
1314 /* Empty the buffer pool */
1315 drain_pool(priv);
1316
1317 return 0;
1318}
1319
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001320static int dpaa2_eth_set_addr(struct net_device *net_dev, void *addr)
1321{
1322 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1323 struct device *dev = net_dev->dev.parent;
1324 int err;
1325
1326 err = eth_mac_addr(net_dev, addr);
1327 if (err < 0) {
1328 dev_err(dev, "eth_mac_addr() failed (%d)\n", err);
1329 return err;
1330 }
1331
1332 err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token,
1333 net_dev->dev_addr);
1334 if (err) {
1335 dev_err(dev, "dpni_set_primary_mac_addr() failed (%d)\n", err);
1336 return err;
1337 }
1338
1339 return 0;
1340}
1341
1342/** Fill in counters maintained by the GPP driver. These may be different from
1343 * the hardware counters obtained by ethtool.
1344 */
Ioana Radulescuacbff8e2017-06-06 10:00:24 -05001345static void dpaa2_eth_get_stats(struct net_device *net_dev,
1346 struct rtnl_link_stats64 *stats)
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001347{
1348 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1349 struct rtnl_link_stats64 *percpu_stats;
1350 u64 *cpustats;
1351 u64 *netstats = (u64 *)stats;
1352 int i, j;
1353 int num = sizeof(struct rtnl_link_stats64) / sizeof(u64);
1354
1355 for_each_possible_cpu(i) {
1356 percpu_stats = per_cpu_ptr(priv->percpu_stats, i);
1357 cpustats = (u64 *)percpu_stats;
1358 for (j = 0; j < num; j++)
1359 netstats[j] += cpustats[j];
1360 }
1361}
1362
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001363/* Copy mac unicast addresses from @net_dev to @priv.
1364 * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable.
1365 */
1366static void add_uc_hw_addr(const struct net_device *net_dev,
1367 struct dpaa2_eth_priv *priv)
1368{
1369 struct netdev_hw_addr *ha;
1370 int err;
1371
1372 netdev_for_each_uc_addr(ha, net_dev) {
1373 err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token,
1374 ha->addr);
1375 if (err)
1376 netdev_warn(priv->net_dev,
1377 "Could not add ucast MAC %pM to the filtering table (err %d)\n",
1378 ha->addr, err);
1379 }
1380}
1381
1382/* Copy mac multicast addresses from @net_dev to @priv
1383 * Its sole purpose is to make dpaa2_eth_set_rx_mode() more readable.
1384 */
1385static void add_mc_hw_addr(const struct net_device *net_dev,
1386 struct dpaa2_eth_priv *priv)
1387{
1388 struct netdev_hw_addr *ha;
1389 int err;
1390
1391 netdev_for_each_mc_addr(ha, net_dev) {
1392 err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token,
1393 ha->addr);
1394 if (err)
1395 netdev_warn(priv->net_dev,
1396 "Could not add mcast MAC %pM to the filtering table (err %d)\n",
1397 ha->addr, err);
1398 }
1399}
1400
1401static void dpaa2_eth_set_rx_mode(struct net_device *net_dev)
1402{
1403 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1404 int uc_count = netdev_uc_count(net_dev);
1405 int mc_count = netdev_mc_count(net_dev);
1406 u8 max_mac = priv->dpni_attrs.mac_filter_entries;
1407 u32 options = priv->dpni_attrs.options;
1408 u16 mc_token = priv->mc_token;
1409 struct fsl_mc_io *mc_io = priv->mc_io;
1410 int err;
1411
1412 /* Basic sanity checks; these probably indicate a misconfiguration */
1413 if (options & DPNI_OPT_NO_MAC_FILTER && max_mac != 0)
1414 netdev_info(net_dev,
1415 "mac_filter_entries=%d, DPNI_OPT_NO_MAC_FILTER option must be disabled\n",
1416 max_mac);
1417
1418 /* Force promiscuous if the uc or mc counts exceed our capabilities. */
1419 if (uc_count > max_mac) {
1420 netdev_info(net_dev,
1421 "Unicast addr count reached %d, max allowed is %d; forcing promisc\n",
1422 uc_count, max_mac);
1423 goto force_promisc;
1424 }
1425 if (mc_count + uc_count > max_mac) {
1426 netdev_info(net_dev,
1427 "Unicast + multicast addr count reached %d, max allowed is %d; forcing promisc\n",
1428 uc_count + mc_count, max_mac);
1429 goto force_mc_promisc;
1430 }
1431
1432 /* Adjust promisc settings due to flag combinations */
1433 if (net_dev->flags & IFF_PROMISC)
1434 goto force_promisc;
1435 if (net_dev->flags & IFF_ALLMULTI) {
1436 /* First, rebuild unicast filtering table. This should be done
1437 * in promisc mode, in order to avoid frame loss while we
1438 * progressively add entries to the table.
1439 * We don't know whether we had been in promisc already, and
1440 * making an MC call to find out is expensive; so set uc promisc
1441 * nonetheless.
1442 */
1443 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1);
1444 if (err)
1445 netdev_warn(net_dev, "Can't set uc promisc\n");
1446
1447 /* Actual uc table reconstruction. */
1448 err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 0);
1449 if (err)
1450 netdev_warn(net_dev, "Can't clear uc filters\n");
1451 add_uc_hw_addr(net_dev, priv);
1452
1453 /* Finally, clear uc promisc and set mc promisc as requested. */
1454 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0);
1455 if (err)
1456 netdev_warn(net_dev, "Can't clear uc promisc\n");
1457 goto force_mc_promisc;
1458 }
1459
1460 /* Neither unicast, nor multicast promisc will be on... eventually.
1461 * For now, rebuild mac filtering tables while forcing both of them on.
1462 */
1463 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1);
1464 if (err)
1465 netdev_warn(net_dev, "Can't set uc promisc (%d)\n", err);
1466 err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1);
1467 if (err)
1468 netdev_warn(net_dev, "Can't set mc promisc (%d)\n", err);
1469
1470 /* Actual mac filtering tables reconstruction */
1471 err = dpni_clear_mac_filters(mc_io, 0, mc_token, 1, 1);
1472 if (err)
1473 netdev_warn(net_dev, "Can't clear mac filters\n");
1474 add_mc_hw_addr(net_dev, priv);
1475 add_uc_hw_addr(net_dev, priv);
1476
1477 /* Now we can clear both ucast and mcast promisc, without risking
1478 * to drop legitimate frames anymore.
1479 */
1480 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 0);
1481 if (err)
1482 netdev_warn(net_dev, "Can't clear ucast promisc\n");
1483 err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 0);
1484 if (err)
1485 netdev_warn(net_dev, "Can't clear mcast promisc\n");
1486
1487 return;
1488
1489force_promisc:
1490 err = dpni_set_unicast_promisc(mc_io, 0, mc_token, 1);
1491 if (err)
1492 netdev_warn(net_dev, "Can't set ucast promisc\n");
1493force_mc_promisc:
1494 err = dpni_set_multicast_promisc(mc_io, 0, mc_token, 1);
1495 if (err)
1496 netdev_warn(net_dev, "Can't set mcast promisc\n");
1497}
1498
1499static int dpaa2_eth_set_features(struct net_device *net_dev,
1500 netdev_features_t features)
1501{
1502 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
1503 netdev_features_t changed = features ^ net_dev->features;
1504 bool enable;
1505 int err;
1506
1507 if (changed & NETIF_F_RXCSUM) {
1508 enable = !!(features & NETIF_F_RXCSUM);
1509 err = set_rx_csum(priv, enable);
1510 if (err)
1511 return err;
1512 }
1513
1514 if (changed & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)) {
1515 enable = !!(features & (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM));
1516 err = set_tx_csum(priv, enable);
1517 if (err)
1518 return err;
1519 }
1520
1521 return 0;
1522}
1523
Ioana Radulescu859f9982018-04-26 18:23:47 +08001524static int dpaa2_eth_ts_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
1525{
1526 struct dpaa2_eth_priv *priv = netdev_priv(dev);
1527 struct hwtstamp_config config;
1528
1529 if (copy_from_user(&config, rq->ifr_data, sizeof(config)))
1530 return -EFAULT;
1531
1532 switch (config.tx_type) {
1533 case HWTSTAMP_TX_OFF:
1534 priv->tx_tstamp = false;
1535 break;
1536 case HWTSTAMP_TX_ON:
1537 priv->tx_tstamp = true;
1538 break;
1539 default:
1540 return -ERANGE;
1541 }
1542
1543 if (config.rx_filter == HWTSTAMP_FILTER_NONE) {
1544 priv->rx_tstamp = false;
1545 } else {
1546 priv->rx_tstamp = true;
1547 /* TS is set for all frame types, not only those requested */
1548 config.rx_filter = HWTSTAMP_FILTER_ALL;
1549 }
1550
1551 return copy_to_user(rq->ifr_data, &config, sizeof(config)) ?
1552 -EFAULT : 0;
1553}
1554
1555static int dpaa2_eth_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
1556{
1557 if (cmd == SIOCSHWTSTAMP)
1558 return dpaa2_eth_ts_ioctl(dev, rq, cmd);
1559
1560 return -EINVAL;
1561}
1562
Ioana Ciocoi Radulescu7e273a82018-11-26 16:27:29 +00001563static bool xdp_mtu_valid(struct dpaa2_eth_priv *priv, int mtu)
1564{
1565 int mfl, linear_mfl;
1566
1567 mfl = DPAA2_ETH_L2_MAX_FRM(mtu);
1568 linear_mfl = DPAA2_ETH_RX_BUF_SIZE - DPAA2_ETH_RX_HWA_SIZE -
Ioana Ciocoi Radulescu7b1eea12018-11-26 16:27:30 +00001569 dpaa2_eth_rx_head_room(priv) - XDP_PACKET_HEADROOM;
Ioana Ciocoi Radulescu7e273a82018-11-26 16:27:29 +00001570
1571 if (mfl > linear_mfl) {
1572 netdev_warn(priv->net_dev, "Maximum MTU for XDP is %d\n",
1573 linear_mfl - VLAN_ETH_HLEN);
1574 return false;
1575 }
1576
1577 return true;
1578}
1579
1580static int set_rx_mfl(struct dpaa2_eth_priv *priv, int mtu, bool has_xdp)
1581{
1582 int mfl, err;
1583
1584 /* We enforce a maximum Rx frame length based on MTU only if we have
1585 * an XDP program attached (in order to avoid Rx S/G frames).
1586 * Otherwise, we accept all incoming frames as long as they are not
1587 * larger than maximum size supported in hardware
1588 */
1589 if (has_xdp)
1590 mfl = DPAA2_ETH_L2_MAX_FRM(mtu);
1591 else
1592 mfl = DPAA2_ETH_MFL;
1593
1594 err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token, mfl);
1595 if (err) {
1596 netdev_err(priv->net_dev, "dpni_set_max_frame_length failed\n");
1597 return err;
1598 }
1599
1600 return 0;
1601}
1602
1603static int dpaa2_eth_change_mtu(struct net_device *dev, int new_mtu)
1604{
1605 struct dpaa2_eth_priv *priv = netdev_priv(dev);
1606 int err;
1607
1608 if (!priv->xdp_prog)
1609 goto out;
1610
1611 if (!xdp_mtu_valid(priv, new_mtu))
1612 return -EINVAL;
1613
1614 err = set_rx_mfl(priv, new_mtu, true);
1615 if (err)
1616 return err;
1617
1618out:
1619 dev->mtu = new_mtu;
1620 return 0;
1621}
1622
Ioana Ciocoi Radulescu7b1eea12018-11-26 16:27:30 +00001623static int update_rx_buffer_headroom(struct dpaa2_eth_priv *priv, bool has_xdp)
1624{
1625 struct dpni_buffer_layout buf_layout = {0};
1626 int err;
1627
1628 err = dpni_get_buffer_layout(priv->mc_io, 0, priv->mc_token,
1629 DPNI_QUEUE_RX, &buf_layout);
1630 if (err) {
1631 netdev_err(priv->net_dev, "dpni_get_buffer_layout failed\n");
1632 return err;
1633 }
1634
1635 /* Reserve extra headroom for XDP header size changes */
1636 buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv) +
1637 (has_xdp ? XDP_PACKET_HEADROOM : 0);
1638 buf_layout.options = DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM;
1639 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token,
1640 DPNI_QUEUE_RX, &buf_layout);
1641 if (err) {
1642 netdev_err(priv->net_dev, "dpni_set_buffer_layout failed\n");
1643 return err;
1644 }
1645
1646 return 0;
1647}
1648
Ioana Ciocoi Radulescu7e273a82018-11-26 16:27:29 +00001649static int setup_xdp(struct net_device *dev, struct bpf_prog *prog)
1650{
1651 struct dpaa2_eth_priv *priv = netdev_priv(dev);
1652 struct dpaa2_eth_channel *ch;
1653 struct bpf_prog *old;
1654 bool up, need_update;
1655 int i, err;
1656
1657 if (prog && !xdp_mtu_valid(priv, dev->mtu))
1658 return -EINVAL;
1659
1660 if (prog) {
1661 prog = bpf_prog_add(prog, priv->num_channels);
1662 if (IS_ERR(prog))
1663 return PTR_ERR(prog);
1664 }
1665
1666 up = netif_running(dev);
1667 need_update = (!!priv->xdp_prog != !!prog);
1668
1669 if (up)
1670 dpaa2_eth_stop(dev);
1671
Ioana Ciocoi Radulescu7b1eea12018-11-26 16:27:30 +00001672 /* While in xdp mode, enforce a maximum Rx frame size based on MTU.
1673 * Also, when switching between xdp/non-xdp modes we need to reconfigure
1674 * our Rx buffer layout. Buffer pool was drained on dpaa2_eth_stop,
1675 * so we are sure no old format buffers will be used from now on.
1676 */
Ioana Ciocoi Radulescu7e273a82018-11-26 16:27:29 +00001677 if (need_update) {
1678 err = set_rx_mfl(priv, dev->mtu, !!prog);
1679 if (err)
1680 goto out_err;
Ioana Ciocoi Radulescu7b1eea12018-11-26 16:27:30 +00001681 err = update_rx_buffer_headroom(priv, !!prog);
1682 if (err)
1683 goto out_err;
Ioana Ciocoi Radulescu7e273a82018-11-26 16:27:29 +00001684 }
1685
1686 old = xchg(&priv->xdp_prog, prog);
1687 if (old)
1688 bpf_prog_put(old);
1689
1690 for (i = 0; i < priv->num_channels; i++) {
1691 ch = priv->channel[i];
1692 old = xchg(&ch->xdp.prog, prog);
1693 if (old)
1694 bpf_prog_put(old);
1695 }
1696
1697 if (up) {
1698 err = dpaa2_eth_open(dev);
1699 if (err)
1700 return err;
1701 }
1702
1703 return 0;
1704
1705out_err:
1706 if (prog)
1707 bpf_prog_sub(prog, priv->num_channels);
1708 if (up)
1709 dpaa2_eth_open(dev);
1710
1711 return err;
1712}
1713
1714static int dpaa2_eth_xdp(struct net_device *dev, struct netdev_bpf *xdp)
1715{
1716 struct dpaa2_eth_priv *priv = netdev_priv(dev);
1717
1718 switch (xdp->command) {
1719 case XDP_SETUP_PROG:
1720 return setup_xdp(dev, xdp->prog);
1721 case XDP_QUERY_PROG:
1722 xdp->prog_id = priv->xdp_prog ? priv->xdp_prog->aux->id : 0;
1723 break;
1724 default:
1725 return -EINVAL;
1726 }
1727
1728 return 0;
1729}
1730
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001731static const struct net_device_ops dpaa2_eth_ops = {
1732 .ndo_open = dpaa2_eth_open,
1733 .ndo_start_xmit = dpaa2_eth_tx,
1734 .ndo_stop = dpaa2_eth_stop,
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001735 .ndo_set_mac_address = dpaa2_eth_set_addr,
1736 .ndo_get_stats64 = dpaa2_eth_get_stats,
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001737 .ndo_set_rx_mode = dpaa2_eth_set_rx_mode,
1738 .ndo_set_features = dpaa2_eth_set_features,
Ioana Radulescu859f9982018-04-26 18:23:47 +08001739 .ndo_do_ioctl = dpaa2_eth_ioctl,
Ioana Ciocoi Radulescu7e273a82018-11-26 16:27:29 +00001740 .ndo_change_mtu = dpaa2_eth_change_mtu,
1741 .ndo_bpf = dpaa2_eth_xdp,
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001742};
1743
1744static void cdan_cb(struct dpaa2_io_notification_ctx *ctx)
1745{
1746 struct dpaa2_eth_channel *ch;
1747
1748 ch = container_of(ctx, struct dpaa2_eth_channel, nctx);
Ioana Radulescu85047ab2017-04-28 04:50:31 -05001749
1750 /* Update NAPI statistics */
1751 ch->stats.cdan++;
1752
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001753 napi_schedule_irqoff(&ch->napi);
1754}
1755
1756/* Allocate and configure a DPCON object */
1757static struct fsl_mc_device *setup_dpcon(struct dpaa2_eth_priv *priv)
1758{
1759 struct fsl_mc_device *dpcon;
1760 struct device *dev = priv->net_dev->dev.parent;
1761 struct dpcon_attr attrs;
1762 int err;
1763
1764 err = fsl_mc_object_allocate(to_fsl_mc_device(dev),
1765 FSL_MC_POOL_DPCON, &dpcon);
1766 if (err) {
Ioana Ciorneid7f5a9d2018-11-09 15:26:45 +00001767 if (err == -ENXIO)
1768 err = -EPROBE_DEFER;
1769 else
1770 dev_info(dev, "Not enough DPCONs, will go on as-is\n");
1771 return ERR_PTR(err);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001772 }
1773
1774 err = dpcon_open(priv->mc_io, 0, dpcon->obj_desc.id, &dpcon->mc_handle);
1775 if (err) {
1776 dev_err(dev, "dpcon_open() failed\n");
Ioana Radulescuf6dda802017-10-29 08:20:39 +00001777 goto free;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001778 }
1779
1780 err = dpcon_reset(priv->mc_io, 0, dpcon->mc_handle);
1781 if (err) {
1782 dev_err(dev, "dpcon_reset() failed\n");
Ioana Radulescuf6dda802017-10-29 08:20:39 +00001783 goto close;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001784 }
1785
1786 err = dpcon_get_attributes(priv->mc_io, 0, dpcon->mc_handle, &attrs);
1787 if (err) {
1788 dev_err(dev, "dpcon_get_attributes() failed\n");
Ioana Radulescuf6dda802017-10-29 08:20:39 +00001789 goto close;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001790 }
1791
1792 err = dpcon_enable(priv->mc_io, 0, dpcon->mc_handle);
1793 if (err) {
1794 dev_err(dev, "dpcon_enable() failed\n");
Ioana Radulescuf6dda802017-10-29 08:20:39 +00001795 goto close;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001796 }
1797
1798 return dpcon;
1799
Ioana Radulescuf6dda802017-10-29 08:20:39 +00001800close:
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001801 dpcon_close(priv->mc_io, 0, dpcon->mc_handle);
Ioana Radulescuf6dda802017-10-29 08:20:39 +00001802free:
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001803 fsl_mc_object_free(dpcon);
1804
1805 return NULL;
1806}
1807
1808static void free_dpcon(struct dpaa2_eth_priv *priv,
1809 struct fsl_mc_device *dpcon)
1810{
1811 dpcon_disable(priv->mc_io, 0, dpcon->mc_handle);
1812 dpcon_close(priv->mc_io, 0, dpcon->mc_handle);
1813 fsl_mc_object_free(dpcon);
1814}
1815
1816static struct dpaa2_eth_channel *
1817alloc_channel(struct dpaa2_eth_priv *priv)
1818{
1819 struct dpaa2_eth_channel *channel;
1820 struct dpcon_attr attr;
1821 struct device *dev = priv->net_dev->dev.parent;
1822 int err;
1823
1824 channel = kzalloc(sizeof(*channel), GFP_KERNEL);
1825 if (!channel)
1826 return NULL;
1827
1828 channel->dpcon = setup_dpcon(priv);
Ioana Ciorneid7f5a9d2018-11-09 15:26:45 +00001829 if (IS_ERR_OR_NULL(channel->dpcon)) {
1830 err = PTR_ERR(channel->dpcon);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001831 goto err_setup;
Ioana Ciorneid7f5a9d2018-11-09 15:26:45 +00001832 }
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001833
1834 err = dpcon_get_attributes(priv->mc_io, 0, channel->dpcon->mc_handle,
1835 &attr);
1836 if (err) {
1837 dev_err(dev, "dpcon_get_attributes() failed\n");
1838 goto err_get_attr;
1839 }
1840
1841 channel->dpcon_id = attr.id;
1842 channel->ch_id = attr.qbman_ch_id;
1843 channel->priv = priv;
1844
1845 return channel;
1846
1847err_get_attr:
1848 free_dpcon(priv, channel->dpcon);
1849err_setup:
1850 kfree(channel);
Ioana Ciorneid7f5a9d2018-11-09 15:26:45 +00001851 return ERR_PTR(err);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001852}
1853
1854static void free_channel(struct dpaa2_eth_priv *priv,
1855 struct dpaa2_eth_channel *channel)
1856{
1857 free_dpcon(priv, channel->dpcon);
1858 kfree(channel);
1859}
1860
1861/* DPIO setup: allocate and configure QBMan channels, setup core affinity
1862 * and register data availability notifications
1863 */
1864static int setup_dpio(struct dpaa2_eth_priv *priv)
1865{
1866 struct dpaa2_io_notification_ctx *nctx;
1867 struct dpaa2_eth_channel *channel;
1868 struct dpcon_notification_cfg dpcon_notif_cfg;
1869 struct device *dev = priv->net_dev->dev.parent;
1870 int i, err;
1871
1872 /* We want the ability to spread ingress traffic (RX, TX conf) to as
1873 * many cores as possible, so we need one channel for each core
1874 * (unless there's fewer queues than cores, in which case the extra
1875 * channels would be wasted).
1876 * Allocate one channel per core and register it to the core's
1877 * affine DPIO. If not enough channels are available for all cores
1878 * or if some cores don't have an affine DPIO, there will be no
1879 * ingress frame processing on those cores.
1880 */
1881 cpumask_clear(&priv->dpio_cpumask);
1882 for_each_online_cpu(i) {
1883 /* Try to allocate a channel */
1884 channel = alloc_channel(priv);
Ioana Ciorneid7f5a9d2018-11-09 15:26:45 +00001885 if (IS_ERR_OR_NULL(channel)) {
1886 err = PTR_ERR(channel);
1887 if (err != -EPROBE_DEFER)
1888 dev_info(dev,
1889 "No affine channel for cpu %d and above\n", i);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001890 goto err_alloc_ch;
1891 }
1892
1893 priv->channel[priv->num_channels] = channel;
1894
1895 nctx = &channel->nctx;
1896 nctx->is_cdan = 1;
1897 nctx->cb = cdan_cb;
1898 nctx->id = channel->ch_id;
1899 nctx->desired_cpu = i;
1900
1901 /* Register the new context */
Ioana Radulescu7ec05962018-01-05 05:04:32 -06001902 channel->dpio = dpaa2_io_service_select(i);
1903 err = dpaa2_io_service_register(channel->dpio, nctx);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001904 if (err) {
Ioana Radulescu5206d8d2017-06-06 10:00:33 -05001905 dev_dbg(dev, "No affine DPIO for cpu %d\n", i);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001906 /* If no affine DPIO for this core, there's probably
Ioana Radulescu5206d8d2017-06-06 10:00:33 -05001907 * none available for next cores either. Signal we want
1908 * to retry later, in case the DPIO devices weren't
1909 * probed yet.
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001910 */
Ioana Radulescu5206d8d2017-06-06 10:00:33 -05001911 err = -EPROBE_DEFER;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001912 goto err_service_reg;
1913 }
1914
1915 /* Register DPCON notification with MC */
1916 dpcon_notif_cfg.dpio_id = nctx->dpio_id;
1917 dpcon_notif_cfg.priority = 0;
1918 dpcon_notif_cfg.user_ctx = nctx->qman64;
1919 err = dpcon_set_notification(priv->mc_io, 0,
1920 channel->dpcon->mc_handle,
1921 &dpcon_notif_cfg);
1922 if (err) {
1923 dev_err(dev, "dpcon_set_notification failed()\n");
1924 goto err_set_cdan;
1925 }
1926
1927 /* If we managed to allocate a channel and also found an affine
1928 * DPIO for this core, add it to the final mask
1929 */
1930 cpumask_set_cpu(i, &priv->dpio_cpumask);
1931 priv->num_channels++;
1932
1933 /* Stop if we already have enough channels to accommodate all
1934 * RX and TX conf queues
1935 */
Ioana Ciocoi Radulescub0e4f372018-11-14 11:48:35 +00001936 if (priv->num_channels == priv->dpni_attrs.num_queues)
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001937 break;
1938 }
1939
1940 return 0;
1941
1942err_set_cdan:
Ioana Radulescu7ec05962018-01-05 05:04:32 -06001943 dpaa2_io_service_deregister(channel->dpio, nctx);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001944err_service_reg:
1945 free_channel(priv, channel);
1946err_alloc_ch:
Ioana Ciorneid7f5a9d2018-11-09 15:26:45 +00001947 if (err == -EPROBE_DEFER)
1948 return err;
1949
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001950 if (cpumask_empty(&priv->dpio_cpumask)) {
1951 dev_err(dev, "No cpu with an affine DPIO/DPCON\n");
Ioana Ciorneid7f5a9d2018-11-09 15:26:45 +00001952 return -ENODEV;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001953 }
1954
1955 dev_info(dev, "Cores %*pbl available for processing ingress traffic\n",
1956 cpumask_pr_args(&priv->dpio_cpumask));
1957
1958 return 0;
1959}
1960
1961static void free_dpio(struct dpaa2_eth_priv *priv)
1962{
1963 int i;
1964 struct dpaa2_eth_channel *ch;
1965
1966 /* deregister CDAN notifications and free channels */
1967 for (i = 0; i < priv->num_channels; i++) {
1968 ch = priv->channel[i];
Ioana Radulescu7ec05962018-01-05 05:04:32 -06001969 dpaa2_io_service_deregister(ch->dpio, &ch->nctx);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001970 free_channel(priv, ch);
1971 }
1972}
1973
1974static struct dpaa2_eth_channel *get_affine_channel(struct dpaa2_eth_priv *priv,
1975 int cpu)
1976{
1977 struct device *dev = priv->net_dev->dev.parent;
1978 int i;
1979
1980 for (i = 0; i < priv->num_channels; i++)
1981 if (priv->channel[i]->nctx.desired_cpu == cpu)
1982 return priv->channel[i];
1983
1984 /* We should never get here. Issue a warning and return
1985 * the first channel, because it's still better than nothing
1986 */
1987 dev_warn(dev, "No affine channel found for cpu %d\n", cpu);
1988
1989 return priv->channel[0];
1990}
1991
1992static void set_fq_affinity(struct dpaa2_eth_priv *priv)
1993{
1994 struct device *dev = priv->net_dev->dev.parent;
Ioana Radulescu93ddf0b2017-12-21 06:33:21 -06001995 struct cpumask xps_mask;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001996 struct dpaa2_eth_fq *fq;
1997 int rx_cpu, txc_cpu;
Ioana Radulescu93ddf0b2017-12-21 06:33:21 -06001998 int i, err;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05001999
2000 /* For each FQ, pick one channel/CPU to deliver frames to.
2001 * This may well change at runtime, either through irqbalance or
2002 * through direct user intervention.
2003 */
2004 rx_cpu = txc_cpu = cpumask_first(&priv->dpio_cpumask);
2005
2006 for (i = 0; i < priv->num_fqs; i++) {
2007 fq = &priv->fq[i];
2008 switch (fq->type) {
2009 case DPAA2_RX_FQ:
2010 fq->target_cpu = rx_cpu;
2011 rx_cpu = cpumask_next(rx_cpu, &priv->dpio_cpumask);
2012 if (rx_cpu >= nr_cpu_ids)
2013 rx_cpu = cpumask_first(&priv->dpio_cpumask);
2014 break;
2015 case DPAA2_TX_CONF_FQ:
2016 fq->target_cpu = txc_cpu;
Ioana Radulescu93ddf0b2017-12-21 06:33:21 -06002017
2018 /* Tell the stack to affine to txc_cpu the Tx queue
2019 * associated with the confirmation one
2020 */
2021 cpumask_clear(&xps_mask);
2022 cpumask_set_cpu(txc_cpu, &xps_mask);
2023 err = netif_set_xps_queue(priv->net_dev, &xps_mask,
2024 fq->flowid);
2025 if (err)
2026 dev_err(dev, "Error setting XPS queue\n");
2027
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002028 txc_cpu = cpumask_next(txc_cpu, &priv->dpio_cpumask);
2029 if (txc_cpu >= nr_cpu_ids)
2030 txc_cpu = cpumask_first(&priv->dpio_cpumask);
2031 break;
2032 default:
2033 dev_err(dev, "Unknown FQ type: %d\n", fq->type);
2034 }
2035 fq->channel = get_affine_channel(priv, fq->target_cpu);
2036 }
2037}
2038
2039static void setup_fqs(struct dpaa2_eth_priv *priv)
2040{
2041 int i;
2042
2043 /* We have one TxConf FQ per Tx flow.
2044 * The number of Tx and Rx queues is the same.
2045 * Tx queues come first in the fq array.
2046 */
2047 for (i = 0; i < dpaa2_eth_queue_count(priv); i++) {
2048 priv->fq[priv->num_fqs].type = DPAA2_TX_CONF_FQ;
2049 priv->fq[priv->num_fqs].consume = dpaa2_eth_tx_conf;
2050 priv->fq[priv->num_fqs++].flowid = (u16)i;
2051 }
2052
2053 for (i = 0; i < dpaa2_eth_queue_count(priv); i++) {
2054 priv->fq[priv->num_fqs].type = DPAA2_RX_FQ;
2055 priv->fq[priv->num_fqs].consume = dpaa2_eth_rx;
2056 priv->fq[priv->num_fqs++].flowid = (u16)i;
2057 }
2058
2059 /* For each FQ, decide on which core to process incoming frames */
2060 set_fq_affinity(priv);
2061}
2062
2063/* Allocate and configure one buffer pool for each interface */
2064static int setup_dpbp(struct dpaa2_eth_priv *priv)
2065{
2066 int err;
2067 struct fsl_mc_device *dpbp_dev;
2068 struct device *dev = priv->net_dev->dev.parent;
Ioana Radulescu05fa39c2017-06-06 10:00:37 -05002069 struct dpbp_attr dpbp_attrs;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002070
2071 err = fsl_mc_object_allocate(to_fsl_mc_device(dev), FSL_MC_POOL_DPBP,
2072 &dpbp_dev);
2073 if (err) {
Ioana Ciorneid7f5a9d2018-11-09 15:26:45 +00002074 if (err == -ENXIO)
2075 err = -EPROBE_DEFER;
2076 else
2077 dev_err(dev, "DPBP device allocation failed\n");
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002078 return err;
2079 }
2080
2081 priv->dpbp_dev = dpbp_dev;
2082
2083 err = dpbp_open(priv->mc_io, 0, priv->dpbp_dev->obj_desc.id,
2084 &dpbp_dev->mc_handle);
2085 if (err) {
2086 dev_err(dev, "dpbp_open() failed\n");
2087 goto err_open;
2088 }
2089
Ioana Radulescud00defe2017-06-06 10:00:32 -05002090 err = dpbp_reset(priv->mc_io, 0, dpbp_dev->mc_handle);
2091 if (err) {
2092 dev_err(dev, "dpbp_reset() failed\n");
2093 goto err_reset;
2094 }
2095
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002096 err = dpbp_enable(priv->mc_io, 0, dpbp_dev->mc_handle);
2097 if (err) {
2098 dev_err(dev, "dpbp_enable() failed\n");
2099 goto err_enable;
2100 }
2101
2102 err = dpbp_get_attributes(priv->mc_io, 0, dpbp_dev->mc_handle,
Ioana Radulescu05fa39c2017-06-06 10:00:37 -05002103 &dpbp_attrs);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002104 if (err) {
2105 dev_err(dev, "dpbp_get_attributes() failed\n");
2106 goto err_get_attr;
2107 }
Ioana Radulescu05fa39c2017-06-06 10:00:37 -05002108 priv->bpid = dpbp_attrs.bpid;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002109
2110 return 0;
2111
2112err_get_attr:
2113 dpbp_disable(priv->mc_io, 0, dpbp_dev->mc_handle);
2114err_enable:
Ioana Radulescud00defe2017-06-06 10:00:32 -05002115err_reset:
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002116 dpbp_close(priv->mc_io, 0, dpbp_dev->mc_handle);
2117err_open:
2118 fsl_mc_object_free(dpbp_dev);
2119
2120 return err;
2121}
2122
2123static void free_dpbp(struct dpaa2_eth_priv *priv)
2124{
2125 drain_pool(priv);
2126 dpbp_disable(priv->mc_io, 0, priv->dpbp_dev->mc_handle);
2127 dpbp_close(priv->mc_io, 0, priv->dpbp_dev->mc_handle);
2128 fsl_mc_object_free(priv->dpbp_dev);
2129}
2130
Ioana Radulescu308f64e2017-10-29 08:20:40 +00002131static int set_buffer_layout(struct dpaa2_eth_priv *priv)
2132{
2133 struct device *dev = priv->net_dev->dev.parent;
2134 struct dpni_buffer_layout buf_layout = {0};
2135 int err;
2136
Bogdan Purcareata8a4fd872017-10-29 08:20:42 +00002137 /* We need to check for WRIOP version 1.0.0, but depending on the MC
2138 * version, this number is not always provided correctly on rev1.
2139 * We need to check for both alternatives in this situation.
2140 */
2141 if (priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(0, 0, 0) ||
2142 priv->dpni_attrs.wriop_version == DPAA2_WRIOP_VERSION(1, 0, 0))
2143 priv->rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN_REV1;
2144 else
2145 priv->rx_buf_align = DPAA2_ETH_RX_BUF_ALIGN;
2146
Bogdan Purcareata4b2d9fe2017-10-29 08:20:43 +00002147 /* tx buffer */
Ioana Radulescu308f64e2017-10-29 08:20:40 +00002148 buf_layout.private_data_size = DPAA2_ETH_SWA_SIZE;
Ioana Radulescu859f9982018-04-26 18:23:47 +08002149 buf_layout.pass_timestamp = true;
2150 buf_layout.options = DPNI_BUF_LAYOUT_OPT_PRIVATE_DATA_SIZE |
2151 DPNI_BUF_LAYOUT_OPT_TIMESTAMP;
Ioana Radulescu308f64e2017-10-29 08:20:40 +00002152 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token,
2153 DPNI_QUEUE_TX, &buf_layout);
2154 if (err) {
2155 dev_err(dev, "dpni_set_buffer_layout(TX) failed\n");
2156 return err;
2157 }
2158
2159 /* tx-confirm buffer */
Ioana Radulescu859f9982018-04-26 18:23:47 +08002160 buf_layout.options = DPNI_BUF_LAYOUT_OPT_TIMESTAMP;
Ioana Radulescu308f64e2017-10-29 08:20:40 +00002161 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token,
2162 DPNI_QUEUE_TX_CONFIRM, &buf_layout);
2163 if (err) {
2164 dev_err(dev, "dpni_set_buffer_layout(TX_CONF) failed\n");
2165 return err;
2166 }
2167
Bogdan Purcareata4b2d9fe2017-10-29 08:20:43 +00002168 /* Now that we've set our tx buffer layout, retrieve the minimum
2169 * required tx data offset.
2170 */
2171 err = dpni_get_tx_data_offset(priv->mc_io, 0, priv->mc_token,
2172 &priv->tx_data_offset);
2173 if (err) {
2174 dev_err(dev, "dpni_get_tx_data_offset() failed\n");
2175 return err;
2176 }
2177
2178 if ((priv->tx_data_offset % 64) != 0)
2179 dev_warn(dev, "Tx data offset (%d) not a multiple of 64B\n",
2180 priv->tx_data_offset);
2181
2182 /* rx buffer */
Ioana Radulescu2b7c86e2017-12-08 06:47:56 -06002183 buf_layout.pass_frame_status = true;
Bogdan Purcareata4b2d9fe2017-10-29 08:20:43 +00002184 buf_layout.pass_parser_result = true;
2185 buf_layout.data_align = priv->rx_buf_align;
2186 buf_layout.data_head_room = dpaa2_eth_rx_head_room(priv);
2187 buf_layout.private_data_size = 0;
2188 buf_layout.options = DPNI_BUF_LAYOUT_OPT_PARSER_RESULT |
2189 DPNI_BUF_LAYOUT_OPT_FRAME_STATUS |
2190 DPNI_BUF_LAYOUT_OPT_DATA_ALIGN |
Ioana Radulescu859f9982018-04-26 18:23:47 +08002191 DPNI_BUF_LAYOUT_OPT_DATA_HEAD_ROOM |
2192 DPNI_BUF_LAYOUT_OPT_TIMESTAMP;
Bogdan Purcareata4b2d9fe2017-10-29 08:20:43 +00002193 err = dpni_set_buffer_layout(priv->mc_io, 0, priv->mc_token,
2194 DPNI_QUEUE_RX, &buf_layout);
2195 if (err) {
2196 dev_err(dev, "dpni_set_buffer_layout(RX) failed\n");
2197 return err;
2198 }
2199
Ioana Radulescu308f64e2017-10-29 08:20:40 +00002200 return 0;
2201}
2202
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002203/* Configure the DPNI object this interface is associated with */
2204static int setup_dpni(struct fsl_mc_device *ls_dev)
2205{
2206 struct device *dev = &ls_dev->dev;
2207 struct dpaa2_eth_priv *priv;
2208 struct net_device *net_dev;
2209 int err;
2210
2211 net_dev = dev_get_drvdata(dev);
2212 priv = netdev_priv(net_dev);
2213
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002214 /* get a handle for the DPNI object */
Ioana Radulescu50eacbc2017-06-06 10:00:36 -05002215 err = dpni_open(priv->mc_io, 0, ls_dev->obj_desc.id, &priv->mc_token);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002216 if (err) {
2217 dev_err(dev, "dpni_open() failed\n");
Ioana Radulescuf6dda802017-10-29 08:20:39 +00002218 return err;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002219 }
2220
Ioana Radulescu311cffa2018-03-23 08:44:09 -05002221 /* Check if we can work with this DPNI object */
2222 err = dpni_get_api_version(priv->mc_io, 0, &priv->dpni_ver_major,
2223 &priv->dpni_ver_minor);
2224 if (err) {
2225 dev_err(dev, "dpni_get_api_version() failed\n");
2226 goto close;
2227 }
2228 if (dpaa2_eth_cmp_dpni_ver(priv, DPNI_VER_MAJOR, DPNI_VER_MINOR) < 0) {
2229 dev_err(dev, "DPNI version %u.%u not supported, need >= %u.%u\n",
2230 priv->dpni_ver_major, priv->dpni_ver_minor,
2231 DPNI_VER_MAJOR, DPNI_VER_MINOR);
2232 err = -ENOTSUPP;
2233 goto close;
2234 }
2235
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002236 ls_dev->mc_io = priv->mc_io;
2237 ls_dev->mc_handle = priv->mc_token;
2238
2239 err = dpni_reset(priv->mc_io, 0, priv->mc_token);
2240 if (err) {
2241 dev_err(dev, "dpni_reset() failed\n");
Ioana Radulescuf6dda802017-10-29 08:20:39 +00002242 goto close;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002243 }
2244
2245 err = dpni_get_attributes(priv->mc_io, 0, priv->mc_token,
2246 &priv->dpni_attrs);
2247 if (err) {
2248 dev_err(dev, "dpni_get_attributes() failed (err=%d)\n", err);
Ioana Radulescuf6dda802017-10-29 08:20:39 +00002249 goto close;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002250 }
2251
Ioana Radulescu308f64e2017-10-29 08:20:40 +00002252 err = set_buffer_layout(priv);
2253 if (err)
Ioana Radulescuf6dda802017-10-29 08:20:39 +00002254 goto close;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002255
Ioana Radulescuafb90db2018-10-01 13:44:58 +03002256 priv->cls_rules = devm_kzalloc(dev, sizeof(struct dpaa2_eth_cls_rule) *
2257 dpaa2_eth_fs_count(priv), GFP_KERNEL);
2258 if (!priv->cls_rules)
2259 goto close;
2260
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002261 return 0;
2262
Ioana Radulescuf6dda802017-10-29 08:20:39 +00002263close:
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002264 dpni_close(priv->mc_io, 0, priv->mc_token);
Ioana Radulescuf6dda802017-10-29 08:20:39 +00002265
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002266 return err;
2267}
2268
2269static void free_dpni(struct dpaa2_eth_priv *priv)
2270{
2271 int err;
2272
2273 err = dpni_reset(priv->mc_io, 0, priv->mc_token);
2274 if (err)
2275 netdev_warn(priv->net_dev, "dpni_reset() failed (err %d)\n",
2276 err);
2277
2278 dpni_close(priv->mc_io, 0, priv->mc_token);
2279}
2280
2281static int setup_rx_flow(struct dpaa2_eth_priv *priv,
2282 struct dpaa2_eth_fq *fq)
2283{
2284 struct device *dev = priv->net_dev->dev.parent;
2285 struct dpni_queue queue;
2286 struct dpni_queue_id qid;
2287 struct dpni_taildrop td;
2288 int err;
2289
2290 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token,
2291 DPNI_QUEUE_RX, 0, fq->flowid, &queue, &qid);
2292 if (err) {
2293 dev_err(dev, "dpni_get_queue(RX) failed\n");
2294 return err;
2295 }
2296
2297 fq->fqid = qid.fqid;
2298
2299 queue.destination.id = fq->channel->dpcon_id;
2300 queue.destination.type = DPNI_DEST_DPCON;
2301 queue.destination.priority = 1;
Ioana Radulescu75c583a2018-02-26 10:28:06 -06002302 queue.user_context = (u64)(uintptr_t)fq;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002303 err = dpni_set_queue(priv->mc_io, 0, priv->mc_token,
2304 DPNI_QUEUE_RX, 0, fq->flowid,
2305 DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST,
2306 &queue);
2307 if (err) {
2308 dev_err(dev, "dpni_set_queue(RX) failed\n");
2309 return err;
2310 }
2311
2312 td.enable = 1;
2313 td.threshold = DPAA2_ETH_TAILDROP_THRESH;
2314 err = dpni_set_taildrop(priv->mc_io, 0, priv->mc_token, DPNI_CP_QUEUE,
2315 DPNI_QUEUE_RX, 0, fq->flowid, &td);
2316 if (err) {
2317 dev_err(dev, "dpni_set_threshold() failed\n");
2318 return err;
2319 }
2320
2321 return 0;
2322}
2323
2324static int setup_tx_flow(struct dpaa2_eth_priv *priv,
2325 struct dpaa2_eth_fq *fq)
2326{
2327 struct device *dev = priv->net_dev->dev.parent;
2328 struct dpni_queue queue;
2329 struct dpni_queue_id qid;
2330 int err;
2331
2332 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token,
2333 DPNI_QUEUE_TX, 0, fq->flowid, &queue, &qid);
2334 if (err) {
2335 dev_err(dev, "dpni_get_queue(TX) failed\n");
2336 return err;
2337 }
2338
2339 fq->tx_qdbin = qid.qdbin;
2340
2341 err = dpni_get_queue(priv->mc_io, 0, priv->mc_token,
2342 DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid,
2343 &queue, &qid);
2344 if (err) {
2345 dev_err(dev, "dpni_get_queue(TX_CONF) failed\n");
2346 return err;
2347 }
2348
2349 fq->fqid = qid.fqid;
2350
2351 queue.destination.id = fq->channel->dpcon_id;
2352 queue.destination.type = DPNI_DEST_DPCON;
2353 queue.destination.priority = 0;
Ioana Radulescu75c583a2018-02-26 10:28:06 -06002354 queue.user_context = (u64)(uintptr_t)fq;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002355 err = dpni_set_queue(priv->mc_io, 0, priv->mc_token,
2356 DPNI_QUEUE_TX_CONFIRM, 0, fq->flowid,
2357 DPNI_QUEUE_OPT_USER_CTX | DPNI_QUEUE_OPT_DEST,
2358 &queue);
2359 if (err) {
2360 dev_err(dev, "dpni_set_queue(TX_CONF) failed\n");
2361 return err;
2362 }
2363
2364 return 0;
2365}
2366
Ioana Ciocoi Radulescuedad8d22018-09-24 15:36:21 +00002367/* Supported header fields for Rx hash distribution key */
Ioana Radulescuf76c4832018-10-01 13:44:56 +03002368static const struct dpaa2_eth_dist_fields dist_fields[] = {
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002369 {
Ioana Ciocoi Radulescuedad8d22018-09-24 15:36:21 +00002370 /* L2 header */
2371 .rxnfc_field = RXH_L2DA,
2372 .cls_prot = NET_PROT_ETH,
2373 .cls_field = NH_FLD_ETH_DA,
2374 .size = 6,
2375 }, {
Ioana Radulescuafb90db2018-10-01 13:44:58 +03002376 .cls_prot = NET_PROT_ETH,
2377 .cls_field = NH_FLD_ETH_SA,
2378 .size = 6,
2379 }, {
2380 /* This is the last ethertype field parsed:
2381 * depending on frame format, it can be the MAC ethertype
2382 * or the VLAN etype.
2383 */
2384 .cls_prot = NET_PROT_ETH,
2385 .cls_field = NH_FLD_ETH_TYPE,
2386 .size = 2,
2387 }, {
Ioana Ciocoi Radulescuedad8d22018-09-24 15:36:21 +00002388 /* VLAN header */
2389 .rxnfc_field = RXH_VLAN,
2390 .cls_prot = NET_PROT_VLAN,
2391 .cls_field = NH_FLD_VLAN_TCI,
2392 .size = 2,
2393 }, {
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002394 /* IP header */
2395 .rxnfc_field = RXH_IP_SRC,
2396 .cls_prot = NET_PROT_IP,
2397 .cls_field = NH_FLD_IP_SRC,
2398 .size = 4,
2399 }, {
2400 .rxnfc_field = RXH_IP_DST,
2401 .cls_prot = NET_PROT_IP,
2402 .cls_field = NH_FLD_IP_DST,
2403 .size = 4,
2404 }, {
2405 .rxnfc_field = RXH_L3_PROTO,
2406 .cls_prot = NET_PROT_IP,
2407 .cls_field = NH_FLD_IP_PROTO,
2408 .size = 1,
2409 }, {
2410 /* Using UDP ports, this is functionally equivalent to raw
2411 * byte pairs from L4 header.
2412 */
2413 .rxnfc_field = RXH_L4_B_0_1,
2414 .cls_prot = NET_PROT_UDP,
2415 .cls_field = NH_FLD_UDP_PORT_SRC,
2416 .size = 2,
2417 }, {
2418 .rxnfc_field = RXH_L4_B_2_3,
2419 .cls_prot = NET_PROT_UDP,
2420 .cls_field = NH_FLD_UDP_PORT_DST,
2421 .size = 2,
2422 },
2423};
2424
Ioana Radulescudf85aeb2018-10-01 13:44:55 +03002425/* Configure the Rx hash key using the legacy API */
2426static int config_legacy_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key)
2427{
2428 struct device *dev = priv->net_dev->dev.parent;
2429 struct dpni_rx_tc_dist_cfg dist_cfg;
2430 int err;
2431
2432 memset(&dist_cfg, 0, sizeof(dist_cfg));
2433
2434 dist_cfg.key_cfg_iova = key;
2435 dist_cfg.dist_size = dpaa2_eth_queue_count(priv);
2436 dist_cfg.dist_mode = DPNI_DIST_MODE_HASH;
2437
2438 err = dpni_set_rx_tc_dist(priv->mc_io, 0, priv->mc_token, 0, &dist_cfg);
2439 if (err)
2440 dev_err(dev, "dpni_set_rx_tc_dist failed\n");
2441
2442 return err;
2443}
2444
2445/* Configure the Rx hash key using the new API */
2446static int config_hash_key(struct dpaa2_eth_priv *priv, dma_addr_t key)
2447{
2448 struct device *dev = priv->net_dev->dev.parent;
2449 struct dpni_rx_dist_cfg dist_cfg;
2450 int err;
2451
2452 memset(&dist_cfg, 0, sizeof(dist_cfg));
2453
2454 dist_cfg.key_cfg_iova = key;
2455 dist_cfg.dist_size = dpaa2_eth_queue_count(priv);
2456 dist_cfg.enable = 1;
2457
2458 err = dpni_set_rx_hash_dist(priv->mc_io, 0, priv->mc_token, &dist_cfg);
2459 if (err)
2460 dev_err(dev, "dpni_set_rx_hash_dist failed\n");
2461
2462 return err;
2463}
2464
Ioana Radulescu4aaaf9b2018-10-01 13:44:57 +03002465/* Configure the Rx flow classification key */
2466static int config_cls_key(struct dpaa2_eth_priv *priv, dma_addr_t key)
2467{
2468 struct device *dev = priv->net_dev->dev.parent;
2469 struct dpni_rx_dist_cfg dist_cfg;
2470 int err;
2471
2472 memset(&dist_cfg, 0, sizeof(dist_cfg));
2473
2474 dist_cfg.key_cfg_iova = key;
2475 dist_cfg.dist_size = dpaa2_eth_queue_count(priv);
2476 dist_cfg.enable = 1;
2477
2478 err = dpni_set_rx_fs_dist(priv->mc_io, 0, priv->mc_token, &dist_cfg);
2479 if (err)
2480 dev_err(dev, "dpni_set_rx_fs_dist failed\n");
2481
2482 return err;
2483}
2484
Ioana Radulescuafb90db2018-10-01 13:44:58 +03002485/* Size of the Rx flow classification key */
2486int dpaa2_eth_cls_key_size(void)
2487{
2488 int i, size = 0;
2489
2490 for (i = 0; i < ARRAY_SIZE(dist_fields); i++)
2491 size += dist_fields[i].size;
2492
2493 return size;
2494}
2495
2496/* Offset of header field in Rx classification key */
2497int dpaa2_eth_cls_fld_off(int prot, int field)
2498{
2499 int i, off = 0;
2500
2501 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) {
2502 if (dist_fields[i].cls_prot == prot &&
2503 dist_fields[i].cls_field == field)
2504 return off;
2505 off += dist_fields[i].size;
2506 }
2507
2508 WARN_ONCE(1, "Unsupported header field used for Rx flow cls\n");
2509 return 0;
2510}
2511
Ioana Radulescu4aaaf9b2018-10-01 13:44:57 +03002512/* Set Rx distribution (hash or flow classification) key
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002513 * flags is a combination of RXH_ bits
2514 */
Ioana Ciornei3233c152018-10-12 16:27:29 +00002515static int dpaa2_eth_set_dist_key(struct net_device *net_dev,
2516 enum dpaa2_eth_rx_dist type, u64 flags)
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002517{
2518 struct device *dev = net_dev->dev.parent;
2519 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
2520 struct dpkg_profile_cfg cls_cfg;
Ioana Ciocoi Radulescuedad8d22018-09-24 15:36:21 +00002521 u32 rx_hash_fields = 0;
Ioana Radulescudf85aeb2018-10-01 13:44:55 +03002522 dma_addr_t key_iova;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002523 u8 *dma_mem;
2524 int i;
2525 int err = 0;
2526
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002527 memset(&cls_cfg, 0, sizeof(cls_cfg));
2528
Ioana Radulescuf76c4832018-10-01 13:44:56 +03002529 for (i = 0; i < ARRAY_SIZE(dist_fields); i++) {
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002530 struct dpkg_extract *key =
2531 &cls_cfg.extracts[cls_cfg.num_extracts];
2532
Ioana Radulescu4aaaf9b2018-10-01 13:44:57 +03002533 /* For Rx hashing key we set only the selected fields.
2534 * For Rx flow classification key we set all supported fields
2535 */
2536 if (type == DPAA2_ETH_RX_DIST_HASH) {
2537 if (!(flags & dist_fields[i].rxnfc_field))
2538 continue;
2539 rx_hash_fields |= dist_fields[i].rxnfc_field;
2540 }
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002541
2542 if (cls_cfg.num_extracts >= DPKG_MAX_NUM_OF_EXTRACTS) {
2543 dev_err(dev, "error adding key extraction rule, too many rules?\n");
2544 return -E2BIG;
2545 }
2546
2547 key->type = DPKG_EXTRACT_FROM_HDR;
Ioana Radulescuf76c4832018-10-01 13:44:56 +03002548 key->extract.from_hdr.prot = dist_fields[i].cls_prot;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002549 key->extract.from_hdr.type = DPKG_FULL_FIELD;
Ioana Radulescuf76c4832018-10-01 13:44:56 +03002550 key->extract.from_hdr.field = dist_fields[i].cls_field;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002551 cls_cfg.num_extracts++;
2552 }
2553
Ioana Radulescue40ef9e2017-06-06 10:00:30 -05002554 dma_mem = kzalloc(DPAA2_CLASSIFIER_DMA_SIZE, GFP_KERNEL);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002555 if (!dma_mem)
2556 return -ENOMEM;
2557
2558 err = dpni_prepare_key_cfg(&cls_cfg, dma_mem);
2559 if (err) {
Ioana Radulescu77160af2017-06-06 10:00:28 -05002560 dev_err(dev, "dpni_prepare_key_cfg error %d\n", err);
Ioana Radulescudf85aeb2018-10-01 13:44:55 +03002561 goto free_key;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002562 }
2563
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002564 /* Prepare for setting the rx dist */
Ioana Radulescudf85aeb2018-10-01 13:44:55 +03002565 key_iova = dma_map_single(dev, dma_mem, DPAA2_CLASSIFIER_DMA_SIZE,
2566 DMA_TO_DEVICE);
2567 if (dma_mapping_error(dev, key_iova)) {
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002568 dev_err(dev, "DMA mapping failed\n");
2569 err = -ENOMEM;
Ioana Radulescudf85aeb2018-10-01 13:44:55 +03002570 goto free_key;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002571 }
2572
Ioana Radulescu4aaaf9b2018-10-01 13:44:57 +03002573 if (type == DPAA2_ETH_RX_DIST_HASH) {
2574 if (dpaa2_eth_has_legacy_dist(priv))
2575 err = config_legacy_hash_key(priv, key_iova);
2576 else
2577 err = config_hash_key(priv, key_iova);
2578 } else {
2579 err = config_cls_key(priv, key_iova);
2580 }
Ioana Radulescudf85aeb2018-10-01 13:44:55 +03002581
2582 dma_unmap_single(dev, key_iova, DPAA2_CLASSIFIER_DMA_SIZE,
2583 DMA_TO_DEVICE);
Ioana Radulescu4aaaf9b2018-10-01 13:44:57 +03002584 if (!err && type == DPAA2_ETH_RX_DIST_HASH)
Ioana Ciocoi Radulescuedad8d22018-09-24 15:36:21 +00002585 priv->rx_hash_fields = rx_hash_fields;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002586
Ioana Radulescudf85aeb2018-10-01 13:44:55 +03002587free_key:
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002588 kfree(dma_mem);
2589 return err;
2590}
2591
Ioana Radulescu4aaaf9b2018-10-01 13:44:57 +03002592int dpaa2_eth_set_hash(struct net_device *net_dev, u64 flags)
2593{
2594 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
2595
2596 if (!dpaa2_eth_hash_enabled(priv))
2597 return -EOPNOTSUPP;
2598
2599 return dpaa2_eth_set_dist_key(net_dev, DPAA2_ETH_RX_DIST_HASH, flags);
2600}
2601
2602static int dpaa2_eth_set_cls(struct dpaa2_eth_priv *priv)
2603{
2604 struct device *dev = priv->net_dev->dev.parent;
2605
2606 /* Check if we actually support Rx flow classification */
2607 if (dpaa2_eth_has_legacy_dist(priv)) {
2608 dev_dbg(dev, "Rx cls not supported by current MC version\n");
2609 return -EOPNOTSUPP;
2610 }
2611
2612 if (priv->dpni_attrs.options & DPNI_OPT_NO_FS ||
2613 !(priv->dpni_attrs.options & DPNI_OPT_HAS_KEY_MASKING)) {
2614 dev_dbg(dev, "Rx cls disabled in DPNI options\n");
2615 return -EOPNOTSUPP;
2616 }
2617
2618 if (!dpaa2_eth_hash_enabled(priv)) {
2619 dev_dbg(dev, "Rx cls disabled for single queue DPNIs\n");
2620 return -EOPNOTSUPP;
2621 }
2622
2623 priv->rx_cls_enabled = 1;
2624
2625 return dpaa2_eth_set_dist_key(priv->net_dev, DPAA2_ETH_RX_DIST_CLS, 0);
2626}
2627
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002628/* Bind the DPNI to its needed objects and resources: buffer pool, DPIOs,
2629 * frame queues and channels
2630 */
2631static int bind_dpni(struct dpaa2_eth_priv *priv)
2632{
2633 struct net_device *net_dev = priv->net_dev;
2634 struct device *dev = net_dev->dev.parent;
2635 struct dpni_pools_cfg pools_params;
2636 struct dpni_error_cfg err_cfg;
2637 int err = 0;
2638 int i;
2639
2640 pools_params.num_dpbp = 1;
2641 pools_params.pools[0].dpbp_id = priv->dpbp_dev->obj_desc.id;
2642 pools_params.pools[0].backup_pool = 0;
2643 pools_params.pools[0].buffer_size = DPAA2_ETH_RX_BUF_SIZE;
2644 err = dpni_set_pools(priv->mc_io, 0, priv->mc_token, &pools_params);
2645 if (err) {
2646 dev_err(dev, "dpni_set_pools() failed\n");
2647 return err;
2648 }
2649
Ioana Radulescu227686b2018-07-27 09:12:59 -05002650 /* have the interface implicitly distribute traffic based on
2651 * the default hash key
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002652 */
Ioana Radulescu227686b2018-07-27 09:12:59 -05002653 err = dpaa2_eth_set_hash(net_dev, DPAA2_RXH_DEFAULT);
Ioana Ciocoi Radulescuedad8d22018-09-24 15:36:21 +00002654 if (err && err != -EOPNOTSUPP)
Ioana Radulescu0f4c2952017-10-11 08:29:50 -05002655 dev_err(dev, "Failed to configure hashing\n");
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002656
Ioana Radulescu4aaaf9b2018-10-01 13:44:57 +03002657 /* Configure the flow classification key; it includes all
2658 * supported header fields and cannot be modified at runtime
2659 */
2660 err = dpaa2_eth_set_cls(priv);
2661 if (err && err != -EOPNOTSUPP)
2662 dev_err(dev, "Failed to configure Rx classification key\n");
2663
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002664 /* Configure handling of error frames */
Ioana Radulescu39163c02017-06-06 10:00:39 -05002665 err_cfg.errors = DPAA2_FAS_RX_ERR_MASK;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002666 err_cfg.set_frame_annotation = 1;
2667 err_cfg.error_action = DPNI_ERROR_ACTION_DISCARD;
2668 err = dpni_set_errors_behavior(priv->mc_io, 0, priv->mc_token,
2669 &err_cfg);
2670 if (err) {
2671 dev_err(dev, "dpni_set_errors_behavior failed\n");
2672 return err;
2673 }
2674
2675 /* Configure Rx and Tx conf queues to generate CDANs */
2676 for (i = 0; i < priv->num_fqs; i++) {
2677 switch (priv->fq[i].type) {
2678 case DPAA2_RX_FQ:
2679 err = setup_rx_flow(priv, &priv->fq[i]);
2680 break;
2681 case DPAA2_TX_CONF_FQ:
2682 err = setup_tx_flow(priv, &priv->fq[i]);
2683 break;
2684 default:
2685 dev_err(dev, "Invalid FQ type %d\n", priv->fq[i].type);
2686 return -EINVAL;
2687 }
2688 if (err)
2689 return err;
2690 }
2691
2692 err = dpni_get_qdid(priv->mc_io, 0, priv->mc_token,
2693 DPNI_QUEUE_TX, &priv->tx_qdid);
2694 if (err) {
2695 dev_err(dev, "dpni_get_qdid() failed\n");
2696 return err;
2697 }
2698
2699 return 0;
2700}
2701
2702/* Allocate rings for storing incoming frame descriptors */
2703static int alloc_rings(struct dpaa2_eth_priv *priv)
2704{
2705 struct net_device *net_dev = priv->net_dev;
2706 struct device *dev = net_dev->dev.parent;
2707 int i;
2708
2709 for (i = 0; i < priv->num_channels; i++) {
2710 priv->channel[i]->store =
2711 dpaa2_io_store_create(DPAA2_ETH_STORE_SIZE, dev);
2712 if (!priv->channel[i]->store) {
2713 netdev_err(net_dev, "dpaa2_io_store_create() failed\n");
2714 goto err_ring;
2715 }
2716 }
2717
2718 return 0;
2719
2720err_ring:
2721 for (i = 0; i < priv->num_channels; i++) {
2722 if (!priv->channel[i]->store)
2723 break;
2724 dpaa2_io_store_destroy(priv->channel[i]->store);
2725 }
2726
2727 return -ENOMEM;
2728}
2729
2730static void free_rings(struct dpaa2_eth_priv *priv)
2731{
2732 int i;
2733
2734 for (i = 0; i < priv->num_channels; i++)
2735 dpaa2_io_store_destroy(priv->channel[i]->store);
2736}
2737
Ioana Radulescu6ab00862017-06-06 10:00:40 -05002738static int set_mac_addr(struct dpaa2_eth_priv *priv)
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002739{
Ioana Radulescu6ab00862017-06-06 10:00:40 -05002740 struct net_device *net_dev = priv->net_dev;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002741 struct device *dev = net_dev->dev.parent;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002742 u8 mac_addr[ETH_ALEN], dpni_mac_addr[ETH_ALEN];
Ioana Radulescu6ab00862017-06-06 10:00:40 -05002743 int err;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002744
2745 /* Get firmware address, if any */
2746 err = dpni_get_port_mac_addr(priv->mc_io, 0, priv->mc_token, mac_addr);
2747 if (err) {
2748 dev_err(dev, "dpni_get_port_mac_addr() failed\n");
2749 return err;
2750 }
2751
2752 /* Get DPNI attributes address, if any */
2753 err = dpni_get_primary_mac_addr(priv->mc_io, 0, priv->mc_token,
2754 dpni_mac_addr);
2755 if (err) {
Ioana Radulescu6ab00862017-06-06 10:00:40 -05002756 dev_err(dev, "dpni_get_primary_mac_addr() failed\n");
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002757 return err;
2758 }
2759
2760 /* First check if firmware has any address configured by bootloader */
2761 if (!is_zero_ether_addr(mac_addr)) {
2762 /* If the DPMAC addr != DPNI addr, update it */
2763 if (!ether_addr_equal(mac_addr, dpni_mac_addr)) {
2764 err = dpni_set_primary_mac_addr(priv->mc_io, 0,
2765 priv->mc_token,
2766 mac_addr);
2767 if (err) {
2768 dev_err(dev, "dpni_set_primary_mac_addr() failed\n");
2769 return err;
2770 }
2771 }
2772 memcpy(net_dev->dev_addr, mac_addr, net_dev->addr_len);
2773 } else if (is_zero_ether_addr(dpni_mac_addr)) {
Ioana Radulescu6ab00862017-06-06 10:00:40 -05002774 /* No MAC address configured, fill in net_dev->dev_addr
2775 * with a random one
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002776 */
2777 eth_hw_addr_random(net_dev);
Ioana Radulescu6ab00862017-06-06 10:00:40 -05002778 dev_dbg_once(dev, "device(s) have all-zero hwaddr, replaced with random\n");
2779
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002780 err = dpni_set_primary_mac_addr(priv->mc_io, 0, priv->mc_token,
2781 net_dev->dev_addr);
2782 if (err) {
Ioana Radulescu6ab00862017-06-06 10:00:40 -05002783 dev_err(dev, "dpni_set_primary_mac_addr() failed\n");
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002784 return err;
2785 }
Ioana Radulescu6ab00862017-06-06 10:00:40 -05002786
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002787 /* Override NET_ADDR_RANDOM set by eth_hw_addr_random(); for all
2788 * practical purposes, this will be our "permanent" mac address,
2789 * at least until the next reboot. This move will also permit
2790 * register_netdevice() to properly fill up net_dev->perm_addr.
2791 */
2792 net_dev->addr_assign_type = NET_ADDR_PERM;
2793 } else {
2794 /* NET_ADDR_PERM is default, all we have to do is
2795 * fill in the device addr.
2796 */
2797 memcpy(net_dev->dev_addr, dpni_mac_addr, net_dev->addr_len);
2798 }
2799
Ioana Radulescu6ab00862017-06-06 10:00:40 -05002800 return 0;
2801}
2802
2803static int netdev_init(struct net_device *net_dev)
2804{
2805 struct device *dev = net_dev->dev.parent;
2806 struct dpaa2_eth_priv *priv = netdev_priv(net_dev);
Ioana Radulescu7f12c8a32018-08-29 04:42:39 -05002807 u32 options = priv->dpni_attrs.options;
2808 u64 supported = 0, not_supported = 0;
Ioana Radulescu6ab00862017-06-06 10:00:40 -05002809 u8 bcast_addr[ETH_ALEN];
Ioana Radulescubb5b42c2017-06-06 10:00:41 -05002810 u8 num_queues;
Ioana Radulescu6ab00862017-06-06 10:00:40 -05002811 int err;
2812
2813 net_dev->netdev_ops = &dpaa2_eth_ops;
Ioana Radulescu7f12c8a32018-08-29 04:42:39 -05002814 net_dev->ethtool_ops = &dpaa2_ethtool_ops;
Ioana Radulescu6ab00862017-06-06 10:00:40 -05002815
2816 err = set_mac_addr(priv);
2817 if (err)
2818 return err;
2819
2820 /* Explicitly add the broadcast address to the MAC filtering table */
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002821 eth_broadcast_addr(bcast_addr);
2822 err = dpni_add_mac_addr(priv->mc_io, 0, priv->mc_token, bcast_addr);
2823 if (err) {
Ioana Radulescu6ab00862017-06-06 10:00:40 -05002824 dev_err(dev, "dpni_add_mac_addr() failed\n");
2825 return err;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002826 }
2827
Ioana Radulescu3ccc8d42018-07-09 10:01:10 -05002828 /* Set MTU upper limit; lower limit is 68B (default value) */
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002829 net_dev->max_mtu = DPAA2_ETH_MAX_MTU;
Ioana Radulescu00fee002018-07-09 10:01:11 -05002830 err = dpni_set_max_frame_length(priv->mc_io, 0, priv->mc_token,
Ioana Radulescu81f34e92018-07-12 12:12:29 -05002831 DPAA2_ETH_MFL);
Ioana Radulescu00fee002018-07-09 10:01:11 -05002832 if (err) {
2833 dev_err(dev, "dpni_set_max_frame_length() failed\n");
2834 return err;
2835 }
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002836
Ioana Radulescubb5b42c2017-06-06 10:00:41 -05002837 /* Set actual number of queues in the net device */
2838 num_queues = dpaa2_eth_queue_count(priv);
2839 err = netif_set_real_num_tx_queues(net_dev, num_queues);
2840 if (err) {
2841 dev_err(dev, "netif_set_real_num_tx_queues() failed\n");
2842 return err;
2843 }
2844 err = netif_set_real_num_rx_queues(net_dev, num_queues);
2845 if (err) {
2846 dev_err(dev, "netif_set_real_num_rx_queues() failed\n");
2847 return err;
2848 }
2849
Ioana Radulescu7f12c8a32018-08-29 04:42:39 -05002850 /* Capabilities listing */
2851 supported |= IFF_LIVE_ADDR_CHANGE;
2852
2853 if (options & DPNI_OPT_NO_MAC_FILTER)
2854 not_supported |= IFF_UNICAST_FLT;
2855 else
2856 supported |= IFF_UNICAST_FLT;
2857
2858 net_dev->priv_flags |= supported;
2859 net_dev->priv_flags &= ~not_supported;
2860
2861 /* Features */
2862 net_dev->features = NETIF_F_RXCSUM |
2863 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
2864 NETIF_F_SG | NETIF_F_HIGHDMA |
2865 NETIF_F_LLTX;
2866 net_dev->hw_features = net_dev->features;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002867
2868 return 0;
2869}
2870
2871static int poll_link_state(void *arg)
2872{
2873 struct dpaa2_eth_priv *priv = (struct dpaa2_eth_priv *)arg;
2874 int err;
2875
2876 while (!kthread_should_stop()) {
2877 err = link_state_update(priv);
2878 if (unlikely(err))
2879 return err;
2880
2881 msleep(DPAA2_ETH_LINK_STATE_REFRESH);
2882 }
2883
2884 return 0;
2885}
2886
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002887static irqreturn_t dpni_irq0_handler_thread(int irq_num, void *arg)
2888{
Ioana Radulescu112197d2017-10-11 08:29:49 -05002889 u32 status = ~0;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002890 struct device *dev = (struct device *)arg;
2891 struct fsl_mc_device *dpni_dev = to_fsl_mc_device(dev);
2892 struct net_device *net_dev = dev_get_drvdata(dev);
2893 int err;
2894
2895 err = dpni_get_irq_status(dpni_dev->mc_io, 0, dpni_dev->mc_handle,
2896 DPNI_IRQ_INDEX, &status);
2897 if (unlikely(err)) {
Ioana Radulescu77160af2017-06-06 10:00:28 -05002898 netdev_err(net_dev, "Can't get irq status (err %d)\n", err);
Ioana Radulescu112197d2017-10-11 08:29:49 -05002899 return IRQ_HANDLED;
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002900 }
2901
Ioana Radulescu112197d2017-10-11 08:29:49 -05002902 if (status & DPNI_IRQ_EVENT_LINK_CHANGED)
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002903 link_state_update(netdev_priv(net_dev));
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002904
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002905 return IRQ_HANDLED;
2906}
2907
2908static int setup_irqs(struct fsl_mc_device *ls_dev)
2909{
2910 int err = 0;
2911 struct fsl_mc_device_irq *irq;
2912
2913 err = fsl_mc_allocate_irqs(ls_dev);
2914 if (err) {
2915 dev_err(&ls_dev->dev, "MC irqs allocation failed\n");
2916 return err;
2917 }
2918
2919 irq = ls_dev->irqs[0];
2920 err = devm_request_threaded_irq(&ls_dev->dev, irq->msi_desc->irq,
Ioana Radulescufdc9b532018-03-23 08:44:05 -05002921 NULL, dpni_irq0_handler_thread,
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002922 IRQF_NO_SUSPEND | IRQF_ONESHOT,
2923 dev_name(&ls_dev->dev), &ls_dev->dev);
2924 if (err < 0) {
Ioana Radulescu77160af2017-06-06 10:00:28 -05002925 dev_err(&ls_dev->dev, "devm_request_threaded_irq(): %d\n", err);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002926 goto free_mc_irq;
2927 }
2928
2929 err = dpni_set_irq_mask(ls_dev->mc_io, 0, ls_dev->mc_handle,
2930 DPNI_IRQ_INDEX, DPNI_IRQ_EVENT_LINK_CHANGED);
2931 if (err < 0) {
Ioana Radulescu77160af2017-06-06 10:00:28 -05002932 dev_err(&ls_dev->dev, "dpni_set_irq_mask(): %d\n", err);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002933 goto free_irq;
2934 }
2935
2936 err = dpni_set_irq_enable(ls_dev->mc_io, 0, ls_dev->mc_handle,
2937 DPNI_IRQ_INDEX, 1);
2938 if (err < 0) {
Ioana Radulescu77160af2017-06-06 10:00:28 -05002939 dev_err(&ls_dev->dev, "dpni_set_irq_enable(): %d\n", err);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05002940 goto free_irq;
2941 }
2942
2943 return 0;
2944
2945free_irq:
2946 devm_free_irq(&ls_dev->dev, irq->msi_desc->irq, &ls_dev->dev);
2947free_mc_irq:
2948 fsl_mc_free_irqs(ls_dev);
2949
2950 return err;
2951}
2952
2953static void add_ch_napi(struct dpaa2_eth_priv *priv)
2954{
2955 int i;
2956 struct dpaa2_eth_channel *ch;
2957
2958 for (i = 0; i < priv->num_channels; i++) {
2959 ch = priv->channel[i];
2960 /* NAPI weight *MUST* be a multiple of DPAA2_ETH_STORE_SIZE */
2961 netif_napi_add(priv->net_dev, &ch->napi, dpaa2_eth_poll,
2962 NAPI_POLL_WEIGHT);
2963 }
2964}
2965
2966static void del_ch_napi(struct dpaa2_eth_priv *priv)
2967{
2968 int i;
2969 struct dpaa2_eth_channel *ch;
2970
2971 for (i = 0; i < priv->num_channels; i++) {
2972 ch = priv->channel[i];
2973 netif_napi_del(&ch->napi);
2974 }
2975}
2976
2977static int dpaa2_eth_probe(struct fsl_mc_device *dpni_dev)
2978{
2979 struct device *dev;
2980 struct net_device *net_dev = NULL;
2981 struct dpaa2_eth_priv *priv = NULL;
2982 int err = 0;
2983
2984 dev = &dpni_dev->dev;
2985
2986 /* Net device */
2987 net_dev = alloc_etherdev_mq(sizeof(*priv), DPAA2_ETH_MAX_TX_QUEUES);
2988 if (!net_dev) {
2989 dev_err(dev, "alloc_etherdev_mq() failed\n");
2990 return -ENOMEM;
2991 }
2992
2993 SET_NETDEV_DEV(net_dev, dev);
2994 dev_set_drvdata(dev, net_dev);
2995
2996 priv = netdev_priv(net_dev);
2997 priv->net_dev = net_dev;
2998
Ioana Radulescu08eb2392017-05-24 07:13:27 -05002999 priv->iommu_domain = iommu_get_domain_for_dev(dev);
3000
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05003001 /* Obtain a MC portal */
3002 err = fsl_mc_portal_allocate(dpni_dev, FSL_MC_IO_ATOMIC_CONTEXT_PORTAL,
3003 &priv->mc_io);
3004 if (err) {
Ioana Radulescu8c369612018-03-20 07:04:46 -05003005 if (err == -ENXIO)
3006 err = -EPROBE_DEFER;
3007 else
3008 dev_err(dev, "MC portal allocation failed\n");
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05003009 goto err_portal_alloc;
3010 }
3011
3012 /* MC objects initialization and configuration */
3013 err = setup_dpni(dpni_dev);
3014 if (err)
3015 goto err_dpni_setup;
3016
3017 err = setup_dpio(priv);
3018 if (err)
3019 goto err_dpio_setup;
3020
3021 setup_fqs(priv);
3022
3023 err = setup_dpbp(priv);
3024 if (err)
3025 goto err_dpbp_setup;
3026
3027 err = bind_dpni(priv);
3028 if (err)
3029 goto err_bind;
3030
3031 /* Add a NAPI context for each channel */
3032 add_ch_napi(priv);
3033
3034 /* Percpu statistics */
3035 priv->percpu_stats = alloc_percpu(*priv->percpu_stats);
3036 if (!priv->percpu_stats) {
3037 dev_err(dev, "alloc_percpu(percpu_stats) failed\n");
3038 err = -ENOMEM;
3039 goto err_alloc_percpu_stats;
3040 }
Ioana Radulescu85047ab2017-04-28 04:50:31 -05003041 priv->percpu_extras = alloc_percpu(*priv->percpu_extras);
3042 if (!priv->percpu_extras) {
3043 dev_err(dev, "alloc_percpu(percpu_extras) failed\n");
3044 err = -ENOMEM;
3045 goto err_alloc_percpu_extras;
3046 }
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05003047
3048 err = netdev_init(net_dev);
3049 if (err)
3050 goto err_netdev_init;
3051
3052 /* Configure checksum offload based on current interface flags */
3053 err = set_rx_csum(priv, !!(net_dev->features & NETIF_F_RXCSUM));
3054 if (err)
3055 goto err_csum;
3056
3057 err = set_tx_csum(priv, !!(net_dev->features &
3058 (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM)));
3059 if (err)
3060 goto err_csum;
3061
3062 err = alloc_rings(priv);
3063 if (err)
3064 goto err_alloc_rings;
3065
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05003066 err = setup_irqs(dpni_dev);
3067 if (err) {
3068 netdev_warn(net_dev, "Failed to set link interrupt, fall back to polling\n");
3069 priv->poll_thread = kthread_run(poll_link_state, priv,
3070 "%s_poll_link", net_dev->name);
3071 if (IS_ERR(priv->poll_thread)) {
Ioana Radulescu7f12c8a32018-08-29 04:42:39 -05003072 dev_err(dev, "Error starting polling thread\n");
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05003073 goto err_poll_thread;
3074 }
3075 priv->do_link_poll = true;
3076 }
3077
Ioana Radulescu7f12c8a32018-08-29 04:42:39 -05003078 err = register_netdev(net_dev);
3079 if (err < 0) {
3080 dev_err(dev, "register_netdev() failed\n");
3081 goto err_netdev_reg;
3082 }
3083
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05003084 dev_info(dev, "Probed interface %s\n", net_dev->name);
3085 return 0;
3086
Ioana Radulescu7f12c8a32018-08-29 04:42:39 -05003087err_netdev_reg:
3088 if (priv->do_link_poll)
3089 kthread_stop(priv->poll_thread);
3090 else
3091 fsl_mc_free_irqs(dpni_dev);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05003092err_poll_thread:
3093 free_rings(priv);
3094err_alloc_rings:
3095err_csum:
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05003096err_netdev_init:
Ioana Radulescu85047ab2017-04-28 04:50:31 -05003097 free_percpu(priv->percpu_extras);
3098err_alloc_percpu_extras:
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05003099 free_percpu(priv->percpu_stats);
3100err_alloc_percpu_stats:
3101 del_ch_napi(priv);
3102err_bind:
3103 free_dpbp(priv);
3104err_dpbp_setup:
3105 free_dpio(priv);
3106err_dpio_setup:
3107 free_dpni(priv);
3108err_dpni_setup:
3109 fsl_mc_portal_free(priv->mc_io);
3110err_portal_alloc:
3111 dev_set_drvdata(dev, NULL);
3112 free_netdev(net_dev);
3113
3114 return err;
3115}
3116
3117static int dpaa2_eth_remove(struct fsl_mc_device *ls_dev)
3118{
3119 struct device *dev;
3120 struct net_device *net_dev;
3121 struct dpaa2_eth_priv *priv;
3122
3123 dev = &ls_dev->dev;
3124 net_dev = dev_get_drvdata(dev);
3125 priv = netdev_priv(net_dev);
3126
3127 unregister_netdev(net_dev);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05003128
3129 if (priv->do_link_poll)
3130 kthread_stop(priv->poll_thread);
3131 else
3132 fsl_mc_free_irqs(ls_dev);
3133
3134 free_rings(priv);
3135 free_percpu(priv->percpu_stats);
Ioana Radulescu85047ab2017-04-28 04:50:31 -05003136 free_percpu(priv->percpu_extras);
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05003137
3138 del_ch_napi(priv);
3139 free_dpbp(priv);
3140 free_dpio(priv);
3141 free_dpni(priv);
3142
3143 fsl_mc_portal_free(priv->mc_io);
3144
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05003145 free_netdev(net_dev);
3146
Ioana Radulescu4bc07aa2018-03-23 10:23:36 -05003147 dev_dbg(net_dev->dev.parent, "Removed interface %s\n", net_dev->name);
Ioana Radulescu7472dd92018-03-23 08:44:06 -05003148
Ioana Radulescu6e2387e2017-04-28 04:50:29 -05003149 return 0;
3150}
3151
3152static const struct fsl_mc_device_id dpaa2_eth_match_id_table[] = {
3153 {
3154 .vendor = FSL_MC_VENDOR_FREESCALE,
3155 .obj_type = "dpni",
3156 },
3157 { .vendor = 0x0 }
3158};
3159MODULE_DEVICE_TABLE(fslmc, dpaa2_eth_match_id_table);
3160
3161static struct fsl_mc_driver dpaa2_eth_driver = {
3162 .driver = {
3163 .name = KBUILD_MODNAME,
3164 .owner = THIS_MODULE,
3165 },
3166 .probe = dpaa2_eth_probe,
3167 .remove = dpaa2_eth_remove,
3168 .match_id_table = dpaa2_eth_match_id_table
3169};
3170
3171module_fsl_mc_driver(dpaa2_eth_driver);