drivers/net/ethernet/sfc/ef100_tx.c - SHIFTPHONES/kernel/common - Gitiles

 // SPDX-License-Identifier: GPL-2.0-only
 /****************************************************************************
  * Driver for Solarflare network controllers and boards
  * Copyright 2018 Solarflare Communications Inc.
  * Copyright 2019-2020 Xilinx Inc.
  *
  * This program is free software; you can redistribute it and/or modify it
  * under the terms of the GNU General Public License version 2 as published
  * by the Free Software Foundation, incorporated herein by reference.
  */

 #include <net/ip6_checksum.h>

 #include "net_driver.h"
 #include "tx_common.h"
 #include "nic_common.h"
 #include "mcdi_functions.h"
 #include "ef100_regs.h"
 #include "io.h"
 #include "ef100_tx.h"
 #include "ef100_nic.h"

 int ef100_tx_probe(struct efx_tx_queue *tx_queue)
 {
 	/* Allocate an extra descriptor for the QMDA status completion entry */
 	return efx_nic_alloc_buffer(tx_queue->efx, &tx_queue->txd.buf,
 				    (tx_queue->ptr_mask + 2) *
 				    sizeof(efx_oword_t),
 				    GFP_KERNEL);
 }

 void ef100_tx_init(struct efx_tx_queue *tx_queue)
 {
 	/* must be the inverse of lookup in efx_get_tx_channel */
 	tx_queue->core_txq =
 		netdev_get_tx_queue(tx_queue->efx->net_dev,
 				    tx_queue->channel->channel -
 				    tx_queue->efx->tx_channel_offset);

 	/* This value is purely documentational; as EF100 never passes through
 	 * the switch statement in tx.c:__efx_enqueue_skb(), that switch does
 	 * not handle case 3.  EF100's TSOv3 descriptors are generated by
 	 * ef100_make_tso_desc().
 	 * Meanwhile, all efx_mcdi_tx_init() cares about is that it's not 2.
 	 */
 	tx_queue->tso_version = 3;
 	if (efx_mcdi_tx_init(tx_queue))
 		netdev_WARN(tx_queue->efx->net_dev,
 			    "failed to initialise TXQ %d\n", tx_queue->queue);
 }

 static bool ef100_tx_can_tso(struct efx_tx_queue *tx_queue, struct sk_buff *skb)
 {
 	struct efx_nic *efx = tx_queue->efx;
 	struct ef100_nic_data *nic_data;
 	struct efx_tx_buffer *buffer;
 	struct tcphdr *tcphdr;
 	struct iphdr *iphdr;
 	size_t header_len;
 	u32 mss;

 	nic_data = efx->nic_data;

 	if (!skb_is_gso_tcp(skb))
 		return false;
 	if (!(efx->net_dev->features & NETIF_F_TSO))
 		return false;

 	mss = skb_shinfo(skb)->gso_size;
 	if (unlikely(mss < 4)) {
 		WARN_ONCE(1, "MSS of %u is too small for TSO\n", mss);
 		return false;
 	}

 	header_len = efx_tx_tso_header_length(skb);
 	if (header_len > nic_data->tso_max_hdr_len)
 		return false;

 	if (skb_shinfo(skb)->gso_segs > nic_data->tso_max_payload_num_segs) {
 		/* net_dev->gso_max_segs should've caught this */
 		WARN_ON_ONCE(1);
 		return false;
 	}

 	if (skb->data_len / mss > nic_data->tso_max_frames)
 		return false;

 	/* net_dev->gso_max_size should've caught this */
 	if (WARN_ON_ONCE(skb->data_len > nic_data->tso_max_payload_len))
 		return false;

 	/* Reserve an empty buffer for the TSO V3 descriptor.
 	 * Convey the length of the header since we already know it.
 	 */
 	buffer = efx_tx_queue_get_insert_buffer(tx_queue);
 	buffer->flags = EFX_TX_BUF_TSO_V3 | EFX_TX_BUF_CONT;
 	buffer->len = header_len;
 	buffer->unmap_len = 0;
 	buffer->skb = skb;
 	++tx_queue->insert_count;

 	/* Adjust the TCP checksum to exclude the total length, since we set
 	 * ED_INNER_IP_LEN in the descriptor.
 	 */
 	tcphdr = tcp_hdr(skb);
 	if (skb_is_gso_v6(skb)) {
 		tcphdr->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
 						 &ipv6_hdr(skb)->daddr,
 						 0, IPPROTO_TCP, 0);
 	} else {
 		iphdr = ip_hdr(skb);
 		tcphdr->check = ~csum_tcpudp_magic(iphdr->saddr, iphdr->daddr,
 						   0, IPPROTO_TCP, 0);
 	}
 	return true;
 }

 static efx_oword_t *ef100_tx_desc(struct efx_tx_queue *tx_queue, unsigned int index)
 {
 	if (likely(tx_queue->txd.buf.addr))
 		return ((efx_oword_t *)tx_queue->txd.buf.addr) + index;
 	else
 		return NULL;
 }

 static void ef100_notify_tx_desc(struct efx_tx_queue *tx_queue)
 {
 	unsigned int write_ptr;
 	efx_dword_t reg;

 	tx_queue->xmit_pending = false;

 	if (unlikely(tx_queue->notify_count == tx_queue->write_count))
 		return;

 	write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
 	/* The write pointer goes into the high word */
 	EFX_POPULATE_DWORD_1(reg, ERF_GZ_TX_RING_PIDX, write_ptr);
 	efx_writed_page(tx_queue->efx, &reg,
 			ER_GZ_TX_RING_DOORBELL, tx_queue->queue);
 	tx_queue->notify_count = tx_queue->write_count;
 }

 static void ef100_tx_push_buffers(struct efx_tx_queue *tx_queue)
 {
 	ef100_notify_tx_desc(tx_queue);
 	++tx_queue->pushes;
 }

 static void ef100_set_tx_csum_partial(const struct sk_buff *skb,
 				      struct efx_tx_buffer *buffer, efx_oword_t *txd)
 {
 	efx_oword_t csum;
 	int csum_start;

 	if (!skb || skb->ip_summed != CHECKSUM_PARTIAL)
 		return;

 	/* skb->csum_start has the offset from head, but we need the offset
 	 * from data.
 	 */
 	csum_start = skb_checksum_start_offset(skb);
 	EFX_POPULATE_OWORD_3(csum,
 			     ESF_GZ_TX_SEND_CSO_PARTIAL_EN, 1,
 			     ESF_GZ_TX_SEND_CSO_PARTIAL_START_W,
 			     csum_start >> 1,
 			     ESF_GZ_TX_SEND_CSO_PARTIAL_CSUM_W,
 			     skb->csum_offset >> 1);
 	EFX_OR_OWORD(*txd, *txd, csum);
 }

 static void ef100_set_tx_hw_vlan(const struct sk_buff *skb, efx_oword_t *txd)
 {
 	u16 vlan_tci = skb_vlan_tag_get(skb);
 	efx_oword_t vlan;

 	EFX_POPULATE_OWORD_2(vlan,
 			     ESF_GZ_TX_SEND_VLAN_INSERT_EN, 1,
 			     ESF_GZ_TX_SEND_VLAN_INSERT_TCI, vlan_tci);
 	EFX_OR_OWORD(*txd, *txd, vlan);
 }

 static void ef100_make_send_desc(struct efx_nic *efx,
 				 const struct sk_buff *skb,
 				 struct efx_tx_buffer *buffer, efx_oword_t *txd,
 				 unsigned int segment_count)
 {
 	/* TX send descriptor */
 	EFX_POPULATE_OWORD_3(*txd,
 			     ESF_GZ_TX_SEND_NUM_SEGS, segment_count,
 			     ESF_GZ_TX_SEND_LEN, buffer->len,
 			     ESF_GZ_TX_SEND_ADDR, buffer->dma_addr);

 	if (likely(efx->net_dev->features & NETIF_F_HW_CSUM))
 		ef100_set_tx_csum_partial(skb, buffer, txd);
 	if (efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_TX &&
 	    skb && skb_vlan_tag_present(skb))
 		ef100_set_tx_hw_vlan(skb, txd);
 }

 static void ef100_make_tso_desc(struct efx_nic *efx,
 				const struct sk_buff *skb,
 				struct efx_tx_buffer *buffer, efx_oword_t *txd,
 				unsigned int segment_count)
 {
 	u32 mangleid = (efx->net_dev->features & NETIF_F_TSO_MANGLEID) ||
 		skb_shinfo(skb)->gso_type & SKB_GSO_TCP_FIXEDID ?
 		ESE_GZ_TX_DESC_IP4_ID_NO_OP :
 		ESE_GZ_TX_DESC_IP4_ID_INC_MOD16;
 	u16 vlan_enable =  efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_TX ?
 		skb_vlan_tag_present(skb) : 0;
 	unsigned int len, ip_offset, tcp_offset, payload_segs;
 	u16 vlan_tci = skb_vlan_tag_get(skb);
 	u32 mss = skb_shinfo(skb)->gso_size;

 	len = skb->len - buffer->len;
 	/* We use 1 for the TSO descriptor and 1 for the header */
 	payload_segs = segment_count - 2;
 	ip_offset =  skb_network_offset(skb);
 	tcp_offset = skb_transport_offset(skb);

 	EFX_POPULATE_OWORD_13(*txd,
 			      ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_TSO,
 			      ESF_GZ_TX_TSO_MSS, mss,
 			      ESF_GZ_TX_TSO_HDR_NUM_SEGS, 1,
 			      ESF_GZ_TX_TSO_PAYLOAD_NUM_SEGS, payload_segs,
 			      ESF_GZ_TX_TSO_HDR_LEN_W, buffer->len >> 1,
 			      ESF_GZ_TX_TSO_PAYLOAD_LEN, len,
 			      ESF_GZ_TX_TSO_CSO_INNER_L4, 1,
 			      ESF_GZ_TX_TSO_INNER_L3_OFF_W, ip_offset >> 1,
 			      ESF_GZ_TX_TSO_INNER_L4_OFF_W, tcp_offset >> 1,
 			      ESF_GZ_TX_TSO_ED_INNER_IP4_ID, mangleid,
 			      ESF_GZ_TX_TSO_ED_INNER_IP_LEN, 1,
 			      ESF_GZ_TX_TSO_VLAN_INSERT_EN, vlan_enable,
 			      ESF_GZ_TX_TSO_VLAN_INSERT_TCI, vlan_tci
 		);
 }

 static void ef100_tx_make_descriptors(struct efx_tx_queue *tx_queue,
 				      const struct sk_buff *skb,
 				      unsigned int segment_count)
 {
 	unsigned int old_write_count = tx_queue->write_count;
 	unsigned int new_write_count = old_write_count;
 	struct efx_tx_buffer *buffer;
 	unsigned int next_desc_type;
 	unsigned int write_ptr;
 	efx_oword_t *txd;
 	unsigned int nr_descs = tx_queue->insert_count - old_write_count;

 	if (unlikely(nr_descs == 0))
 		return;

 	if (segment_count)
 		next_desc_type = ESE_GZ_TX_DESC_TYPE_TSO;
 	else
 		next_desc_type = ESE_GZ_TX_DESC_TYPE_SEND;

 	/* if it's a raw write (such as XDP) then always SEND single frames */
 	if (!skb)
 		nr_descs = 1;

 	do {
 		write_ptr = new_write_count & tx_queue->ptr_mask;
 		buffer = &tx_queue->buffer[write_ptr];
 		txd = ef100_tx_desc(tx_queue, write_ptr);
 		++new_write_count;

 		/* Create TX descriptor ring entry */
 		tx_queue->packet_write_count = new_write_count;

 		switch (next_desc_type) {
 		case ESE_GZ_TX_DESC_TYPE_SEND:
 			ef100_make_send_desc(tx_queue->efx, skb,
 					     buffer, txd, nr_descs);
 			break;
 		case ESE_GZ_TX_DESC_TYPE_TSO:
 			/* TX TSO descriptor */
 			WARN_ON_ONCE(!(buffer->flags & EFX_TX_BUF_TSO_V3));
 			ef100_make_tso_desc(tx_queue->efx, skb,
 					    buffer, txd, nr_descs);
 			break;
 		default:
 			/* TX segment descriptor */
 			EFX_POPULATE_OWORD_3(*txd,
 					     ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_SEG,
 					     ESF_GZ_TX_SEG_LEN, buffer->len,
 					     ESF_GZ_TX_SEG_ADDR, buffer->dma_addr);
 		}
 		/* if it's a raw write (such as XDP) then always SEND */
 		next_desc_type = skb ? ESE_GZ_TX_DESC_TYPE_SEG :
 				       ESE_GZ_TX_DESC_TYPE_SEND;

 	} while (new_write_count != tx_queue->insert_count);

 	wmb(); /* Ensure descriptors are written before they are fetched */

 	tx_queue->write_count = new_write_count;

 	/* The write_count above must be updated before reading
 	 * channel->holdoff_doorbell to avoid a race with the
 	 * completion path, so ensure these operations are not
 	 * re-ordered.  This also flushes the update of write_count
 	 * back into the cache.
 	 */
 	smp_mb();
 }

 void ef100_tx_write(struct efx_tx_queue *tx_queue)
 {
 	ef100_tx_make_descriptors(tx_queue, NULL, 0);
 	ef100_tx_push_buffers(tx_queue);
 }

 void ef100_ev_tx(struct efx_channel *channel, const efx_qword_t *p_event)
 {
 	unsigned int tx_done =
 		EFX_QWORD_FIELD(*p_event, ESF_GZ_EV_TXCMPL_NUM_DESC);
 	unsigned int qlabel =
 		EFX_QWORD_FIELD(*p_event, ESF_GZ_EV_TXCMPL_Q_LABEL);
 	struct efx_tx_queue *tx_queue =
 		efx_channel_get_tx_queue(channel, qlabel);
 	unsigned int tx_index = (tx_queue->read_count + tx_done - 1) &
 				tx_queue->ptr_mask;

 	efx_xmit_done(tx_queue, tx_index);
 }

 /* Add a socket buffer to a TX queue
  *
  * You must hold netif_tx_lock() to call this function.
  *
  * Returns 0 on success, error code otherwise. In case of an error this
  * function will free the SKB.
  */
 int ef100_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb)
 {
 	unsigned int old_insert_count = tx_queue->insert_count;
 	struct efx_nic *efx = tx_queue->efx;
 	bool xmit_more = netdev_xmit_more();
 	unsigned int fill_level;
 	unsigned int segments;
 	int rc;

 	if (!tx_queue->buffer || !tx_queue->ptr_mask) {
 		netif_stop_queue(efx->net_dev);
 		dev_kfree_skb_any(skb);
 		return -ENODEV;
 	}

 	segments = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 0;
 	if (segments == 1)
 		segments = 0;	/* Don't use TSO/GSO for a single segment. */
 	if (segments && !ef100_tx_can_tso(tx_queue, skb)) {
 		rc = efx_tx_tso_fallback(tx_queue, skb);
 		tx_queue->tso_fallbacks++;
 		if (rc)
 			goto err;
 		else
 			return 0;
 	}

 	/* Map for DMA and create descriptors */
 	rc = efx_tx_map_data(tx_queue, skb, segments);
 	if (rc)
 		goto err;
 	ef100_tx_make_descriptors(tx_queue, skb, segments);

 	fill_level = efx_channel_tx_old_fill_level(tx_queue->channel);
 	if (fill_level > efx->txq_stop_thresh) {
 		struct efx_tx_queue *txq2;

 		netif_tx_stop_queue(tx_queue->core_txq);
 		/* Re-read after a memory barrier in case we've raced with
 		 * the completion path. Otherwise there's a danger we'll never
 		 * restart the queue if all completions have just happened.
 		 */
 		smp_mb();
 		efx_for_each_channel_tx_queue(txq2, tx_queue->channel)
 			txq2->old_read_count = READ_ONCE(txq2->read_count);
 		fill_level = efx_channel_tx_old_fill_level(tx_queue->channel);
 		if (fill_level < efx->txq_stop_thresh)
 			netif_tx_start_queue(tx_queue->core_txq);
 	}

 	tx_queue->xmit_pending = true;

 	/* If xmit_more then we don't need to push the doorbell, unless there
 	 * are 256 descriptors already queued in which case we have to push to
 	 * ensure we never push more than 256 at once.
 	 */
 	if (__netdev_tx_sent_queue(tx_queue->core_txq, skb->len, xmit_more) ||
 	    tx_queue->write_count - tx_queue->notify_count > 255)
 		ef100_tx_push_buffers(tx_queue);

 	if (segments) {
 		tx_queue->tso_bursts++;
 		tx_queue->tso_packets += segments;
 		tx_queue->tx_packets  += segments;
 	} else {
 		tx_queue->tx_packets++;
 	}
 	return 0;

 err:
 	efx_enqueue_unwind(tx_queue, old_insert_count);
 	if (!IS_ERR_OR_NULL(skb))
 		dev_kfree_skb_any(skb);

 	/* If we're not expecting another transmit and we had something to push
 	 * on this queue then we need to push here to get the previous packets
 	 * out.  We only enter this branch from before the xmit_more handling
 	 * above, so xmit_pending still refers to the old state.
 	 */
 	if (tx_queue->xmit_pending && !xmit_more)
 		ef100_tx_push_buffers(tx_queue);
 	return rc;
 }
	// SPDX-License-Identifier: GPL-2.0-only
	/****************************************************************************
	* Driver for Solarflare network controllers and boards
	* Copyright 2018 Solarflare Communications Inc.
	* Copyright 2019-2020 Xilinx Inc.
	*
	* This program is free software; you can redistribute it and/or modify it
	* under the terms of the GNU General Public License version 2 as published
	* by the Free Software Foundation, incorporated herein by reference.
	*/

	#include <net/ip6_checksum.h>

	#include "net_driver.h"
	#include "tx_common.h"
	#include "nic_common.h"
	#include "mcdi_functions.h"
	#include "ef100_regs.h"
	#include "io.h"
	#include "ef100_tx.h"
	#include "ef100_nic.h"

	int ef100_tx_probe(struct efx_tx_queue *tx_queue)
	{
	/* Allocate an extra descriptor for the QMDA status completion entry */
	return efx_nic_alloc_buffer(tx_queue->efx, &tx_queue->txd.buf,
	(tx_queue->ptr_mask + 2) *
	sizeof(efx_oword_t),
	GFP_KERNEL);
	}

	void ef100_tx_init(struct efx_tx_queue *tx_queue)
	{
	/* must be the inverse of lookup in efx_get_tx_channel */
	tx_queue->core_txq =
	netdev_get_tx_queue(tx_queue->efx->net_dev,
	tx_queue->channel->channel -
	tx_queue->efx->tx_channel_offset);

	/* This value is purely documentational; as EF100 never passes through
	* the switch statement in tx.c:__efx_enqueue_skb(), that switch does
	* not handle case 3. EF100's TSOv3 descriptors are generated by
	* ef100_make_tso_desc().
	* Meanwhile, all efx_mcdi_tx_init() cares about is that it's not 2.
	*/
	tx_queue->tso_version = 3;
	if (efx_mcdi_tx_init(tx_queue))
	netdev_WARN(tx_queue->efx->net_dev,
	"failed to initialise TXQ %d\n", tx_queue->queue);
	}

	static bool ef100_tx_can_tso(struct efx_tx_queue tx_queue, struct sk_buff skb)
	{
	struct efx_nic *efx = tx_queue->efx;
	struct ef100_nic_data *nic_data;
	struct efx_tx_buffer *buffer;
	struct tcphdr *tcphdr;
	struct iphdr *iphdr;
	size_t header_len;
	u32 mss;

	nic_data = efx->nic_data;

	if (!skb_is_gso_tcp(skb))
	return false;
	if (!(efx->net_dev->features & NETIF_F_TSO))
	return false;

	mss = skb_shinfo(skb)->gso_size;
	if (unlikely(mss < 4)) {
	WARN_ONCE(1, "MSS of %u is too small for TSO\n", mss);
	return false;
	}

	header_len = efx_tx_tso_header_length(skb);
	if (header_len > nic_data->tso_max_hdr_len)
	return false;

	if (skb_shinfo(skb)->gso_segs > nic_data->tso_max_payload_num_segs) {
	/* net_dev->gso_max_segs should've caught this */
	WARN_ON_ONCE(1);
	return false;
	}

	if (skb->data_len / mss > nic_data->tso_max_frames)
	return false;

	/* net_dev->gso_max_size should've caught this */
	if (WARN_ON_ONCE(skb->data_len > nic_data->tso_max_payload_len))
	return false;

	/* Reserve an empty buffer for the TSO V3 descriptor.
	* Convey the length of the header since we already know it.
	*/
	buffer = efx_tx_queue_get_insert_buffer(tx_queue);
	buffer->flags = EFX_TX_BUF_TSO_V3 \| EFX_TX_BUF_CONT;
	buffer->len = header_len;
	buffer->unmap_len = 0;
	buffer->skb = skb;
	++tx_queue->insert_count;

	/* Adjust the TCP checksum to exclude the total length, since we set
	* ED_INNER_IP_LEN in the descriptor.
	*/
	tcphdr = tcp_hdr(skb);
	if (skb_is_gso_v6(skb)) {
	tcphdr->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
	&ipv6_hdr(skb)->daddr,
	0, IPPROTO_TCP, 0);
	} else {
	iphdr = ip_hdr(skb);
	tcphdr->check = ~csum_tcpudp_magic(iphdr->saddr, iphdr->daddr,
	0, IPPROTO_TCP, 0);
	}
	return true;
	}

	static efx_oword_t ef100_tx_desc(struct efx_tx_queue tx_queue, unsigned int index)
	{
	if (likely(tx_queue->txd.buf.addr))
	return ((efx_oword_t *)tx_queue->txd.buf.addr) + index;
	else
	return NULL;
	}

	static void ef100_notify_tx_desc(struct efx_tx_queue *tx_queue)
	{
	unsigned int write_ptr;
	efx_dword_t reg;

	tx_queue->xmit_pending = false;

	if (unlikely(tx_queue->notify_count == tx_queue->write_count))
	return;

	write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
	/* The write pointer goes into the high word */
	EFX_POPULATE_DWORD_1(reg, ERF_GZ_TX_RING_PIDX, write_ptr);
	efx_writed_page(tx_queue->efx, &reg,
	ER_GZ_TX_RING_DOORBELL, tx_queue->queue);
	tx_queue->notify_count = tx_queue->write_count;
	}

	static void ef100_tx_push_buffers(struct efx_tx_queue *tx_queue)
	{
	ef100_notify_tx_desc(tx_queue);
	++tx_queue->pushes;
	}

	static void ef100_set_tx_csum_partial(const struct sk_buff *skb,
	struct efx_tx_buffer buffer, efx_oword_t txd)
	{
	efx_oword_t csum;
	int csum_start;

	if (!skb \|\| skb->ip_summed != CHECKSUM_PARTIAL)
	return;

	/* skb->csum_start has the offset from head, but we need the offset
	* from data.
	*/
	csum_start = skb_checksum_start_offset(skb);
	EFX_POPULATE_OWORD_3(csum,
	ESF_GZ_TX_SEND_CSO_PARTIAL_EN, 1,
	ESF_GZ_TX_SEND_CSO_PARTIAL_START_W,
	csum_start >> 1,
	ESF_GZ_TX_SEND_CSO_PARTIAL_CSUM_W,
	skb->csum_offset >> 1);
	EFX_OR_OWORD(txd, txd, csum);
	}

	static void ef100_set_tx_hw_vlan(const struct sk_buff skb, efx_oword_t txd)
	{
	u16 vlan_tci = skb_vlan_tag_get(skb);
	efx_oword_t vlan;

	EFX_POPULATE_OWORD_2(vlan,
	ESF_GZ_TX_SEND_VLAN_INSERT_EN, 1,
	ESF_GZ_TX_SEND_VLAN_INSERT_TCI, vlan_tci);
	EFX_OR_OWORD(txd, txd, vlan);
	}

	static void ef100_make_send_desc(struct efx_nic *efx,
	const struct sk_buff *skb,
	struct efx_tx_buffer buffer, efx_oword_t txd,
	unsigned int segment_count)
	{
	/* TX send descriptor */
	EFX_POPULATE_OWORD_3(*txd,
	ESF_GZ_TX_SEND_NUM_SEGS, segment_count,
	ESF_GZ_TX_SEND_LEN, buffer->len,
	ESF_GZ_TX_SEND_ADDR, buffer->dma_addr);

	if (likely(efx->net_dev->features & NETIF_F_HW_CSUM))
	ef100_set_tx_csum_partial(skb, buffer, txd);
	if (efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_TX &&
	skb && skb_vlan_tag_present(skb))
	ef100_set_tx_hw_vlan(skb, txd);
	}

	static void ef100_make_tso_desc(struct efx_nic *efx,
	const struct sk_buff *skb,
	struct efx_tx_buffer buffer, efx_oword_t txd,
	unsigned int segment_count)
	{
	u32 mangleid = (efx->net_dev->features & NETIF_F_TSO_MANGLEID) \|\|
	skb_shinfo(skb)->gso_type & SKB_GSO_TCP_FIXEDID ?
	ESE_GZ_TX_DESC_IP4_ID_NO_OP :
	ESE_GZ_TX_DESC_IP4_ID_INC_MOD16;
	u16 vlan_enable = efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_TX ?
	skb_vlan_tag_present(skb) : 0;
	unsigned int len, ip_offset, tcp_offset, payload_segs;
	u16 vlan_tci = skb_vlan_tag_get(skb);
	u32 mss = skb_shinfo(skb)->gso_size;

	len = skb->len - buffer->len;
	/* We use 1 for the TSO descriptor and 1 for the header */
	payload_segs = segment_count - 2;
	ip_offset = skb_network_offset(skb);
	tcp_offset = skb_transport_offset(skb);

	EFX_POPULATE_OWORD_13(*txd,
	ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_TSO,
	ESF_GZ_TX_TSO_MSS, mss,
	ESF_GZ_TX_TSO_HDR_NUM_SEGS, 1,
	ESF_GZ_TX_TSO_PAYLOAD_NUM_SEGS, payload_segs,
	ESF_GZ_TX_TSO_HDR_LEN_W, buffer->len >> 1,
	ESF_GZ_TX_TSO_PAYLOAD_LEN, len,
	ESF_GZ_TX_TSO_CSO_INNER_L4, 1,
	ESF_GZ_TX_TSO_INNER_L3_OFF_W, ip_offset >> 1,
	ESF_GZ_TX_TSO_INNER_L4_OFF_W, tcp_offset >> 1,
	ESF_GZ_TX_TSO_ED_INNER_IP4_ID, mangleid,
	ESF_GZ_TX_TSO_ED_INNER_IP_LEN, 1,
	ESF_GZ_TX_TSO_VLAN_INSERT_EN, vlan_enable,
	ESF_GZ_TX_TSO_VLAN_INSERT_TCI, vlan_tci
	);
	}

	static void ef100_tx_make_descriptors(struct efx_tx_queue *tx_queue,
	const struct sk_buff *skb,
	unsigned int segment_count)
	{
	unsigned int old_write_count = tx_queue->write_count;
	unsigned int new_write_count = old_write_count;
	struct efx_tx_buffer *buffer;
	unsigned int next_desc_type;
	unsigned int write_ptr;
	efx_oword_t *txd;
	unsigned int nr_descs = tx_queue->insert_count - old_write_count;

	if (unlikely(nr_descs == 0))
	return;

	if (segment_count)
	next_desc_type = ESE_GZ_TX_DESC_TYPE_TSO;
	else
	next_desc_type = ESE_GZ_TX_DESC_TYPE_SEND;

	/* if it's a raw write (such as XDP) then always SEND single frames */
	if (!skb)
	nr_descs = 1;

	do {
	write_ptr = new_write_count & tx_queue->ptr_mask;
	buffer = &tx_queue->buffer[write_ptr];
	txd = ef100_tx_desc(tx_queue, write_ptr);
	++new_write_count;

	/* Create TX descriptor ring entry */
	tx_queue->packet_write_count = new_write_count;

	switch (next_desc_type) {
	case ESE_GZ_TX_DESC_TYPE_SEND:
	ef100_make_send_desc(tx_queue->efx, skb,
	buffer, txd, nr_descs);
	break;
	case ESE_GZ_TX_DESC_TYPE_TSO:
	/* TX TSO descriptor */
	WARN_ON_ONCE(!(buffer->flags & EFX_TX_BUF_TSO_V3));
	ef100_make_tso_desc(tx_queue->efx, skb,
	buffer, txd, nr_descs);
	break;
	default:
	/* TX segment descriptor */
	EFX_POPULATE_OWORD_3(*txd,
	ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_SEG,
	ESF_GZ_TX_SEG_LEN, buffer->len,
	ESF_GZ_TX_SEG_ADDR, buffer->dma_addr);
	}
	/* if it's a raw write (such as XDP) then always SEND */
	next_desc_type = skb ? ESE_GZ_TX_DESC_TYPE_SEG :
	ESE_GZ_TX_DESC_TYPE_SEND;

	} while (new_write_count != tx_queue->insert_count);

	wmb(); /* Ensure descriptors are written before they are fetched */

	tx_queue->write_count = new_write_count;

	/* The write_count above must be updated before reading
	* channel->holdoff_doorbell to avoid a race with the
	* completion path, so ensure these operations are not
	* re-ordered. This also flushes the update of write_count
	* back into the cache.
	*/
	smp_mb();
	}

	void ef100_tx_write(struct efx_tx_queue *tx_queue)
	{
	ef100_tx_make_descriptors(tx_queue, NULL, 0);
	ef100_tx_push_buffers(tx_queue);
	}

	void ef100_ev_tx(struct efx_channel channel, const efx_qword_t p_event)
	{
	unsigned int tx_done =
	EFX_QWORD_FIELD(*p_event, ESF_GZ_EV_TXCMPL_NUM_DESC);
	unsigned int qlabel =
	EFX_QWORD_FIELD(*p_event, ESF_GZ_EV_TXCMPL_Q_LABEL);
	struct efx_tx_queue *tx_queue =
	efx_channel_get_tx_queue(channel, qlabel);
	unsigned int tx_index = (tx_queue->read_count + tx_done - 1) &
	tx_queue->ptr_mask;

	efx_xmit_done(tx_queue, tx_index);
	}

	/* Add a socket buffer to a TX queue
	*
	* You must hold netif_tx_lock() to call this function.
	*
	* Returns 0 on success, error code otherwise. In case of an error this
	* function will free the SKB.
	*/
	int ef100_enqueue_skb(struct efx_tx_queue tx_queue, struct sk_buff skb)
	{
	unsigned int old_insert_count = tx_queue->insert_count;
	struct efx_nic *efx = tx_queue->efx;
	bool xmit_more = netdev_xmit_more();
	unsigned int fill_level;
	unsigned int segments;
	int rc;

	if (!tx_queue->buffer \|\| !tx_queue->ptr_mask) {
	netif_stop_queue(efx->net_dev);
	dev_kfree_skb_any(skb);
	return -ENODEV;
	}

	segments = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 0;
	if (segments == 1)
	segments = 0; /* Don't use TSO/GSO for a single segment. */
	if (segments && !ef100_tx_can_tso(tx_queue, skb)) {
	rc = efx_tx_tso_fallback(tx_queue, skb);
	tx_queue->tso_fallbacks++;
	if (rc)
	goto err;
	else
	return 0;
	}

	/* Map for DMA and create descriptors */
	rc = efx_tx_map_data(tx_queue, skb, segments);
	if (rc)
	goto err;
	ef100_tx_make_descriptors(tx_queue, skb, segments);

	fill_level = efx_channel_tx_old_fill_level(tx_queue->channel);
	if (fill_level > efx->txq_stop_thresh) {
	struct efx_tx_queue *txq2;

	netif_tx_stop_queue(tx_queue->core_txq);
	/* Re-read after a memory barrier in case we've raced with
	* the completion path. Otherwise there's a danger we'll never
	* restart the queue if all completions have just happened.
	*/
	smp_mb();
	efx_for_each_channel_tx_queue(txq2, tx_queue->channel)
	txq2->old_read_count = READ_ONCE(txq2->read_count);
	fill_level = efx_channel_tx_old_fill_level(tx_queue->channel);
	if (fill_level < efx->txq_stop_thresh)
	netif_tx_start_queue(tx_queue->core_txq);
	}

	tx_queue->xmit_pending = true;

	/* If xmit_more then we don't need to push the doorbell, unless there
	* are 256 descriptors already queued in which case we have to push to
	* ensure we never push more than 256 at once.
	*/
	if (__netdev_tx_sent_queue(tx_queue->core_txq, skb->len, xmit_more) \|\|
	tx_queue->write_count - tx_queue->notify_count > 255)
	ef100_tx_push_buffers(tx_queue);

	if (segments) {
	tx_queue->tso_bursts++;
	tx_queue->tso_packets += segments;
	tx_queue->tx_packets += segments;
	} else {
	tx_queue->tx_packets++;
	}
	return 0;

	err:
	efx_enqueue_unwind(tx_queue, old_insert_count);
	if (!IS_ERR_OR_NULL(skb))
	dev_kfree_skb_any(skb);

	/* If we're not expecting another transmit and we had something to push
	* on this queue then we need to push here to get the previous packets
	* out. We only enter this branch from before the xmit_more handling
	* above, so xmit_pending still refers to the old state.
	*/
	if (tx_queue->xmit_pending && !xmit_more)
	ef100_tx_push_buffers(tx_queue);
	return rc;
	}