Skip to content

Commit

Permalink
Merge pull request #40 from GoogleCloudPlatform/dqo-qpl
Browse files Browse the repository at this point in the history 
Dqo qpl
  • Loading branch information
@praveenkaligineedi
praveenkaligineedi authored Aug 10, 2023
2 parents 9e38f70 + f43ded1 commit f662ee6
Show file tree
Hide file tree
Showing 7 changed files with 642 additions and 124 deletions.
112 changes: 107 additions & 5 deletions google/gve/gve.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,26 @@

#define GVE_GQ_TX_MIN_PKT_DESC_BYTES 182

#define DQO_QPL_DEFAULT_TX_PAGES 512
#define DQO_QPL_DEFAULT_RX_PAGES 2048

/* Maximum TSO size supported on DQO */
#define GVE_DQO_TX_MAX 0x3FFFF

#define GVE_TX_BUF_SHIFT_DQO 11

/* 2K buffers for DQO-QPL */
#define GVE_TX_BUF_SIZE_DQO BIT(GVE_TX_BUF_SHIFT_DQO)
#define GVE_TX_BUFS_PER_PAGE_DQO (PAGE_SIZE >> GVE_TX_BUF_SHIFT_DQO)
#define GVE_MAX_TX_BUFS_PER_PKT (DIV_ROUND_UP(GVE_DQO_TX_MAX, GVE_TX_BUF_SIZE_DQO))

/* If number of free/recyclable buffers are less than this threshold; driver
* allocs and uses a non-qpl page on the receive path of DQO QPL to free
* up buffers.
* Value is set big enough to post at least 3 64K LRO packet via 2K buffer to NIC.
*/
#define GVE_DQO_QPL_ONDEMAND_ALLOC_THRESHOLD 96

/* Each slot in the desc ring has a 1:1 mapping to a slot in the data ring */
struct gve_rx_desc_queue {
struct gve_rx_desc *desc_ring; /* the descriptor ring */
Expand Down Expand Up @@ -240,6 +260,15 @@ struct gve_rx_ring {

/* Array of buffers for header-split */
struct gve_header_buf *hdr_bufs;

/* qpl assigned to this queue */
struct gve_queue_page_list *qpl;

/* index into queue page list */
u32 next_qpl_page_idx;

/* track number of used buffers */
u16 used_buf_states_cnt;
} dqo;
};

Expand Down Expand Up @@ -356,8 +385,14 @@ struct gve_tx_pending_packet_dqo {
* All others correspond to `skb`'s frags and should be unmapped with
* `dma_unmap_page`.
*/
DEFINE_DMA_UNMAP_ADDR(dma[MAX_SKB_FRAGS + 1]);
DEFINE_DMA_UNMAP_LEN(len[MAX_SKB_FRAGS + 1]);
union {
struct {
DEFINE_DMA_UNMAP_ADDR(dma[MAX_SKB_FRAGS + 1]);
DEFINE_DMA_UNMAP_LEN(len[MAX_SKB_FRAGS + 1]);
};
s16 tx_qpl_buf_ids[GVE_MAX_TX_BUFS_PER_PKT];
};

u16 num_bufs;

/* Linked list index to next element in the list, or -1 if none */
Expand Down Expand Up @@ -412,6 +447,32 @@ struct gve_tx_ring {
* set.
*/
u32 last_re_idx;

/* free running number of packet buf descriptors posted */
u16 posted_packet_desc_cnt;
/* free running number of packet buf descriptors completed */
u16 completed_packet_desc_cnt;

/* QPL fields */
struct {
/* Linked list of gve_tx_buf_dqo. Index into
* tx_qpl_buf_next, or -1 if empty.
*
* This is a consumer list owned by the TX path. When it
* runs out, the producer list is stolen from the
* completion handling path
* (dqo_compl.free_tx_qpl_buf_head).
*/
s16 free_tx_qpl_buf_head;

/* Free running count of the number of QPL tx buffers
* allocated
*/
u32 alloc_tx_qpl_buf_cnt;

/* Cached value of `dqo_compl.free_tx_qpl_buf_cnt` */
u32 free_tx_qpl_buf_cnt;
};
} dqo_tx;
};

Expand Down Expand Up @@ -459,6 +520,24 @@ struct gve_tx_ring {
* reached a specified timeout.
*/
struct gve_index_list timed_out_completions;

/* QPL fields */
struct {
/* Linked list of gve_tx_buf_dqo. Index into
* tx_qpl_buf_next, or -1 if empty.
*
* This is the producer list, owned by the completion
* handling path. When the consumer list
* (dqo_tx.free_tx_qpl_buf_head) is runs out, this list
* will be stolen.
*/
atomic_t free_tx_qpl_buf_head;

/* Free running count of the number of tx buffers
* freed
*/
atomic_t free_tx_qpl_buf_cnt;
};
} dqo_compl;
} ____cacheline_aligned;
u64 pkt_done; /* free-running - total packets completed */
Expand All @@ -485,6 +564,21 @@ struct gve_tx_ring {
s16 num_pending_packets;

u32 complq_mask; /* complq size is complq_mask + 1 */

/* QPL fields */
struct {
/* qpl assigned to this queue */
struct gve_queue_page_list *qpl;

/* Each QPL page is divided into TX bounce buffers
* of size GVE_TX_BUF_SIZE_DQO. tx_qpl_buf_next is
* an array to manage linked lists of TX buffers.
* An entry j at index i implies that j'th buffer
* is next on the list after i
*/
s16 *tx_qpl_buf_next;
u32 num_tx_qpl_bufs;
};
} dqo;
} ____cacheline_aligned;
struct netdev_queue *netdev_txq;
Expand Down Expand Up @@ -576,6 +670,7 @@ enum gve_queue_format {
GVE_GQI_RDA_FORMAT = 0x1,
GVE_GQI_QPL_FORMAT = 0x2,
GVE_DQO_RDA_FORMAT = 0x3,
GVE_DQO_QPL_FORMAT = 0x4,
};

struct gve_flow_spec {
Expand Down Expand Up @@ -620,7 +715,8 @@ struct gve_priv {
u16 num_event_counters;
u16 tx_desc_cnt; /* num desc per ring */
u16 rx_desc_cnt; /* num desc per ring */
u16 tx_pages_per_qpl; /* tx buffer length */
u16 tx_pages_per_qpl; /* Suggested number of pages per qpl for TX queues by NIC */
u16 rx_pages_per_qpl; /* Suggested number of pages per qpl for RX queues by NIC */
u16 rx_data_slot_cnt; /* rx buffer length */
u64 max_registered_pages;
u64 num_registered_pages; /* num pages registered with NIC */
Expand Down Expand Up @@ -923,11 +1019,17 @@ static inline u32 gve_rx_idx_to_ntfy(struct gve_priv *priv, u32 queue_idx)
return (priv->num_ntfy_blks / 2) + queue_idx;
}

static inline bool gve_is_qpl(struct gve_priv *priv)
{
return priv->queue_format == GVE_GQI_QPL_FORMAT ||
priv->queue_format == GVE_DQO_QPL_FORMAT;
}

/* Returns the number of tx queue page lists
*/
static inline u32 gve_num_tx_qpls(struct gve_priv *priv)
{
if (priv->queue_format != GVE_GQI_QPL_FORMAT)
if (!gve_is_qpl(priv))
return 0;

return priv->tx_cfg.num_queues + priv->num_xdp_queues;
Expand All @@ -947,7 +1049,7 @@ static inline u32 gve_num_xdp_qpls(struct gve_priv *priv)
*/
static inline u32 gve_num_rx_qpls(struct gve_priv *priv)
{
if (priv->queue_format != GVE_GQI_QPL_FORMAT)
if (!gve_is_qpl(priv))
return 0;

return priv->rx_cfg.num_queues;
Expand Down
86 changes: 76 additions & 10 deletions google/gve/gve_adminq.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ void gve_parse_device_option(struct gve_priv *priv,
struct gve_device_option_dqo_rda **dev_op_dqo_rda,
struct gve_device_option_jumbo_frames **dev_op_jumbo_frames,
struct gve_device_option_buffer_sizes **dev_op_buffer_sizes,
struct gve_device_option_flow_steering **dev_op_flow_steering)
struct gve_device_option_flow_steering **dev_op_flow_steering,
struct gve_device_option_dqo_qpl **dev_op_dqo_qpl)
{
u32 req_feat_mask = be32_to_cpu(option->required_features_mask);
u16 option_length = be16_to_cpu(option->option_length);
Expand Down Expand Up @@ -114,6 +115,22 @@ void gve_parse_device_option(struct gve_priv *priv,
}
*dev_op_dqo_rda = (void *)(option + 1);
break;
case GVE_DEV_OPT_ID_DQO_QPL:
if (option_length < sizeof(**dev_op_dqo_qpl) ||
req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_DQO_QPL) {
dev_warn(&priv->pdev->dev, GVE_DEVICE_OPTION_ERROR_FMT,
"DQO QPL", (int)sizeof(**dev_op_dqo_qpl),
GVE_DEV_OPT_REQ_FEAT_MASK_DQO_QPL,
option_length, req_feat_mask);
break;
}

if (option_length > sizeof(**dev_op_dqo_qpl)) {
dev_warn(&priv->pdev->dev,
GVE_DEVICE_OPTION_TOO_BIG_FMT, "DQO QPL");
}
*dev_op_dqo_qpl = (void *)(option + 1);
break;
case GVE_DEV_OPT_ID_JUMBO_FRAMES:
if (option_length < sizeof(**dev_op_jumbo_frames) ||
req_feat_mask != GVE_DEV_OPT_REQ_FEAT_MASK_JUMBO_FRAMES) {
Expand Down Expand Up @@ -188,7 +205,8 @@ gve_process_device_options(struct gve_priv *priv,
struct gve_device_option_dqo_rda **dev_op_dqo_rda,
struct gve_device_option_jumbo_frames **dev_op_jumbo_frames,
struct gve_device_option_buffer_sizes **dev_op_buffer_sizes,
struct gve_device_option_flow_steering **dev_op_flow_steering)
struct gve_device_option_flow_steering **dev_op_flow_steering,
struct gve_device_option_dqo_qpl **dev_op_dqo_qpl)
{
const int num_options = be16_to_cpu(descriptor->num_device_options);
struct gve_device_option *dev_opt;
Expand All @@ -209,7 +227,8 @@ gve_process_device_options(struct gve_priv *priv,
gve_parse_device_option(priv, descriptor, dev_opt,
dev_op_gqi_rda, dev_op_gqi_qpl,
dev_op_dqo_rda, dev_op_jumbo_frames,
dev_op_buffer_sizes, dev_op_flow_steering);
dev_op_buffer_sizes, dev_op_flow_steering,
dev_op_dqo_qpl);
dev_opt = next_opt;
}

Expand Down Expand Up @@ -590,12 +609,24 @@ static int gve_adminq_create_tx_queue(struct gve_priv *priv, u32 queue_index)

cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
} else {
u16 comp_ring_size;
u32 qpl_id = 0;

if (priv->queue_format == GVE_DQO_RDA_FORMAT) {
qpl_id = GVE_RAW_ADDRESSING_QPL_ID;
comp_ring_size =
priv->options_dqo_rda.tx_comp_ring_entries;
} else {
qpl_id = tx->dqo.qpl->id;
comp_ring_size = priv->tx_desc_cnt;
}
cmd.create_tx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
cmd.create_tx_queue.tx_ring_size =
cpu_to_be16(priv->tx_desc_cnt);
cmd.create_tx_queue.tx_comp_ring_addr =
cpu_to_be64(tx->complq_bus_dqo);
cmd.create_tx_queue.tx_comp_ring_size =
cpu_to_be16(priv->options_dqo_rda.tx_comp_ring_entries);
cpu_to_be16(comp_ring_size);
}

return gve_adminq_issue_cmd(priv, &cmd);
Expand Down Expand Up @@ -640,6 +671,18 @@ static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index)
cmd.create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
cmd.create_rx_queue.packet_buffer_size = cpu_to_be16(rx->packet_buffer_size);
} else {
u16 rx_buff_ring_entries;
u32 qpl_id = 0;

if (priv->queue_format == GVE_DQO_RDA_FORMAT) {
qpl_id = GVE_RAW_ADDRESSING_QPL_ID;
rx_buff_ring_entries =
priv->options_dqo_rda.rx_buff_ring_entries;
} else {
qpl_id = rx->dqo.qpl->id;
rx_buff_ring_entries = priv->rx_desc_cnt;
}
cmd.create_rx_queue.queue_page_list_id = cpu_to_be32(qpl_id);
cmd.create_rx_queue.rx_ring_size =
cpu_to_be16(priv->rx_desc_cnt);
cmd.create_rx_queue.rx_desc_ring_addr =
Expand All @@ -649,7 +692,7 @@ static int gve_adminq_create_rx_queue(struct gve_priv *priv, u32 queue_index)
cmd.create_rx_queue.packet_buffer_size =
cpu_to_be16(priv->data_buffer_size_dqo);
cmd.create_rx_queue.rx_buff_ring_size =
cpu_to_be16(priv->options_dqo_rda.rx_buff_ring_entries);
cpu_to_be16(rx_buff_ring_entries);
cmd.create_rx_queue.enable_rsc =
!!(priv->dev->features & NETIF_F_LRO);
if (gve_get_enable_header_split(priv))
Expand Down Expand Up @@ -763,9 +806,13 @@ gve_set_desc_cnt_dqo(struct gve_priv *priv,
const struct gve_device_option_dqo_rda *dev_op_dqo_rda)
{
priv->tx_desc_cnt = be16_to_cpu(descriptor->tx_queue_entries);
priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);

if (priv->queue_format == GVE_DQO_QPL_FORMAT)
return 0;

priv->options_dqo_rda.tx_comp_ring_entries =
be16_to_cpu(dev_op_dqo_rda->tx_comp_ring_entries);
priv->rx_desc_cnt = be16_to_cpu(descriptor->rx_queue_entries);
priv->options_dqo_rda.rx_buff_ring_entries =
be16_to_cpu(dev_op_dqo_rda->rx_buff_ring_entries);

Expand All @@ -777,7 +824,8 @@ static void gve_enable_supported_features(
u32 supported_features_mask,
const struct gve_device_option_jumbo_frames *dev_op_jumbo_frames,
const struct gve_device_option_buffer_sizes *dev_op_buffer_sizes,
const struct gve_device_option_flow_steering *dev_op_flow_steering)
const struct gve_device_option_flow_steering *dev_op_flow_steering,
const struct gve_device_option_dqo_qpl *dev_op_dqo_qpl)
{
int buf_size;

Expand Down Expand Up @@ -832,6 +880,17 @@ static void gve_enable_supported_features(
be16_to_cpu(dev_op_flow_steering->max_num_rules);
}

/* Override pages for qpl for DQO-QPL */
if (dev_op_dqo_qpl) {
priv->tx_pages_per_qpl =
be16_to_cpu(dev_op_dqo_qpl->tx_pages_per_qpl);
priv->rx_pages_per_qpl =
be16_to_cpu(dev_op_dqo_qpl->rx_pages_per_qpl);
if (priv->tx_pages_per_qpl == 0)
priv->tx_pages_per_qpl = DQO_QPL_DEFAULT_TX_PAGES;
if (priv->rx_pages_per_qpl == 0)
priv->rx_pages_per_qpl = DQO_QPL_DEFAULT_RX_PAGES;
}
}

int gve_adminq_describe_device(struct gve_priv *priv)
Expand All @@ -842,6 +901,7 @@ int gve_adminq_describe_device(struct gve_priv *priv)
struct gve_device_option_gqi_rda *dev_op_gqi_rda = NULL;
struct gve_device_option_gqi_qpl *dev_op_gqi_qpl = NULL;
struct gve_device_option_dqo_rda *dev_op_dqo_rda = NULL;
struct gve_device_option_dqo_qpl *dev_op_dqo_qpl = NULL;
struct gve_device_descriptor *descriptor;
u32 supported_features_mask = 0;
union gve_adminq_command cmd;
Expand Down Expand Up @@ -870,20 +930,25 @@ int gve_adminq_describe_device(struct gve_priv *priv)
&dev_op_gqi_qpl, &dev_op_dqo_rda,
&dev_op_jumbo_frames,
&dev_op_buffer_sizes,
&dev_op_flow_steering);
&dev_op_flow_steering,
&dev_op_dqo_qpl);
if (err)
goto free_device_descriptor;

/* If the GQI_RAW_ADDRESSING option is not enabled and the queue format
* is not set to GqiRda, choose the queue format in a priority order:
* DqoRda, GqiRda, GqiQpl. Use GqiQpl as default.
* DqoRda, DqoQpl, GqiRda, GqiQpl. Use GqiQpl as default.
*/
if (dev_op_dqo_rda) {
priv->queue_format = GVE_DQO_RDA_FORMAT;
dev_info(&priv->pdev->dev,
"Driver is running with DQO RDA queue format.\n");
supported_features_mask =
be32_to_cpu(dev_op_dqo_rda->supported_features_mask);
} else if (dev_op_dqo_qpl) {
priv->queue_format = GVE_DQO_QPL_FORMAT;
supported_features_mask =
be32_to_cpu(dev_op_dqo_qpl->supported_features_mask);
} else if (dev_op_gqi_rda) {
priv->queue_format = GVE_GQI_RDA_FORMAT;
dev_info(&priv->pdev->dev,
Expand Down Expand Up @@ -938,7 +1003,8 @@ int gve_adminq_describe_device(struct gve_priv *priv)
gve_enable_supported_features(priv, supported_features_mask,
dev_op_jumbo_frames,
dev_op_buffer_sizes,
dev_op_flow_steering);
dev_op_flow_steering,
dev_op_dqo_qpl);

free_device_descriptor:
dma_free_coherent(&priv->pdev->dev, PAGE_SIZE, descriptor,
Expand Down
Loading

0 comments on commit f662ee6

Please sign in to comment.