Commit ee020824 authored by Alexey Marchuk's avatar Alexey Marchuk Committed by Tomasz Zawadzki
Browse files

accel: Use STAILQ to link tasks 



That is the first patch in the series to
make accel structures more cache friendly
Size of the task structure before and after
all changes:
before:
/* size: 384, cachelines: 6, members: 23 */
after:
/* size: 176, cachelines: 3, members: 20 */

Signed-off-by: default avatarAlexey Marchuk <alexeymar@nvidia.com>
Change-Id: I85a3a528d427b33cc0e45a44498c0c1930fc7629
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/21119


Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarKonrad Sztyber <konrad.sztyber@intel.com>
Reviewed-by: default avatarJim Harris <jim.harris@samsung.com>
Reviewed-by: default avatarBen Walker <ben@nvidia.com>
parent 86e4665b

include/spdk/accel_module.h

+1 −1
Original line number Diff line number Diff line
@@ -126,7 +126,7 @@ struct spdk_accel_task { 
	uint8_t				op_code;

	int16_t				status;

	int				flags;

	TAILQ_ENTRY(spdk_accel_task)	link;

	STAILQ_ENTRY(spdk_accel_task)	link;

	TAILQ_ENTRY(spdk_accel_task)	seq_link;

	struct iovec			aux_iovs[SPDK_ACCEL_AUX_IOV_MAX];

	struct {

lib/accel/accel.c

+9 −10
Original line number Diff line number Diff line
@@ -143,7 +143,7 @@ struct accel_io_channel { 
	void					*task_pool_base;

	struct spdk_accel_sequence		*seq_pool_base;

	struct accel_buffer			*buf_pool_base;

	TAILQ_HEAD(, spdk_accel_task)		task_pool;

	STAILQ_HEAD(, spdk_accel_task)		task_pool;

	SLIST_HEAD(, spdk_accel_sequence)	seq_pool;

	SLIST_HEAD(, accel_buffer)		buf_pool;

	struct spdk_iobuf_channel		iobuf;
@@ -277,7 +277,7 @@ spdk_accel_task_complete(struct spdk_accel_task *accel_task, int status) 
	 * the accel task list is exhausted when there is recursive call to

	 * allocate accel_task in user's call back function (cb_fn)

	 */

	TAILQ_INSERT_HEAD(&accel_ch->task_pool, accel_task, link);

	STAILQ_INSERT_HEAD(&accel_ch->task_pool, accel_task, link);

	accel_task->seq = NULL;



	accel_update_task_stats(accel_ch, accel_task, executed, 1);
@@ -294,15 +294,14 @@ _get_task(struct accel_io_channel *accel_ch, spdk_accel_completion_cb cb_fn, voi 
{

	struct spdk_accel_task *accel_task;



	accel_task = TAILQ_FIRST(&accel_ch->task_pool);

	accel_task = STAILQ_FIRST(&accel_ch->task_pool);

	if (spdk_unlikely(accel_task == NULL)) {

		accel_update_stats(accel_ch, retry.task, 1);

		return NULL;

	}



	TAILQ_REMOVE(&accel_ch->task_pool, accel_task, link);

	accel_task->link.tqe_next = NULL;

	accel_task->link.tqe_prev = NULL;

	STAILQ_REMOVE_HEAD(&accel_ch->task_pool, link);

	accel_task->link.stqe_next = NULL;



	accel_task->cb_fn = cb_fn;

	accel_task->cb_arg = cb_arg;
@@ -1207,7 +1206,7 @@ accel_sequence_complete_task(struct spdk_accel_sequence *seq, struct spdk_accel_ 
	TAILQ_REMOVE(&seq->tasks, task, seq_link);

	cb_fn = task->step_cb_fn;

	cb_arg = task->step_cb_arg;

	TAILQ_INSERT_HEAD(&ch->task_pool, task, link);

	STAILQ_INSERT_HEAD(&ch->task_pool, task, link);

	if (cb_fn != NULL) {

		cb_fn(cb_arg);

	}
@@ -1726,7 +1725,7 @@ accel_sequence_task_cb(void *cb_arg, int status) 
	 * easiest way to prevent this, even though it is a bit hacky.

	 */

	assert(task != NULL);

	TAILQ_REMOVE(&accel_ch->task_pool, task, link);

	STAILQ_REMOVE_HEAD(&accel_ch->task_pool, link);



	switch (seq->state) {

	case ACCEL_SEQUENCE_STATE_AWAIT_TASK:
@@ -2343,14 +2342,14 @@ accel_create_channel(void *io_device, void *ctx_buf) 
		goto err;

	}



	TAILQ_INIT(&accel_ch->task_pool);

	STAILQ_INIT(&accel_ch->task_pool);

	SLIST_INIT(&accel_ch->seq_pool);

	SLIST_INIT(&accel_ch->buf_pool);



	task_mem = accel_ch->task_pool_base;

	for (i = 0; i < g_opts.task_count; i++) {

		accel_task = (struct spdk_accel_task *)task_mem;

		TAILQ_INSERT_TAIL(&accel_ch->task_pool, accel_task, link);

		STAILQ_INSERT_TAIL(&accel_ch->task_pool, accel_task, link);

		task_mem += g_max_accel_module_size;

	}

	for (i = 0; i < g_opts.sequence_count; i++) {

lib/accel/accel_sw.c

+11 −11
Original line number Diff line number Diff line 
/*   SPDX-License-Identifier: BSD-3-Clause

 *   Copyright (C) 2022 Intel Corporation.

 *   Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES

 *   Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES

 *   All rights reserved.

 */
@@ -35,7 +35,7 @@ struct sw_accel_io_channel { 
	struct inflate_state		state;

#endif

	struct spdk_poller		*completion_poller;

	TAILQ_HEAD(, spdk_accel_task)	tasks_to_complete;

	STAILQ_HEAD(, spdk_accel_task)	tasks_to_complete;

};



typedef void (*sw_accel_crypto_op)(uint8_t *k2, uint8_t *k1, uint8_t *tweak, uint64_t lba_size,
@@ -59,7 +59,7 @@ inline static void 
_add_to_comp_list(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task, int status)

{

	accel_task->status = status;

	TAILQ_INSERT_TAIL(&sw_ch->tasks_to_complete, accel_task, link);

	STAILQ_INSERT_TAIL(&sw_ch->tasks_to_complete, accel_task, link);

}



static bool
@@ -496,7 +496,7 @@ sw_accel_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_ 
			break;

		}



		tmp = TAILQ_NEXT(accel_task, link);

		tmp = STAILQ_NEXT(accel_task, link);



		_add_to_comp_list(sw_ch, accel_task, rc);
@@ -510,18 +510,18 @@ static int 
accel_comp_poll(void *arg)

{

	struct sw_accel_io_channel	*sw_ch = arg;

	TAILQ_HEAD(, spdk_accel_task)	tasks_to_complete;

	STAILQ_HEAD(, spdk_accel_task)	tasks_to_complete;

	struct spdk_accel_task		*accel_task;



	if (TAILQ_EMPTY(&sw_ch->tasks_to_complete)) {

	if (STAILQ_EMPTY(&sw_ch->tasks_to_complete)) {

		return SPDK_POLLER_IDLE;

	}



	TAILQ_INIT(&tasks_to_complete);

	TAILQ_SWAP(&tasks_to_complete, &sw_ch->tasks_to_complete, spdk_accel_task, link);

	STAILQ_INIT(&tasks_to_complete);

	STAILQ_SWAP(&tasks_to_complete, &sw_ch->tasks_to_complete, spdk_accel_task);



	while ((accel_task = TAILQ_FIRST(&tasks_to_complete))) {

		TAILQ_REMOVE(&tasks_to_complete, accel_task, link);

	while ((accel_task = STAILQ_FIRST(&tasks_to_complete))) {

		STAILQ_REMOVE_HEAD(&tasks_to_complete, link);

		spdk_accel_task_complete(accel_task, accel_task->status);

	}
@@ -533,7 +533,7 @@ sw_accel_create_cb(void *io_device, void *ctx_buf) 
{

	struct sw_accel_io_channel *sw_ch = ctx_buf;



	TAILQ_INIT(&sw_ch->tasks_to_complete);

	STAILQ_INIT(&sw_ch->tasks_to_complete);

	sw_ch->completion_poller = SPDK_POLLER_REGISTER(accel_comp_poll, sw_ch, 0);



#ifdef SPDK_CONFIG_ISAL

module/accel/dpdk_compressdev/accel_dpdk_compressdev.c

+11 −11
Original line number Diff line number Diff line 
/*   SPDX-License-Identifier: BSD-3-Clause

 *   Copyright (C) 2018 Intel Corporation.

 *   All rights reserved.

 *   Copyright (c) 2021, 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved.

 *   Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.

 */



#include "accel_dpdk_compressdev.h"
@@ -74,7 +74,7 @@ struct compress_io_channel { 
	struct spdk_poller		*poller;

	struct rte_mbuf			**src_mbufs;

	struct rte_mbuf			**dst_mbufs;

	TAILQ_HEAD(, spdk_accel_task)	queued_tasks;

	STAILQ_HEAD(, spdk_accel_task)	queued_tasks;

};



/* Shared mempools between all devices on this system */
@@ -543,7 +543,7 @@ error_get_op: 
		return rc;

	}



	TAILQ_INSERT_TAIL(&chan->queued_tasks, task, link);

	STAILQ_INSERT_TAIL(&chan->queued_tasks, task, link);

	return 0;

}
@@ -598,11 +598,11 @@ comp_dev_poller(void *args) 
		/* Check if there are any pending comp ops to process, only pull one

		 * at a time off as _compress_operation() may re-queue the op.

		 */

		if (!TAILQ_EMPTY(&chan->queued_tasks)) {

			task_to_resubmit = TAILQ_FIRST(&chan->queued_tasks);

		if (!STAILQ_EMPTY(&chan->queued_tasks)) {

			task_to_resubmit = STAILQ_FIRST(&chan->queued_tasks);

			rc = _compress_operation(chan, task_to_resubmit);

			if (rc == 0) {

				TAILQ_REMOVE(&chan->queued_tasks, task_to_resubmit, link);

				STAILQ_REMOVE_HEAD(&chan->queued_tasks, link);

			}

		}

	}
@@ -634,7 +634,7 @@ compress_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *first_ 


	task = first_task;



	if (!TAILQ_EMPTY(&chan->queued_tasks)) {

	if (!STAILQ_EMPTY(&chan->queued_tasks)) {

		goto queue_tasks;

	}
@@ -645,7 +645,7 @@ compress_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *first_ 
	 * passed in here.  Similar thing is done in the accel framework.

	 */

	while (task) {

		tmp = TAILQ_NEXT(task, link);

		tmp = STAILQ_NEXT(task, link);

		rc = _process_single_task(ch, task);



		if (rc == -EBUSY) {
@@ -660,8 +660,8 @@ compress_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *first_ 


queue_tasks:

	while (task != NULL) {

		tmp = TAILQ_NEXT(task, link);

		TAILQ_INSERT_TAIL(&chan->queued_tasks, task, link);

		tmp = STAILQ_NEXT(task, link);

		STAILQ_INSERT_TAIL(&chan->queued_tasks, task, link);

		task = tmp;

	}

	return 0;
@@ -744,7 +744,7 @@ compress_create_cb(void *io_device, void *ctx_buf) 
	}



	chan->poller = SPDK_POLLER_REGISTER(comp_dev_poller, chan, 0);

	TAILQ_INIT(&chan->queued_tasks);

	STAILQ_INIT(&chan->queued_tasks);



	pthread_mutex_lock(&g_comp_device_qp_lock);

	TAILQ_FOREACH(device_qp, &g_comp_device_qp, link) {

module/accel/dsa/accel_dsa.c

+11 −11
Original line number Diff line number Diff line
@@ -52,7 +52,7 @@ struct idxd_io_channel { 
	enum channel_state		state;

	struct spdk_poller		*poller;

	uint32_t			num_outstanding;

	TAILQ_HEAD(, spdk_accel_task)	queued_tasks;

	STAILQ_HEAD(, spdk_accel_task)	queued_tasks;

};



static struct spdk_io_channel *dsa_get_io_channel(void);
@@ -196,21 +196,21 @@ dsa_submit_task(struct spdk_io_channel *ch, struct spdk_accel_task *task) 
	struct idxd_io_channel *chan = spdk_io_channel_get_ctx(ch);

	int rc = 0;



	assert(TAILQ_NEXT(task, link) == NULL);

	assert(STAILQ_NEXT(task, link) == NULL);



	if (spdk_unlikely(chan->state == IDXD_CHANNEL_ERROR)) {

		spdk_accel_task_complete(task, -EINVAL);

		return 0;

	}



	if (!TAILQ_EMPTY(&chan->queued_tasks)) {

		TAILQ_INSERT_TAIL(&chan->queued_tasks, task, link);

	if (!STAILQ_EMPTY(&chan->queued_tasks)) {

		STAILQ_INSERT_TAIL(&chan->queued_tasks, task, link);

		return 0;

	}



	rc = _process_single_task(ch, task);

	if (rc == -EBUSY) {

		TAILQ_INSERT_TAIL(&chan->queued_tasks, task, link);

		STAILQ_INSERT_TAIL(&chan->queued_tasks, task, link);

	} else if (rc) {

		spdk_accel_task_complete(task, rc);

	}
@@ -227,19 +227,19 @@ dsa_submit_queued_tasks(struct idxd_io_channel *chan) 


	if (spdk_unlikely(chan->state == IDXD_CHANNEL_ERROR)) {

		/* Complete queued tasks with error and clear the list */

		while ((task = TAILQ_FIRST(&chan->queued_tasks))) {

			TAILQ_REMOVE(&chan->queued_tasks, task, link);

		while ((task = STAILQ_FIRST(&chan->queued_tasks))) {

			STAILQ_REMOVE_HEAD(&chan->queued_tasks, link);

			spdk_accel_task_complete(task, -EINVAL);

		}

		return 0;

	}



	TAILQ_FOREACH_SAFE(task, &chan->queued_tasks, link, tmp) {

	STAILQ_FOREACH_SAFE(task, &chan->queued_tasks, link, tmp) {

		rc = _process_single_task(ch, task);

		if (rc == -EBUSY) {

			return rc;

		}

		TAILQ_REMOVE(&chan->queued_tasks, task, link);

		STAILQ_REMOVE_HEAD(&chan->queued_tasks, link);

		if (rc) {

			spdk_accel_task_complete(task, rc);

		}
@@ -257,7 +257,7 @@ idxd_poll(void *arg) 
	count = spdk_idxd_process_events(chan->chan);



	/* Check if there are any pending ops to process if the channel is active */

	if (!TAILQ_EMPTY(&chan->queued_tasks)) {

	if (!STAILQ_EMPTY(&chan->queued_tasks)) {

		dsa_submit_queued_tasks(chan);

	}
@@ -320,7 +320,7 @@ dsa_create_cb(void *io_device, void *ctx_buf) 


	chan->dev = dsa;

	chan->poller = SPDK_POLLER_REGISTER(idxd_poll, chan, 0);

	TAILQ_INIT(&chan->queued_tasks);

	STAILQ_INIT(&chan->queued_tasks);

	chan->num_outstanding = 0;

	chan->state = IDXD_CHANNEL_ACTIVE;