copy_engine, ioat: add spdk_io_channel support (eb605a82) · Commits · Public Repositories / spdk

include/spdk/copy_engine.h

+2 −0

Original line number	Diff line number	Diff line
		@@ -56,6 +56,7 @@ struct spdk_copy_engine {
		int64_t (fill)(void cb_arg, void *dst, uint8_t fill,
		uint64_t nbytes, copy_completion_cb cb);
		void (*check_io)(void);
		struct spdk_io_channel (get_io_channel)(uint32_t priority);
		};

		struct spdk_copy_module_if {
		@@ -84,6 +85,7 @@ struct spdk_copy_module_if {
		};

		void spdk_copy_engine_register(struct spdk_copy_engine *copy_engine);
		struct spdk_io_channel *spdk_copy_engine_get_io_channel(uint32_t priority);
		int64_t spdk_copy_submit(struct copy_task copy_req, void dst, void *src,
		uint64_t nbytes, copy_completion_cb cb);
		int64_t spdk_copy_submit_fill(struct copy_task copy_req, void dst, uint8_t fill,

lib/copy/copy_engine.c

+66 −2

Original line number	Diff line number	Diff line
		@@ -35,14 +35,14 @@

		#include <stdio.h>
		#include <errno.h>
		#include <assert.h>
		#include <rte_config.h>
		#include <rte_debug.h>
		#include <rte_malloc.h>
		#include <rte_memcpy.h>
		#include <rte_lcore.h>

		#include "spdk/log.h"
		#include "spdk/event.h"
		#include "spdk/io_channel.h"

		static struct spdk_copy_engine *hw_copy_engine = NULL;
		/* Memcpy engine always exist */
		@@ -51,6 +51,11 @@ static struct spdk_copy_engine *mem_copy_engine = NULL;
		TAILQ_HEAD(, spdk_copy_module_if) spdk_copy_module_list =
		TAILQ_HEAD_INITIALIZER(spdk_copy_module_list);

		struct copy_io_channel {
		struct spdk_copy_engine *engine;
		struct spdk_io_channel *ch;
		};

		void
		spdk_copy_engine_register(struct spdk_copy_engine *copy_engine)
		{
		@@ -149,11 +154,30 @@ mem_copy_fill(void cb_arg, void dst, uint8_t fill, uint64_t nbytes,
		return nbytes;
		}

		static struct spdk_io_channel *mem_get_io_channel(uint32_t priority);

		static struct spdk_copy_engine memcpy_copy_engine = {
		.copy = mem_copy_submit,
		.fill = mem_copy_fill,
		.get_io_channel = mem_get_io_channel,
		};

		static int
		memcpy_create_cb(void io_device, uint32_t priority, void ctx_buf)
		{
		return 0;
		}

		static void
		memcpy_destroy_cb(void io_device, void ctx_buf)
		{
		}

		static struct spdk_io_channel *mem_get_io_channel(uint32_t priority)
		{
		return spdk_get_io_channel(&memcpy_copy_engine, priority);
		}

		static int
		copy_engine_mem_get_ctx_size(void)
		{
		@@ -178,10 +202,44 @@ void spdk_copy_module_list_add(struct spdk_copy_module_if *copy_module)
		TAILQ_INSERT_TAIL(&spdk_copy_module_list, copy_module, tailq);
		}

		static int
		copy_create_cb(void io_device, uint32_t priority, void ctx_buf)
		{
		struct copy_io_channel *copy_ch = ctx_buf;

		if (hw_copy_engine != NULL) {
		copy_ch->ch = hw_copy_engine->get_io_channel(priority);
		if (copy_ch->ch != NULL) {
		copy_ch->engine = hw_copy_engine;
		return 0;
		}
		}

		copy_ch->ch = mem_copy_engine->get_io_channel(priority);
		assert(copy_ch->ch != NULL);
		copy_ch->engine = mem_copy_engine;
		return 0;
		}

		static void
		copy_destroy_cb(void io_device, void ctx_buf)
		{
		struct copy_io_channel *copy_ch = ctx_buf;

		spdk_put_io_channel(copy_ch->ch);
		}

		struct spdk_io_channel *
		spdk_copy_engine_get_io_channel(uint32_t priority)
		{
		return spdk_get_io_channel(&spdk_copy_module_list, priority);
		}

		static int
		copy_engine_mem_init(void)
		{
		spdk_memcpy_register(&memcpy_copy_engine);
		spdk_io_device_register(&memcpy_copy_engine, memcpy_create_cb, memcpy_destroy_cb, 0);

		return 0;
		}
		@@ -211,6 +269,12 @@ static int
		spdk_copy_engine_initialize(void)
		{
		spdk_copy_engine_module_initialize();
		/*
		* We need a unique identifier for the copy engine framework, so use the
		* spdk_copy_module_list address for this purpose.
		*/
		spdk_io_device_register(&spdk_copy_module_list, copy_create_cb, copy_destroy_cb,
		sizeof(struct copy_io_channel));
		return 0;
		}

lib/copy/ioat/copy_engine_ioat.c

+45 −0

Original line number	Diff line number	Diff line
		@@ -46,6 +46,7 @@
		#include "spdk/log.h"
		#include "spdk/event.h"
		#include "spdk/pci.h"
		#include "spdk/io_channel.h"

		#include "spdk/ioat.h"

		@@ -70,6 +71,12 @@ struct ioat_whitelist {
		uint32_t func;
		};

		struct ioat_io_channel {
		struct spdk_ioat_chan *ioat_ch;
		struct ioat_device *ioat_dev;
		struct spdk_poller *poller;
		};

		static int
		ioat_find_dev_by_whitelist_bdf(struct spdk_pci_device *dev,
		struct ioat_whitelist *whitelist,
		@@ -202,12 +209,48 @@ ioat_check_io(void)
		ioat_poll(chan);
		}

		static struct spdk_io_channel *ioat_get_io_channel(uint32_t priority);

		static struct spdk_copy_engine ioat_copy_engine = {
		.copy = ioat_copy_submit,
		.fill = ioat_copy_submit_fill,
		.check_io = ioat_check_io,
		.get_io_channel = ioat_get_io_channel,
		};

		static int
		ioat_create_cb(void io_device, uint32_t priority, void ctx_buf)
		{
		struct ioat_io_channel *ch = ctx_buf;
		struct ioat_device *ioat_dev;

		ioat_dev = ioat_allocate_device();
		if (ioat_dev == NULL) {
		return -1;
		}

		ch->ioat_dev = ioat_dev;
		ch->ioat_ch = ioat_dev->ioat;
		spdk_poller_register(&ch->poller, ioat_poll, ch->ioat_ch,
		spdk_app_get_current_core(), NULL, 0);
		return 0;
		}

		static void
		ioat_destroy_cb(void io_device, void ctx_buf)
		{
		struct ioat_io_channel *ch = ctx_buf;

		ioat_free_device(ch->ioat_dev);
		spdk_poller_unregister(&ch->poller, NULL);
		}

		static struct spdk_io_channel *
		ioat_get_io_channel(uint32_t priority)
		{
		return spdk_get_io_channel(&ioat_copy_engine, priority);
		}

		struct ioat_probe_ctx {
		int num_whitelist_devices;
		struct ioat_whitelist whitelist[IOAT_MAX_CHANNELS];
		@@ -334,6 +377,8 @@ copy_engine_ioat_init(void)

		SPDK_NOTICELOG("Ioat Copy Engine Offload Enabled\n");
		spdk_copy_engine_register(&ioat_copy_engine);
		spdk_io_device_register(&ioat_copy_engine, ioat_create_cb, ioat_destroy_cb,
		sizeof(struct ioat_io_channel));

		return 0;
		}