Commit ee7e31f9 authored by paul luse's avatar paul luse Committed by Jim Harris
Browse files

lib/accel: remove the need for the app to allocate an accel_task



This was sort of a clunky interface requiring a couple of inline
functions in every app that wants to use the accel_fw moving
forward. By having the accel_fw public API accept a callback arg
instead of an accel_task combined with adding a pool of accel_tasks
in the accel_fw engine we can eliminate this.

After changing the parm to a cb_arg, changes were made to all accel_fw
interfaces to put cb_fn and cb_arg as the last parms in public and
private function calls.

Related bdev_malloc changes need to be in this patch in order to pass CI.

Signed-off-by: default avatarpaul luse <paul.e.luse@intel.com>
Change-Id: I2b75764e534562d91484a094c3352266156d8425
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/3209


Community-CI: Mellanox Build Bot
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
Reviewed-by: default avatarJim Harris <james.r.harris@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
parent 6e152850
Loading
Loading
Loading
Loading
+4 −0
Original line number Diff line number Diff line
@@ -4,6 +4,10 @@

### accel_fw

The accel_fw was updated to no longer require the app to allocate an
accel_task on its behalf. All public APIs now take a callback arg as
the parameter that used to be the accel_task.

The accel_fw was updated to support compare, dualcast, crc32c.

The accel_fw introduced batching support for all commands in all plug-ins.
+28 −32
Original line number Diff line number Diff line
@@ -210,20 +210,18 @@ _submit_single(void *arg1, void *arg2)
	task->worker->current_queue_depth++;
	switch (g_workload_selection) {
	case ACCEL_COPY:
		rc = spdk_accel_submit_copy(__accel_task_from_ap_task(task),
					    worker->ch, task->dst,
					    task->src, g_xfer_size_bytes, accel_done);
		rc = spdk_accel_submit_copy(worker->ch, task->dst,
					    task->src, g_xfer_size_bytes, accel_done,
					    __accel_task_from_ap_task(task));
		break;
	case ACCEL_FILL:
		/* For fill use the first byte of the task->dst buffer */
		rc = spdk_accel_submit_fill(__accel_task_from_ap_task(task),
					    worker->ch, task->dst, *(uint8_t *)task->src,
					    g_xfer_size_bytes, accel_done);
		rc = spdk_accel_submit_fill(worker->ch, task->dst, *(uint8_t *)task->src,
					    g_xfer_size_bytes, accel_done, __accel_task_from_ap_task(task));
		break;
	case ACCEL_CRC32C:
		rc = spdk_accel_submit_crc32c(__accel_task_from_ap_task(task),
					      worker->ch, (uint32_t *)task->dst, task->src, g_crc32c_seed,
					      g_xfer_size_bytes, accel_done);
		rc = spdk_accel_submit_crc32c(worker->ch, (uint32_t *)task->dst, task->src, g_crc32c_seed,
					      g_xfer_size_bytes, accel_done, __accel_task_from_ap_task(task));
		break;
	case ACCEL_COMPARE:
		random_num = rand() % 100;
@@ -234,14 +232,13 @@ _submit_single(void *arg1, void *arg2)
			task->expected_status = 0;
			*(uint8_t *)task->dst = DATA_PATTERN;
		}
		rc = spdk_accel_submit_compare(__accel_task_from_ap_task(task),
					       worker->ch, task->dst, task->src,
					       g_xfer_size_bytes, accel_done);
		rc = spdk_accel_submit_compare(worker->ch, task->dst, task->src,
					       g_xfer_size_bytes, accel_done, __accel_task_from_ap_task(task));
		break;
	case ACCEL_DUALCAST:
		rc = spdk_accel_submit_dualcast(__accel_task_from_ap_task(task),
						worker->ch, task->dst, task->dst2,
						task->src, g_xfer_size_bytes, accel_done);
		rc = spdk_accel_submit_dualcast(worker->ch, task->dst, task->dst2,
						task->src, g_xfer_size_bytes, accel_done,
						__accel_task_from_ap_task(task));
		break;
	default:
		assert(false);
@@ -470,29 +467,29 @@ _batch_prep_cmd(struct worker_thread *worker, struct ap_task *task, struct spdk_

	switch (g_workload_selection) {
	case ACCEL_COPY:
		rc = spdk_accel_batch_prep_copy(__accel_task_from_ap_task(task),
						worker->ch, batch, task->dst,
						task->src, g_xfer_size_bytes, accel_done);
		rc = spdk_accel_batch_prep_copy(worker->ch, batch, task->dst,
						task->src, g_xfer_size_bytes, accel_done,
						__accel_task_from_ap_task(task));
		break;
	case ACCEL_DUALCAST:
		rc = spdk_accel_batch_prep_dualcast(__accel_task_from_ap_task(task),
						    worker->ch, batch, task->dst, task->dst2,
						    task->src, g_xfer_size_bytes, accel_done);
		rc = spdk_accel_batch_prep_dualcast(worker->ch, batch, task->dst, task->dst2,
						    task->src, g_xfer_size_bytes, accel_done,
						    __accel_task_from_ap_task(task));
		break;
	case ACCEL_COMPARE:
		rc = spdk_accel_batch_prep_compare(__accel_task_from_ap_task(task),
						   worker->ch, batch, task->dst, task->src,
						   g_xfer_size_bytes, accel_done);
		rc = spdk_accel_batch_prep_compare(worker->ch, batch, task->dst, task->src,
						   g_xfer_size_bytes, accel_done,
						   __accel_task_from_ap_task(task));
		break;
	case ACCEL_FILL:
		rc = spdk_accel_batch_prep_fill(__accel_task_from_ap_task(task),
						worker->ch, batch, task->dst, *(uint8_t *)task->src,
						g_xfer_size_bytes, accel_done);
		rc = spdk_accel_batch_prep_fill(worker->ch, batch, task->dst, *(uint8_t *)task->src,
						g_xfer_size_bytes, accel_done,
						__accel_task_from_ap_task(task));
		break;
	case ACCEL_CRC32C:
		rc = spdk_accel_batch_prep_crc32c(__accel_task_from_ap_task(task),
						  worker->ch, batch, (uint32_t *)task->dst, task->src,
						  g_crc32c_seed, g_xfer_size_bytes, accel_done);
		rc = spdk_accel_batch_prep_crc32c(worker->ch, batch, (uint32_t *)task->dst, task->src,
						  g_crc32c_seed, g_xfer_size_bytes, accel_done,
						  __accel_task_from_ap_task(task));
		break;
	default:
		assert(false);
@@ -594,8 +591,7 @@ _init_thread(void *arg1)
			task->worker = worker;
			task->worker->current_queue_depth++;

			rc = spdk_accel_batch_submit(__accel_task_from_ap_task(task),
						     worker->ch, batch, batch_done);
			rc = spdk_accel_batch_submit(worker->ch, batch, batch_done, __accel_task_from_ap_task(task));
			if (rc) {
				fprintf(stderr, "error ending batch %d\n", rc);
				goto error;
+58 −60
Original line number Diff line number Diff line
@@ -123,17 +123,17 @@ uint64_t spdk_accel_get_capabilities(struct spdk_io_channel *ch);
/**
 * Submit a copy request.
 *
 * \param accel_req Accel request task.
 * \param ch I/O channel to submit request to the accel engine.
 * \param ch I/O channel associated with this call.
 * \param dst Destination to copy to.
 * \param src Source to copy from.
 * \param nbytes Length in bytes to copy.
 * \param cb Called when this copy operation completes.
 * \param cb_fn Called when this copy operation completes.
 * \param cb_arg Callback argument.
 *
 * \return 0 on success, negative errno on failure.
 */
int spdk_accel_submit_copy(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch, void *dst,
			   void *src, uint64_t nbytes, spdk_accel_completion_cb cb);
int spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, uint64_t nbytes,
			   spdk_accel_completion_cb cb_fn, void *cb_arg);

/**
 * Synchronous call to get batch size. This is the maximum number of
@@ -158,15 +158,15 @@ struct spdk_accel_batch *spdk_accel_batch_create(struct spdk_io_channel *ch);
/**
 * Asynchronous call to submit a batch sequence.
 *
 * \param accel_req Accel request task.
 * \param ch I/O channel associated with this call.
 * \param batch Handle provided when the batch was started with spdk_accel_batch_create().
 * \param cb Called when this operation completes.
 * \param cb_fn Called when this operation completes.
 * \param cb_arg Callback argument.
 *
 * \return 0 on success, negative errno on failure.
 */
int spdk_accel_batch_submit(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch,
			    struct spdk_accel_batch *batch, spdk_accel_completion_cb cb);
int spdk_accel_batch_submit(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
			    spdk_accel_completion_cb cb_fn, void *cb_arg);

/**
 * Synchronous call to prepare a copy request into a previously initialized batch
@@ -174,19 +174,19 @@ int spdk_accel_batch_submit(struct spdk_accel_task *accel_req, struct spdk_io_ch
 *  completes after the batch has been submitted by an asynchronous call to
 *  spdk_accel_batch_submit().
 *
 * \param accel_req Accel request task.
 * \param ch I/O channel associated with this call.
 * \param batch Handle provided when the batch was started with spdk_accel_batch_create().
 * \param dst Destination to copy to.
 * \param src Source to copy from.
 * \param nbytes Length in bytes to copy.
 * \param cb Called when this operation completes.
 * \param cb_fn Called when this operation completes.
 * \param cb_arg Callback argument.
 *
 * \return 0 on success, negative errno on failure.
 */
int spdk_accel_batch_prep_copy(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch,
			       struct spdk_accel_batch *batch, void *dst, void *src,
			       uint64_t nbytes, spdk_accel_completion_cb cb);
int spdk_accel_batch_prep_copy(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
			       void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb_fn,
			       void *cb_arg);

/**
 * Synchronous call to prepare a dualcast request into a previously initialized batch
@@ -194,37 +194,36 @@ int spdk_accel_batch_prep_copy(struct spdk_accel_task *accel_req, struct spdk_io
 *  completes after the batch has been submitted by an asynchronous call to
 *  spdk_accel_batch_submit().
 *
 * \param accel_req Accel request task.
 * \param ch I/O channel associated with this call.
 * \param batch Handle provided when the batch was started with spdk_accel_batch_create().
 * \param dst1 First destination to copy to (must be 4K aligned).
 * \param dst2 Second destination to copy to (must be 4K aligned).
 * \param src Source to copy from.
 * \param nbytes Length in bytes to copy.
 * \param cb Called when this operation completes.
 * \param cb_fn Called when this operation completes.
 * \param cb_arg Callback argument.
 *
 * \return 0 on success, negative errno on failure.
 */
int spdk_accel_batch_prep_dualcast(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch,
				   struct spdk_accel_batch *batch, void *dst1, void *dst2, void *src,
				   uint64_t nbytes, spdk_accel_completion_cb cb);
int spdk_accel_batch_prep_dualcast(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
				   void *dst1, void *dst2, void *src, uint64_t nbytes,
				   spdk_accel_completion_cb cb_fn, void *cb_arg);

/**
 * Submit a dual cast copy request.
 *
 * \param accel_req Accel request task.
 * \param ch I/O channel to submit request to the accel engine.
 * \param ch I/O channel associated with this call.
 * \param dst1 First destination to copy to (must be 4K aligned).
 * \param dst2 Second destination to copy to (must be 4K aligned).
 * \param src Source to copy from.
 * \param nbytes Length in bytes to copy.
 * \param cb Called when this copy operation completes.
 * \param cb_fn Called when this copy operation completes.
 * \param cb_arg Callback argument.
 *
 * \return 0 on success, negative errno on failure.
 */
int spdk_accel_submit_dualcast(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch,
			       void *dst1, void *dst2, void *src, uint64_t nbytes,
			       spdk_accel_completion_cb cb);
int spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, void *dst2, void *src,
			       uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);

/**
 * Synchronous call to prepare a compare request into a previously initialized batch
@@ -232,35 +231,34 @@ int spdk_accel_submit_dualcast(struct spdk_accel_task *accel_req, struct spdk_io
 *  completes after the batch has been submitted by an asynchronous call to
 *  spdk_accel_batch_submit().
 *
 * \param accel_req Accel request task.
 * \param ch I/O channel to submit request to the accel engine.
 * \param ch I/O channel associated with this call.
 * \param batch Handle provided when the batch was started with spdk_accel_batch_create().
 * \param src1 First location to perform compare on.
 * \param src2 Second location to perform compare on.
 * \param nbytes Length in bytes to compare.
 * \param cb Called when this operation completes.
 * \param cb_fn Called when this operation completes.
 * \param cb_arg Callback argument.
 *
 * \return 0 on success, negative errno on failure.
 */
int spdk_accel_batch_prep_compare(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch,
				  struct spdk_accel_batch *batch, void *src1, void *src2,
				  uint64_t nbytes, spdk_accel_completion_cb cb);
int spdk_accel_batch_prep_compare(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
				  void *src1, void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn,
				  void *cb_arg);

/**
 * Submit a compare request.
 *
 * \param accel_req Accel request task.
 * \param ch I/O channel to submit request to the accel engine.
 * \param ch I/O channel associated with this call.
 * \param src1 First location to perform compare on.
 * \param src2 Second location to perform compare on.
 * \param nbytes Length in bytes to compare.
 * \param cb Called when this compare operation completes.
 * \param cb_fn Called when this compare operation completes.
 * \param cb_arg Callback argument.
 *
 * \return 0 on success, any other value means there was a miscompare.
 */
int spdk_accel_submit_compare(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch,
			      void *src1, void *src2, uint64_t nbytes,
			      spdk_accel_completion_cb cb);
int spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1, void *src2, uint64_t nbytes,
			      spdk_accel_completion_cb cb_fn, void *cb_arg);

/**
 * Synchronous call to prepare a fill request into a previously initialized batch
@@ -268,36 +266,36 @@ int spdk_accel_submit_compare(struct spdk_accel_task *accel_req, struct spdk_io_
 *  completes after the batch has been submitted by an asynchronous call to
 *  spdk_accel_batch_submit().
 *
 * \param accel_req Accel request task.
 * \param ch I/O channel to submit request to the accel engine.
 * \param ch I/O channel associated with this call.
 * \param batch Handle provided when the batch was started with spdk_accel_batch_create().
 * \param dst Destination to fill.
 * \param fill Constant byte to fill to the destination.
 * \param nbytes Length in bytes to fill.
 * \param cb Called when this operation completes.
 * \param cb_fn Called when this operation completes.
 * \param cb_arg Callback argument.
 *
 * \return 0 on success, negative errno on failure.
 */
int spdk_accel_batch_prep_fill(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch,
			       struct spdk_accel_batch *batch, void *dst, uint8_t fill,
			       uint64_t nbytes, spdk_accel_completion_cb cb);
int spdk_accel_batch_prep_fill(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
			       void *dst, uint8_t fill, uint64_t nbytes,
			       spdk_accel_completion_cb cb_fn, void *cb_arg);

/**
 * Submit a fill request.
 *
 * This operation will fill the destination buffer with the specified value.
 *
 * \param accel_req Accel request task.
 * \param ch I/O channel to submit request to the accel engine.
 * \param ch I/O channel associated with this call.
 * \param dst Destination to fill.
 * \param fill Constant byte to fill to the destination.
 * \param nbytes Length in bytes to fill.
 * \param cb Called when this fill operation completes.
 * \param cb_fn Called when this fill operation completes.
 * \param cb_arg Callback argument.
 *
 * \return 0 on success, negative errno on failure.
 */
int spdk_accel_submit_fill(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch,
			   void *dst, uint8_t fill, uint64_t nbytes, spdk_accel_completion_cb cb);
int spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, uint8_t fill, uint64_t nbytes,
			   spdk_accel_completion_cb cb_fn, void *cb_arg);

/**
 * Synchronous call to prepare a crc32c request into a previously initialized batch
@@ -305,38 +303,38 @@ int spdk_accel_submit_fill(struct spdk_accel_task *accel_req, struct spdk_io_cha
 *  completes after the batch has been submitted by an asynchronous call to
 *  spdk_accel_batch_submit().
 *
 * \param accel_req Accel request task.
 * \param ch I/O channel to submit request to the accel engine.
 * \param ch I/O channel associated with this call.
 * \param batch Handle provided when the batch was started with spdk_accel_batch_create().
 * \param dst Destination to write the CRC-32C to.
 * \param src The source address for the data.
 * \param seed Four byte seed value.
 * \param nbytes Length in bytes.
 * \param cb Called when this operation completes.
 * \param cb_fn Called when this operation completes.
 * \param cb_arg Callback argument.
 *
 * \return 0 on success, negative errno on failure.
 */
int spdk_accel_batch_prep_crc32c(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch,
				 struct spdk_accel_batch *batch, uint32_t *dst, void *src, uint32_t seed,
				 uint64_t nbytes, spdk_accel_completion_cb cb);
int spdk_accel_batch_prep_crc32c(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
				 uint32_t *dst, void *src, uint32_t seed, uint64_t nbytes,
				 spdk_accel_completion_cb cb_fn, void *cb_arg);

/**
 * Submit a CRC-32C calculation request.
 *
 * This operation will calculate the 4 byte CRC32-C for the given data.
 *
 * \param accel_req Accel request task.
 * \param ch I/O channel to submit request to the accel engine.
 * \param ch I/O channel associated with this call.
 * \param dst Destination to write the CRC-32C to.
 * \param src The source address for the data.
 * \param seed Four byte seed value.
 * \param nbytes Length in bytes.
 * \param cb Called when this CRC-32C operation completes.
 * \param cb_fn Called when this CRC-32C operation completes.
 * \param cb_arg Callback argument.
 *
 * \return 0 on success, negative errno on failure.
 */
int spdk_accel_submit_crc32c(struct spdk_accel_task *accel_req, struct spdk_io_channel *ch,
			     uint32_t *dst, void *src, uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb);
int spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *dst, void *src, uint32_t seed,
			     uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);

/**
 * Get the size of an acceleration task.
+24 −22
Original line number Diff line number Diff line
@@ -41,36 +41,38 @@

struct spdk_accel_task {
	spdk_accel_completion_cb	cb;
	void				*cb_arg;
	uint8_t				offload_ctx[0];
};

struct spdk_accel_engine {
	uint64_t (*get_capabilities)(void);
	int (*copy)(void *cb_arg, struct spdk_io_channel *ch, void *dst, void *src,
		    uint64_t nbytes, spdk_accel_completion_cb cb);
	int (*dualcast)(void *cb_arg, struct spdk_io_channel *ch, void *dst1, void *dst2, void *src,
			uint64_t nbytes, spdk_accel_completion_cb cb);
	int (*copy)(struct spdk_io_channel *ch, void *dst, void *src,
		    uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
	int (*dualcast)(struct spdk_io_channel *ch, void *dst1, void *dst2, void *src,
			uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
	uint32_t (*batch_get_max)(void);
	struct spdk_accel_batch *(*batch_create)(struct spdk_io_channel *ch);
	int (*batch_prep_copy)(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
			       void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb);
	int (*batch_prep_dualcast)(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
				   void *dst1, void *dst2, void *src, uint64_t nbytes, spdk_accel_completion_cb cb);
	int (*batch_prep_compare)(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
				  void *src1, void *src2, uint64_t nbytes, spdk_accel_completion_cb cb);
	int (*batch_prep_fill)(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
			       void *dst, uint8_t fill, uint64_t nbytes, spdk_accel_completion_cb cb);
	int (*batch_prep_crc32c)(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
	int (*batch_prep_copy)(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
			       void *dst, void *src, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
	int (*batch_prep_dualcast)(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
				   void *dst1, void *dst2, void *src, uint64_t nbytes,
				   spdk_accel_completion_cb cb_fn, void *cb_arg);
	int (*batch_prep_compare)(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
				  void *src1, void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
	int (*batch_prep_fill)(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
			       void *dst, uint8_t fill, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
	int (*batch_prep_crc32c)(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
				 uint32_t *dst, void *src, uint32_t seed, uint64_t nbytes,
				 spdk_accel_completion_cb cb);
	int (*batch_submit)(void *cb_arg, struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
			    spdk_accel_completion_cb cb);
	int (*compare)(void *cb_arg, struct spdk_io_channel *ch, void *src1, void *src2,
		       uint64_t nbytes, spdk_accel_completion_cb cb);
	int (*fill)(void *cb_arg, struct spdk_io_channel *ch, void *dst, uint8_t fill,
		    uint64_t nbytes, spdk_accel_completion_cb cb);
	int (*crc32c)(void *cb_arg, struct spdk_io_channel *ch, uint32_t *dst, void *src,
		      uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb);
				 spdk_accel_completion_cb cb_fn, void *cb_arg);
	int (*batch_submit)(struct spdk_io_channel *ch, struct spdk_accel_batch *batch,
			    spdk_accel_completion_cb cb_fn, void *cb_arg);
	int (*compare)(struct spdk_io_channel *ch, void *src1, void *src2,
		       uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
	int (*fill)(struct spdk_io_channel *ch, void *dst, uint8_t fill,
		    uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
	int (*crc32c)(struct spdk_io_channel *ch, uint32_t *dst, void *src,
		      uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg);
	struct spdk_io_channel *(*get_io_channel)(void);
};

+218 −123

File changed.

Preview size limit exceeded, changes collapsed.

Loading