Commit b1906912 authored by Liu Xiaodong's avatar Liu Xiaodong Committed by Jim Harris
Browse files

thread: reap unreg pollers in intr 



When intr mode is enabled, it will be common that
poller is unregistered during interrupt processing.
Since poller unregister is a delayed operation,
mark it in spdk_thread object, and reap unregistered
pollers out of poller execution.

Fixes #2143

Change-Id: Ieb61fc7685f85af5c15e833dd1dd56f8c97a3b12
Signed-off-by: default avatarLiu Xiaodong <xiaodong.liu@intel.com>
Reviewed-on: https://review.spdk.io/gerrit/c/spdk/spdk/+/5770


Community-CI: Broadcom CI <spdk-ci.pdl@broadcom.com>
Reviewed-by: default avatarTomasz Zawadzki <tomasz.zawadzki@intel.com>
Reviewed-by: default avatarBen Walker <benjamin.walker@intel.com>
Tested-by: default avatarSPDK CI Jenkins <sys_sgci@intel.com>
parent 3c271d5e

autotest.sh

+1 −0
Original line number Diff line number Diff line
@@ -188,6 +188,7 @@ if [ $SPDK_RUN_FUNCTIONAL_TEST -eq 1 ]; then 
		run_test "bdevperf_config" test/bdev/bdevperf/test_config.sh

		if [[ $(uname -s) == Linux ]]; then

			run_test "reactor_set_interrupt" test/interrupt/reactor_set_interrupt.sh

			run_test "reap_unregistered_poller" test/interrupt/reap_unregistered_poller.sh

		fi

	fi

lib/thread/thread.c

+34 −4
Original line number Diff line number Diff line
@@ -94,6 +94,7 @@ struct spdk_poller { 
	spdk_poller_fn			fn;

	void				*arg;

	struct spdk_thread		*thread;

	/* Native interruptfd for period or busy poller */

	int				interruptfd;

	spdk_poller_set_interrupt_mode_cb set_intr_cb_fn;

	void				*set_intr_cb_arg;
@@ -153,6 +154,7 @@ struct spdk_thread { 


	/* Indicates whether this spdk_thread currently runs in interrupt. */

	bool				in_interrupt;

	bool				poller_unregistered;

	struct spdk_fd_group		*fgrp;



	/* User context allocated at the end */
@@ -737,7 +739,6 @@ poller_insert_timer(struct spdk_thread *thread, struct spdk_poller *poller, uint 
	}

}



#ifdef __linux__

static inline void

poller_remove_timer(struct spdk_thread *thread, struct spdk_poller *poller)

{
@@ -753,7 +754,6 @@ poller_remove_timer(struct spdk_thread *thread, struct spdk_poller *poller) 
		thread->first_timed_poller = RB_MIN(timed_pollers_tree, &thread->timed_pollers);

	}

}

#endif



static void

thread_insert_poller(struct spdk_thread *thread, struct spdk_poller *poller)
@@ -996,6 +996,28 @@ spdk_thread_poll(struct spdk_thread *thread, uint32_t max_msgs, uint64_t now) 
				SPDK_ERRLOG("failed to acknowledge msg queue: %s.\n", spdk_strerror(errno));

			}

		}



		/* Reap unregistered pollers out of poller execution in intr mode */

		if (spdk_unlikely(thread->poller_unregistered)) {

			struct spdk_poller *poller, *tmp;



			TAILQ_FOREACH_REVERSE_SAFE(poller, &thread->active_pollers,

						   active_pollers_head, tailq, tmp) {

				if (poller->state == SPDK_POLLER_STATE_UNREGISTERED) {

					TAILQ_REMOVE(&thread->active_pollers, poller, tailq);

					free(poller);

				}

			}



			RB_FOREACH_SAFE(poller, timed_pollers_tree, &thread->timed_pollers, tmp) {

				if (poller->state == SPDK_POLLER_STATE_UNREGISTERED) {

					poller_remove_timer(thread, poller);

					free(poller);

				}

			}



			thread->poller_unregistered = false;

		}

	}
@@ -1582,10 +1604,18 @@ spdk_poller_unregister(struct spdk_poller **ppoller) 
		return;

	}



	if (spdk_interrupt_mode_is_enabled() && poller->interruptfd >= 0) {

	if (spdk_interrupt_mode_is_enabled()) {

		/* Release the interrupt resource for period or busy poller */

		if (poller->interruptfd >= 0) {

			poller_interrupt_fini(poller);

		}



		/* Mark there is poller unregistered. Then unregistered pollers will

		 * get reaped by spdk_thread_poll also in intr mode.

		 */

		thread->poller_unregistered = true;

	}



	/* If the poller was paused, put it on the active_pollers list so that

	 * its unregistration can be processed by spdk_thread_poll().

	 */

test/interrupt/reap_unregistered_poller.sh

0 → 100755
+38 −0
Original line number Diff line number Diff line 
#!/usr/bin/env bash



testdir=$(readlink -f $(dirname $0))

rootdir=$(readlink -f $testdir/../..)

source $rootdir/test/common/autotest_common.sh

source $testdir/interrupt_common.sh



export PYTHONPATH=$rootdir/examples/interrupt_tgt



# Set reactors with intr_tgt in intr mode

start_intr_tgt



# Record names of native created pollers.

app_thread=$(rpc_cmd thread_get_pollers | jq -r '.threads[0]')

native_pollers=$(jq -r '.active_pollers[].name' <<< $app_thread)

native_pollers+=" "

native_pollers+=$(jq -r '.timed_pollers[].name' <<< $app_thread)



# Create one aio_bdev.

# During the creation, vbdev examine process will get bdev_aio create

# pollers like bdev_aio_group_poll poller, and then unregister it.

setup_bdev_aio



# Record names of remaining pollers.

app_thread=$(rpc_cmd thread_get_pollers | jq -r '.threads[0]')

remaining_pollers=$(jq -r '.active_pollers[].name' <<< $app_thread)

remaining_pollers+=" "

remaining_pollers+=$(jq -r '.timed_pollers[].name' <<< $app_thread)



# Since bdev_aio created pollers were already unregistered, so

# remaining_pollers should be same with native_pollers.

if [[ "$remaining_pollers" != "$native_pollers" ]]; then

	return 1

fi



trap - SIGINT SIGTERM EXIT

killprocess $intr_tgt_pid

cleanup