Move logic for retrying subtests to the CI

Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>

3 files changed, 41 insertions, 80 deletions

diff --git a/ci/_test-git-branch.sh b/ci/_test-git-branch.sh
index 14d7f34..de43b4f 100644
--- a/ci/_test-git-branch.sh
+++ b/ci/_test-git-branch.sh
@@ -4,6 +4,7 @@ set -o nounset
 set -o errexit
 set -o errtrace
 
+KTEST_DIR=$(dirname "$(readlink -e "$0")")/..
 JOBSERVER_LINUX_REPO=ssh://$JOBSERVER/$JOBSERVER_HOME/linux
 
 git_fetch()
@@ -31,6 +32,7 @@ BRANCH=${TEST_JOB[0]}
 COMMIT=${TEST_JOB[1]}
 TEST_PATH=${TEST_JOB[2]}
 TEST_NAME=$(basename -s .ktest $TEST_PATH)
+SUBTESTS=( "${TEST_JOB[@]:3}" )
 
 if [[ -z $BRANCH ]]; then
     echo "Error getting test job: need git branch"
@@ -47,8 +49,6 @@ if [[ -z $TEST_PATH ]]; then
     exit 1
 fi
 
-echo "Running test job ${TEST_JOB[@]}"
-
 sync_git_repos
 git_fetch linux $COMMIT
 git checkout FETCH_HEAD
@@ -56,10 +56,44 @@ git checkout FETCH_HEAD
 git_fetch linux ci-monkeypatch
 git merge --no-edit FETCH_HEAD || git reset --hard
 
-mkdir -p ktest-out
 rm -rf ktest-out/out
+mkdir -p ktest-out/out
+
+# Mark tests as not run:
+for t in ${SUBTESTS[@]}; do
+    t=$(echo "$t"|tr / .)
+
+    mkdir ktest-out/out/$TEST_NAME.$t
+    echo "========= NOT STARTED" > ktest-out/out/$TEST_NAME.$t/status
+done
+
+make -C "$KTEST_DIR/lib" qemu-wrapper
+
+while (( ${#SUBTESTS[@]} )); do
+    FULL_LOG=$TEST_NAME.$(hostname).$(date -Iseconds).log
+
+    for t in ${SUBTESTS[@]}; do
+	FNAME=$(echo "$t"|tr / .)
+	ln -sfr "ktest-out/out/$FULL_LOG.br"			\
+	    "ktest-out/out/$TEST_NAME.$FNAME/full_log.br"
+    done
+
+    $KTEST_DIR/lib/qemu-wrapper -T 1200 -f "$FULL_LOG" -S -F	\
+	-b $TEST_NAME -o ktest-out/out				\
+	build-test-kernel run $TEST_PATH ${SUBTESTS[@]} || true
+
+    SUBTESTS_REMAINING=()
+
+    for t in ${SUBTESTS[@]:1}; do
+	FNAME=$(echo "$t"|tr / .)
+
+	if grep -q "NOT STARTED" ktest-out/out/$TEST_NAME.$FNAME/status; then
+	    SUBTESTS_REMAINING+=($t)
+	fi
+    done
 
-build-test-kernel run $TEST_PATH ${TEST_JOB[@]:3} || true
+    SUBTESTS=( "${SUBTESTS_REMAINING[@]}" )
+done
 
 find ktest-out/out -type f -name \*log -print0|xargs -0 brotli --rm -9
 
diff --git a/lib/libktest.sh b/lib/libktest.sh
index 72acdbc..301b2dd 100644
--- a/lib/libktest.sh
+++ b/lib/libktest.sh
@@ -277,9 +277,6 @@ start_vm()
 
     get_tmpdir
 
-    # Was outputting to a single log file, now a directory:
-    [[ -f $ktest_out/out ]] && rm -f "$ktest_out/out"
-
     rm -f "$ktest_out/core.*"
     rm -f "$ktest_out/vmcore"
     rm -f "$ktest_out/vm"
@@ -407,65 +404,6 @@ start_vm()
     done
 
     set +o errexit
-
-    ktest_tests=$(echo $ktest_tests)
-    local ktest_vm_idx=0
-
-    while true; do
-	local full_log=$ktest_basename.$(hostname).$(date -Iseconds).log
-
-	for t in $ktest_tests; do
-	    local fname=$(echo "$t"|tr / .)
-	    ln -sfr "$ktest_out/out/$full_log.br"			\
-		    "$ktest_out/out/$ktest_basename.$fname/full_log.br"
-	done
-
-	local qemu_wrapper_cmd=("$ktest_dir/lib/qemu-wrapper")
-
-	qemu_wrapper_cmd+=(-T $ktest_timeout)
-	qemu_wrapper_cmd+=(-f "$full_log")
-
-	if [[ $ktest_interactive = 1 ]]; then
-	    true
-	elif [[ $ktest_exit_on_success = 1 ]]; then
-	    qemu_wrapper_cmd+=(-S)
-	else
-	    qemu_wrapper_cmd+=(-S -F)
-	fi
-
-	local test_basename=$(basename -s .ktest "$ktest_test")
-	qemu_wrapper_cmd+=(-b "$test_basename")
-	qemu_wrapper_cmd+=(-o "$ktest_out/out")
-
-	save_env
-
-	echo "Starting KVM ($ktest_vm_idx)"
-	"${qemu_wrapper_cmd[@]}" -- "${qemu_cmd[@]}"
-	local ret=$?
-
-	$ktest_failfast && break
-
-	local tests_remaining=
-
-	for t in $ktest_tests; do
-	    local fname=$(echo "$t"|tr / .)
-	    if grep -q "NOT STARTED" $ktest_out/out/$ktest_basename.$fname/status; then
-		tests_remaining="$tests_remaining $t"
-	    fi
-	done
-
-	tests_remaining=$(echo $tests_remaining)
-
-	[[ -z $tests_remaining ]] && break
-
-	if [[ $tests_remaining = $ktest_tests ]]; then
-	    echo "Failed to make progress"
-	    break
-	fi
-
-	ktest_tests="$tests_remaining"
-	let ktest_vm_idx++
-    done
-
-    exit $ret
+    save_env
+    "${qemu_cmd[@]}"
 }
diff --git a/lib/parse-test.sh b/lib/parse-test.sh
index a529901..3f37517 100644
--- a/lib/parse-test.sh
+++ b/lib/parse-test.sh
@@ -1,8 +1,6 @@
 
 parse_test_deps()
 {
-    ktest_basename=$(basename -s .ktest "$ktest_test")
-
     #export ktest_crashdump
     eval $("$ktest_test" deps)
 
@@ -18,6 +16,7 @@ parse_test_deps()
     fi
 
     ktest_tests=$("$ktest_test" list-tests)
+    ktest_tests=$(echo $ktest_tests)
 
     if [[ -z $ktest_tests ]]; then
 	echo "No tests found"
@@ -38,14 +37,4 @@ parse_test_deps()
 
 	ktest_tests="$ktest_testargs"
     fi
-
-    # Mark tests not run:
-    rm -rf "$ktest_out/out"
-    mkdir  "$ktest_out/out"
-    for t in $ktest_tests; do
-	t=$(echo "$t"|tr / .)
-
-	mkdir -p $ktest_out/out/$ktest_basename.$t
-	echo "========= NOT STARTED" > $ktest_out/out/$ktest_basename.$t/status
-    done
 }