Restart VM if necessary until tests are complete

Signed-off-by: Kent Overstreet <kent.overstreet@gmail.com>

5 files changed, 109 insertions, 61 deletions

diff --git a/ci/_test-git-branch.sh b/ci/_test-git-branch.sh
index 979940f..14c609c 100644
--- a/ci/_test-git-branch.sh
+++ b/ci/_test-git-branch.sh
@@ -69,11 +69,7 @@ while true; do
 	echo "TEST FAILED" > "ktest-out/out/$TEST_NAME"
     fi
 
-    for log in $(find ktest-out/out -name log); do
-	brotli --rm -9 "$log"
-    done
-
-    brotli --rm -9 ktest-out/out/$TEST_NAME
+    find ktest-out/out -type f -name \*.log -print0|xargs -0 brotli --rm -9
 
     OUTPUT=$JOBSERVER_OUTPUT_DIR/c/$COMMIT
     ssh $JOBSERVER mkdir -p $OUTPUT
diff --git a/ci/test-job-done.sh b/ci/test-job-done.sh
index 411cbb6..39eeac4 100755
--- a/ci/test-job-done.sh
+++ b/ci/test-job-done.sh
@@ -49,12 +49,17 @@ git_commit_html()
 
     for STATUSFILE in $(find $OUTPUT -name status); do
 	STATUS=$(<$STATUSFILE)
-	DURATION=$(echo $STATUS|grep -Eo '[0-9]+s' || true)
 	TESTNAME=$(basename $(dirname $STATUSFILE))
 	TESTFILE=$(echo $TESTNAME|cut -d. -f1)
 	STATUSMSG=Unknown
 	TABLECLASS=table-secondary
 
+	if [[ -f $TESTNAME/duration ]]; then
+	    DURATION=$(<$TESTNAME/duration)
+	else
+	    DURATION=$(echo $STATUS|grep -Eo '[0-9]+s' || true)
+	fi
+
 	case $STATUS in
 	    *PASSED*)
 		STATUSMSG=Passed
@@ -76,9 +81,9 @@ git_commit_html()
 	echo "<td> $TESTNAME </td>"
 	echo "<td> $STATUSMSG </td>"
 	echo "<td> $DURATION </td>"
-	echo "<td> <a href=$TESTNAME/log.br> log    </a> </td>"
-	echo "<td> <a href=$TESTFILE.br>     full log </a> </td>"
-	echo "<td> <a href=$TESTNAME>        output directory </a> </td>"
+	echo "<td> <a href=$TESTNAME/log.br>	    log			</a> </td>"
+	echo "<td> <a href=$TESTFILE/full_log.br>   full log		</a> </td>"
+	echo "<td> <a href=$TESTNAME>		    output directory	</a> </td>"
 	echo "</tr>"
     done
 
diff --git a/lib/libktest.sh b/lib/libktest.sh
index 25f4468..0e5aae7 100644
--- a/lib/libktest.sh
+++ b/lib/libktest.sh
@@ -17,7 +17,7 @@ ktest_priority=0	# hint for how long test should run
 ktest_interactive=0     # if set to 1, timeout is ignored completely
                         #       sets with: -I
 ktest_exit_on_success=0	# if true, exit on success, not failure or timeout
-ktest_failfast=0
+ktest_failfast=false
 ktest_loop=0
 ktest_verbose=0		# if false, append quiet to kernel commad line
 ktest_crashdump=0
@@ -31,6 +31,7 @@ ktest_storage_bus=virtio-scsi-pci
 
 checkdep socat
 checkdep qemu-system-x86_64	qemu-system-x86
+checkdep brotli
 
 # config files:
 [[ -f $ktest_dir/ktestrc ]]	&& . "$ktest_dir/ktestrc"
@@ -64,7 +65,7 @@ parse_ktest_arg()
 	    ktest_exit_on_success=1
 	    ;;
 	F)
-	    ktest_failfast=1
+	    ktest_failfast=true
 	    ;;
 	L)
 	    ktest_loop=1
@@ -252,28 +253,23 @@ ktest_sysrq()
     echo sendkey alt-sysrq-$key | socat - "UNIX-CONNECT:$ktest_out/vm/mon"
 }
 
+save_env()
+{
+    set |grep -v "^PATH=" > "$ktest_out/vm/env_tmp"
+    readonly_variables="$(readonly | cut -d= -f1 | cut -d' ' -f3)"
+    for variable in ${readonly_variables}
+    do
+	grep -v "${variable}" "$ktest_out/vm/env_tmp" > "$ktest_out/vm/env"
+	cp "$ktest_out/vm/env" "$ktest_out/vm/env_tmp"
+    done
+    sed -i "s/^ ;$//g" "$ktest_out/vm/env"
+    rm -rf "$ktest_out/vm/env_tmp"
+}
+
 start_vm()
 {
     make -C "$ktest_dir/lib" qemu-wrapper
 
-    local qemu_cmd=("$ktest_dir/lib/qemu-wrapper")
-
-    if [[ $ktest_interactive = 1 ]]; then
-	true
-    elif [[ $ktest_exit_on_success = 1 ]]; then
-	qemu_cmd+=(-S)
-    else
-	# Inside the VM, we set a timer and on timeout trigger a crash dump. The
-	# timeout here is a backup:
-	qemu_cmd+=(-S -F -T $ktest_timeout)
-    fi
-
-    local test_basename=$(basename -s .ktest "$ktest_test")
-    qemu_cmd+=(-b "$test_basename")
-    qemu_cmd+=(-o "$ktest_out/out")
-
-    qemu_cmd+=(--)
-
     if [[ -z $ktest_kernel_binary ]]; then
 	echo "Required parameter -k missing: kernel"
 	exit 1
@@ -292,8 +288,6 @@ start_vm()
     # Was outputting to a single log file, now a directory:
     [[ -f $ktest_out/out ]] && rm -f "$ktest_out/out"
 
-    mkdir -p "$ktest_out/out"
-
     rm -f "$ktest_out/core.*"
     rm -f "$ktest_out/vmcore"
     rm -f "$ktest_out/vm"
@@ -311,7 +305,7 @@ start_vm()
 
     kernelargs+=("${ktest_kernel_append[@]}")
 
-    qemu_cmd+=("$QEMU_BIN" -nodefaults -nographic)
+    local qemu_cmd=("$QEMU_BIN" -nodefaults -nographic)
     case $ktest_arch in
 	x86|x86_64)
 	    qemu_cmd+=(-cpu host -machine type=q35,accel=kvm,nvdimm=on)
@@ -420,20 +414,66 @@ start_vm()
 	qemu_pmem mem-path="$file",size=$size
     done
 
-    set |grep -v "^PATH=" > "$ktest_out/vm/env_tmp"
-    readonly_variables="$(readonly | cut -d= -f1 | cut -d' ' -f3)"
-    for variable in ${readonly_variables}
-    do
-        grep -v "${variable}" "$ktest_out/vm/env_tmp" > "$ktest_out/vm/env"
-        cp "$ktest_out/vm/env" "$ktest_out/vm/env_tmp"
-    done
-    sed -i "s/^ ;$//g" "$ktest_out/vm/env"
-    rm -rf "$ktest_out/vm/env_tmp"
-
     set +o errexit
-    set -o pipefail
-    shopt -s lastpipe
 
-    "${qemu_cmd[@]}"
-    exit $?
+    ktest_tests=$(echo $ktest_tests)
+    local ktest_vm_idx=0
+
+    while true; do
+	local full_log=$ktest_basename.$(hostname).$(date -Iseconds).log
+
+	for t in $ktest_tests; do
+	    local fname=$(echo "$t"|tr / .)
+	    ln -sfr "$ktest_out/out/$full_log.br"			\
+		    "$ktest_out/out/$ktest_basename.$fname/full_log.br"
+	done
+
+	local qemu_wrapper_cmd=("$ktest_dir/lib/qemu-wrapper")
+
+	qemu_wrapper_cmd+=(-T $ktest_timeout)
+	qemu_wrapper_cmd+=(-f "$full_log")
+
+	if [[ $ktest_interactive = 1 ]]; then
+	    true
+	elif [[ $ktest_exit_on_success = 1 ]]; then
+	    qemu_wrapper_cmd+=(-S)
+	else
+	    qemu_wrapper_cmd+=(-S -F)
+	fi
+
+	local test_basename=$(basename -s .ktest "$ktest_test")
+	qemu_wrapper_cmd+=(-b "$test_basename")
+	qemu_wrapper_cmd+=(-o "$ktest_out/out")
+
+	save_env
+
+	echo "Starting KVM ($ktest_vm_idx)"
+	"${qemu_wrapper_cmd[@]}" -- "${qemu_cmd[@]}"
+	local ret=$?
+
+	$ktest_failfast && break
+
+	local tests_remaining=
+
+	for t in $ktest_tests; do
+	    local fname=$(echo "$t"|tr / .)
+	    if grep -q "NOT STARTED" $ktest_out/out/$ktest_basename.$fname/status; then
+		tests_remaining="$tests_remaining $t"
+	    fi
+	done
+
+	tests_remaining=$(echo $tests_remaining)
+
+	[[ -z $tests_remaining ]] && break
+
+	if [[ $tests_remaining = $ktest_tests ]]; then
+	    echo "Failed to make progress"
+	    break
+	fi
+
+	ktest_tests="$tests_remaining"
+	let ktest_vm_idx++
+    done
+
+    exit $ret
 }
diff --git a/lib/parse-test.sh b/lib/parse-test.sh
index f4a8ce7..ce8c1dc 100644
--- a/lib/parse-test.sh
+++ b/lib/parse-test.sh
@@ -10,6 +10,7 @@ parse_test_deps()
     ktest_make_install=()
     ktest_kernel_config_require=()
     ktest_qemu_append=()
+    ktest_basename=$(basename -s .ktest "$ktest_test")
 
     local NEXT_SCRATCH_DEV="b"
     local TESTPROG=$1
@@ -226,7 +227,7 @@ parse_test_deps()
 		    # the tests - unless failfast is enabled, or there was only one
 		    # test to run:
 
-		    [[ $ktest_failfast = 1 ]] && break
+		    $ktest_failfast  && break
 		    [[ $# = 1 ]] && break
 
 		    for mnt in $(awk '{print $2}' /proc/mounts|grep ^/mnt|sort -r); do
@@ -277,12 +278,12 @@ parse_test_deps()
     fi
 
     # Mark tests not run:
-    local testname=$(basename -s .ktest "$ktest_test")
-    mkdir -p "$ktest_out/out"
+    rm -rf "$ktest_out/out"
+    mkdir  "$ktest_out/out"
     for t in $ktest_tests; do
 	t=$(echo "$t"|tr / .)
 
-	mkdir -p $ktest_out/out/$testname.$t
-	echo "========= NOT STARTED" > $ktest_out/out/$testname.$t/status
+	mkdir -p $ktest_out/out/$ktest_basename.$t
+	echo "========= NOT STARTED" > $ktest_out/out/$ktest_basename.$t/status
     done
 }
diff --git a/lib/qemu-wrapper.c b/lib/qemu-wrapper.c
index 08d3726..8b6223a 100644
--- a/lib/qemu-wrapper.c
+++ b/lib/qemu-wrapper.c
@@ -53,6 +53,7 @@ static unsigned long	timeout;
 
 static char		*logdir;
 static char		*test_basename;
+static char		*full_log;
 
 static char		*current_test;
 static struct timespec	current_test_start;
@@ -98,7 +99,7 @@ static FILE *test_file_open(const char *fname)
 
 static FILE *log_open()
 {
-	char *path = mprintf("%s/%s", logdir, test_basename);
+	char *path = mprintf("%s/%s", logdir, full_log);
 	FILE *f = fopen(path, "w");
 	if (!f)
 		die("error opening %s: %m", path);
@@ -195,11 +196,15 @@ static void update_watchdog(const char *line)
 	}
 }
 
-static void write_test_status(const char *status)
+static void write_test_file(const char *file, const char *fmt, ...)
 {
-	FILE *f = test_file_open("status");
+	va_list args;
+	FILE *f = test_file_open(file);
+
+	va_start(args, fmt);
+	vfprintf(f, fmt, args);
+	va_end(args);
 
-	fputs(status, f);
 	fclose(f);
 }
 
@@ -210,14 +215,12 @@ static void test_start(char *new_test, struct timespec now)
 	current_test_start	= now;
 	current_test_log	= test_file_open("log");
 
-	write_test_status("TEST FAILED");
+	write_test_file("status", "TEST FAILED");
 }
 
 static void test_end(struct timespec now)
 {
-	FILE *duration = test_file_open("duration");
-	fprintf(duration, "%li", now.tv_sec - current_test_start.tv_sec);
-	fclose(duration);
+	write_test_file("duration", "%li", now.tv_sec - current_test_start.tv_sec);
 
 	fclose(current_test_log);
 	current_test_log = NULL;
@@ -236,7 +239,7 @@ int main(int argc, char *argv[])
 	if (clock_gettime(CLOCK_MONOTONIC, &start))
 		die("clock_gettime error: %m");
 
-	while ((opt = getopt(argc, argv, "SFT:b:o:h")) != -1) {
+	while ((opt = getopt(argc, argv, "SFT:b:o:f:h")) != -1) {
 		switch (opt) {
 		case 'S':
 			exit_on_success = true;
@@ -253,6 +256,9 @@ int main(int argc, char *argv[])
 		case 'b':
 			test_basename = strdup(optarg);
 			break;
+		case 'f':
+			full_log = strdup(optarg);
+			break;
 		case 'o':
 			logdir = strdup(optarg);
 			break;
@@ -310,7 +316,7 @@ again:
 		fputs(output, stdout);
 
 		if (current_test_log && test_is_ending(line)) {
-			write_test_status(line);
+			write_test_file("status", "%s", line);
 			test_end(now);
 		}