Read from .capnp files

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

8 files changed, 120 insertions, 18 deletions

diff --git a/ci-web/src/bin/cgi.rs b/ci-web/src/bin/cgi.rs
index b31a98f..aca5769 100644
--- a/ci-web/src/bin/cgi.rs
+++ b/ci-web/src/bin/cgi.rs
@@ -4,7 +4,7 @@ use regex::Regex;
 extern crate cgi;
 extern crate querystring;
 
-use ci_cgi::{Ktestrc, ktestrc_read, TestResultsMap, TestStatus, commitdir_get_results_toml, git_get_commit};
+use ci_cgi::{Ktestrc, ktestrc_read, TestResultsMap, TestStatus, commitdir_get_results, git_get_commit};
 
 const COMMIT_FILTER:    &str = include_str!("../../commit-filter");
 const STYLESHEET:       &str = "bootstrap.min.css";
@@ -28,7 +28,7 @@ struct Ci {
 }
 
 fn commitdir_get_results_filtered(ci: &Ci, commit_id: &String) -> TestResultsMap {
-    let results = commitdir_get_results_toml(&ci.ktestrc, commit_id).unwrap_or(BTreeMap::new());
+    let results = commitdir_get_results(&ci.ktestrc, commit_id).unwrap_or(BTreeMap::new());
 
     filter_results(results, &ci.tests_matching)
 }
diff --git a/ci-web/src/bin/get-test-job.rs b/ci-web/src/bin/get-test-job.rs
index 7155bec..4196ef4 100644
--- a/ci-web/src/bin/get-test-job.rs
+++ b/ci-web/src/bin/get-test-job.rs
@@ -6,7 +6,7 @@ use std::io::ErrorKind;
 use std::path::{Path, PathBuf};
 use std::process;
 use std::time::SystemTime;
-use ci_cgi::{Ktestrc, KtestrcTestGroup, ktestrc_read, git_get_commit, commitdir_get_results_toml};
+use ci_cgi::{Ktestrc, KtestrcTestGroup, ktestrc_read, git_get_commit, commitdir_get_results};
 use die::die;
 use file_lock::{FileLock, FileOptions};
 use memoize::memoize;
@@ -145,7 +145,7 @@ fn branch_get_next_test_job(rc: &Ktestrc, repo: &git2::Repository,
         let commit = commit.id().to_string();
         ret.commit = commit.clone();
 
-        let results = commitdir_get_results_toml(rc, &commit).unwrap_or(BTreeMap::new());
+        let results = commitdir_get_results(rc, &commit).unwrap_or(BTreeMap::new());
 
         for subtest in subtests.iter() {
             let full_subtest_name = subtest_full_name(&test_path, &subtest);
diff --git a/ci-web/src/lib.rs b/ci-web/src/lib.rs
index 5647366..7b93642 100644
--- a/ci-web/src/lib.rs
+++ b/ci-web/src/lib.rs
@@ -140,14 +140,13 @@ fn commitdir_get_results_fs(ktestrc: &Ktestrc, commit_id: &String) -> TestResult
     results
 }
 
-pub fn commitdir_get_results_toml(ktestrc: &Ktestrc, commit_id: &String) -> Result<TestResultsMap, Box<dyn Error>> {
+fn commit_get_results_toml(ktestrc: &Ktestrc, commit_id: &String) -> Result<TestResultsMap, Box<dyn Error>> {
     let toml = read_to_string(ktestrc.output_dir.join(commit_id.to_owned() + ".toml"))?;
     let r: TestResults = toml::from_str(&toml)?;
     Ok(r.d)
 }
 
-fn results_to_toml(ktestrc: &Ktestrc, commit_id: &String, results: &TestResults)
-{
+fn results_to_toml(ktestrc: &Ktestrc, commit_id: &String, results: &TestResults) {
     let file_contents = toml::to_string(&results).unwrap();
 
     let commit_summary_fname = ktestrc.output_dir.join(commit_id.clone() + ".toml");
@@ -206,8 +205,7 @@ fn results_to_capnp(ktestrc: &Ktestrc, commit_id: &String, results_in: &TestResu
     Ok(())
 }
 
-pub fn commit_update_results_from_fs(ktestrc: &Ktestrc, commit_id: &String)
-{
+pub fn commit_update_results_from_fs(ktestrc: &Ktestrc, commit_id: &String) {
     let results = TestResults { d: commitdir_get_results_fs(&ktestrc, commit_id) };
 
     results_to_toml(ktestrc, commit_id, &results);
@@ -215,7 +213,7 @@ pub fn commit_update_results_from_fs(ktestrc: &Ktestrc, commit_id: &String)
         .map_err(|e| eprintln!("error generating capnp: {}", e)).ok();
 }
 
-pub fn commit_get_results_capnp(ktestrc: &Ktestrc, commit_id: &String) -> Result<TestResultsMap, Box<dyn Error>> {
+fn commit_get_results_capnp(ktestrc: &Ktestrc, commit_id: &String) -> Result<TestResultsMap, Box<dyn Error>> {
     let f = std::fs::read(ktestrc.output_dir.join(commit_id.to_owned() + ".capnp"))?;
 
     let message_reader = serialize::read_message_from_flat_slice(&mut &f[..], capnp::message::ReaderOptions::new())?;
@@ -234,3 +232,7 @@ pub fn commit_get_results_capnp(ktestrc: &Ktestrc, commit_id: &String) -> Result
 
     Ok(results)
 }
+
+pub fn commitdir_get_results(ktestrc: &Ktestrc, commit_id: &String) -> Result<TestResultsMap, Box<dyn Error>> {
+    commit_get_results_capnp(ktestrc, commit_id)
+}
diff --git a/tests/bcachefs/ec.ktest b/tests/bcachefs/ec.ktest
index 9914284..514c534 100755
--- a/tests/bcachefs/ec.ktest
+++ b/tests/bcachefs/ec.ktest
@@ -172,7 +172,7 @@ test_ec_gc()
     bcachefs_antagonist
 
     run_quiet "" bcachefs format -f		\
-	--errors=continue			\
+	--errors=panic				\
 	--replicas=2				\
 	"${ktest_scratch_dev[@]}"
 
diff --git a/tests/bcachefs/perf.ktest b/tests/bcachefs/perf.ktest
index c725795..ad8bb0b 100755
--- a/tests/bcachefs/perf.ktest
+++ b/tests/bcachefs/perf.ktest
@@ -13,15 +13,15 @@ require-kernel-config BCACHEFS_TESTS
 require-kernel-config BCACHEFS_DEBUG=n
 require-kernel-config BCACHEFS_DEBUG_TRANSACTIONS=n
 require-kernel-config BCACHEFS_QUOTA=n
-require-kernel-config BCACHEFS_LOCK_TIME_STATS=n
-require-kernel-config BCACHEFS_NO_LATENCY_ACCT
+require-kernel-config BCACHEFS_LOCK_TIME_STATS=y
+require-kernel-config BCACHEFS_NO_LATENCY_ACCT=n
 require-kernel-config LATENCYTOP
 
 if [[ $ktest_arch = x86 ]]; then
     require-kernel-config CRYPTO_CRC32C_INTEL
 fi
 
-btree_node_size=512k
+btree_node_size=64k
 t=12
 
 config-mem	    16G
@@ -90,7 +90,7 @@ test_rand_insert_multi()
     #perf record -a --
     echo rand_insert_multi		$n 1	> p
     #perf report --sort=symbol|head -n60
-    umount /mnt
+    #umount /mnt
 }
 
 test_perf_rand_insert_multi()
diff --git a/tests/bcachefs/single_device.ktest b/tests/bcachefs/single_device.ktest
index a2d550d..550b3c9 100755
--- a/tests/bcachefs/single_device.ktest
+++ b/tests/bcachefs/single_device.ktest
@@ -23,7 +23,7 @@ test_stress_ng()
     stress-ng  -v -t 60 --class filesystem --all 1 || true
     stress-ng  -v -t 60 --class filesystem --all 2 || true
     cd /root
-    umount /mnt
+    #umount /mnt
 
     check_counters ${ktest_scratch_dev[0]}
 }
@@ -133,7 +133,7 @@ test_reflink()
 
     for i in `seq 0 10`; do
 	sleep 0.5
-	cp --reflink /mnt/fiotest /mnt/fiotest2
+	strace cp --reflink /mnt/fiotest /mnt/fiotest2
 	rm /mnt/fiotest2
     done
 
@@ -225,6 +225,8 @@ test_merge_torture_flakey()
 	umount /mnt
 	mdadm -G /dev/md0 -pnone
 
+	bcachefs fsck -ny /dev/md0
+
 	mount -t bcachefs -o fsck /dev/md0 /mnt
 
 	echo seq_delete		$n 1	> /sys/fs/bcachefs/*/perf_test &
@@ -234,6 +236,8 @@ test_merge_torture_flakey()
 
 	umount /mnt
 	mdadm -G /dev/md0 -pnone
+
+	bcachefs fsck -ny /dev/md0
     done
 
     mdadm --stop /dev/md0
@@ -1449,4 +1453,57 @@ test_kill_btree_node()
     check_counters ${ktest_scratch_dev[0]}
 }
 
+test_make_image()
+{
+    set_watchdog 240
+    run_quiet "" bcachefs format --errors=panic --version=11 --no_initialize ${ktest_scratch_dev[0]}
+
+    mount -t bcachefs ${ktest_scratch_dev[0]} /mnt
+    cp -a /usr /mnt
+    umount /mnt
+}
+
+test_fio_crashing()
+{
+cat > /root/fio_config <<EOF
+[global]
+ioengine=libaio
+bs=4k
+directory=/mnt
+filesize=1G
+size=9999T
+continue_on_error=write
+ignore_error=EIO,ENOSPC:EIO
+error_dump=0
+
+[stress_dio_aio_activity]
+create_on_open=1
+fallocate=none
+iodepth=128
+direct=1
+buffered=0
+numjobs=4
+rw=randwrite
+runtime=40
+time_based
+
+[stress_mmap_activity]
+ioengine=mmap
+create_on_open=0
+fallocate=1
+fdatasync=40960
+filesize=8M
+size=9999T
+numjobs=4
+rw=randwrite
+runtime=40
+time_based
+
+EOF
+    run_quiet "" bcachefs format --no_initialize --version=11 -f ${ktest_scratch_dev[0]}
+    mount -t bcachefs ${ktest_scratch_dev[0]} /mnt
+    fio /root/fio_config
+    umount /mnt
+}
+
 main "$@"
diff --git a/tests/bcachefs/subvol.ktest b/tests/bcachefs/subvol.ktest
index 18f734c..da0ff9d 100755
--- a/tests/bcachefs/subvol.ktest
+++ b/tests/bcachefs/subvol.ktest
@@ -511,4 +511,44 @@ test_snapshot_prjquota()
     check_counters ${ktest_scratch_dev[0]}
 }
 
+snapshot_rw_fio()
+{
+    fio --eta=always				\
+	--exitall_on_error=1			\
+	--ioengine=libaio			\
+	--iodepth=64				\
+	--iodepth_batch=16			\
+	--direct=1				\
+	--numjobs=1				\
+	--verify=meta				\
+	--verify_fatal=1			\
+	--filename=$1				\
+	--filesize=1G				\
+	--name=randrw				\
+	--rw=randrw				\
+	--bsrange=4k-1M
+}
+
+test_snapshot_rw()
+{
+    run_quiet "" bcachefs format -f --errors=panic --prjquota ${ktest_scratch_dev[0]}
+    mount -t bcachefs ${ktest_scratch_dev[0]} /mnt
+
+    bcachefs subvolume snapshot /mnt/snap-1
+    bcachefs subvolume snapshot /mnt/snap-2
+
+    snapshot_rw_fio /mnt/fiotest &
+    snapshot_rw_fio /mnt/snap-1/fiotest &
+    snapshot_rw_fio /mnt/snap-2/fiotest &
+    wait
+
+    exit
+    umount /mnt
+
+    mount -t bcachefs -o fsck ${ktest_scratch_dev[0]} /mnt
+    umount /mnt
+
+    check_counters ${ktest_scratch_dev[0]}
+}
+
 main "$@"
diff --git a/tests/memory_alloc.ktest b/tests/memory_alloc.ktest
index 80b5c7d..ab2cf5b 100755
--- a/tests/memory_alloc.ktest
+++ b/tests/memory_alloc.ktest
@@ -4,7 +4,10 @@
 
 config-timeout $(stress_timeout)
 
-#require-kernel-append nomem_profiling
+require-kernel-config MEM_ALLOC_PROFILING
+require-kernel-config MEM_ALLOC_PROFILING_DEBUG=n
+
+#require-kernel-append sysctl.vm.mem_profiling=0
 
 test_bench()
 {