path: root/tests/bcachefs/replication.ktest

#!/bin/bash

require-lib bcachefs-test-libs.sh

require-kernel-config MD
require-kernel-config BLK_DEV_MD
require-kernel-config MD_FAULTY

config-scratch-devs 4G
config-scratch-devs 4G
config-scratch-devs 4G
config-scratch-devs 4G
config-scratch-devs 4G

config-mem 4G

config-timeout $(stress_timeout)

test_recovery()
{
    set_watchdog 180

    run_quiet "" bcachefs format		\
	--discard				\
	--bucket=64k				\
	--block_size=4k				\
	/dev/sd[bcd]

    for i in `seq 0 20`; do
	mount -t bcachefs /dev/sdb:/dev/sdc:/dev/sdd /mnt

	#run_fio_randrw
	fio --eta=always			\
	    --exitall_on_error=1		\
	    --randrepeat=0			\
	    --ioengine=libaio			\
	    --iodepth=64			\
	    --iodepth_batch=16			\
	    --direct=1				\
	    --numjobs=1				\
	    --filename=/mnt/fiotest		\
	    --filesize=3500M			\
	    --name=randwrite			\
	    --rw=randwrite			\
	    --bsrange=4k-8k			\
	    --runtime=2

	$ktest_dir/tests/xfstests/src/godown -f /mnt

	umount /mnt
    done
}

test_multimount()
{
    set_watchdog 10
    bcachefs format				\
	/dev/sdb

    mkdir -p /mnt/1 /mnt/2
    mount -t bcachefs /dev/sdb /mnt/1
    mount -t bcachefs /dev/sdb /mnt/2

    umount /mnt/1
    umount /mnt/2
}

test_twodevices()
{
    run_basic_fio_test				\
	/dev/sdb /dev/sdc /dev/sdd
}

test_largebuckets()
{
    set_watchdog 30
    run_quiet "" bcachefs format -f --bucket=8M /dev/sd[bcd]

    mount -t bcachefs /dev/sdb:/dev/sdc:/dev/sdd /mnt
    dd if=/dev/zero of=/mnt/foo bs=8M count=1024 oflag=direct
    umount /mnt
}

test_replicas()
{
    run_basic_fio_test				\
	--replicas=2				\
	/dev/sd[bcd]
}

test_replicas_variable_buckets()
{
    set_watchdog 60
    run_quiet "" bcachefs format -f			\
	--replicas=2					\
	--bucket=32k	/dev/sdb			\
	--bucket=64k	/dev/sdc			\
	--bucket=128k	/dev/sdd			\
	--bucket=128k	/dev/sde			\
	--bucket=128k	/dev/sdf

    mount -t bcachefs /dev/sdb:/dev/sdc:/dev/sdd:/dev/sde:/dev/sdf /mnt
    fs_mark -v -n 20000 -s 4096 -k -S 0 -D 1000 -N 100000 -d /mnt -t 60
    df -h /mnt
    umount /mnt
}

d_test_copygc_torture()
{
    loops=$((($ktest_priority + 1) * 4))

    bcachefs_antagonist
    set_watchdog 600

    run_quiet "" bcachefs format -f		\
	"$@" /dev/sd[bcd]

    mount -t bcachefs /dev/sdb:/dev/sdc:/dev/sdd /mnt
    #enable_memory_faults

    fio						\
	--filename=/mnt/fiotest			\
	--ioengine=sync				\
	--name=write				\
	--rw=write				\
	--bs=16M				\
	--fill_fs=1
    echo 3 > /proc/sys/vm/drop_caches

    for i in $(seq 1 $loops); do
	run_fio_base				\
	    --buffer_compress_percentage=50	\
						\
	    --name=randwrite			\
	    --stonewall				\
	    --rw=randwrite			\
	    --loops=$loops			\
	    --bsrange=4k-4k

	cur_size=$(stat -c '%s' /mnt/fiotest)
	new_size=$(($cur_size - 1048576))
	echo "truncating to $new_size"

	truncate --size=$new_size /mnt/fiotest
    done

    rm /mnt/fiotest

    expect_sysfs cache dirty_buckets	0
    expect_sysfs cache dirty_data	0
    expect_sysfs cache cached_buckets	0
    expect_sysfs cache cached_data	0

    #disable_memory_faults
    umount /mnt
}

do_remove_test()
{
    set_watchdog 60
    offline=$1
    evacuate=$2

    run_quiet "" bcachefs format		\
	--replicas=2				\
	/dev/sd[bcd]

    mount -t bcachefs /dev/sdb:/dev/sdc:/dev/sdd /mnt

    local fioout="$ktest_out/fio-out"
    run_fio_randrw >"$fioout" 2>&1 &
    local fiopid=$!

    sleep 1

    if [[ $offline = 1 ]]; then
	echo -n "offlining /dev/sdb... "
	bcachefs device offline --force	/dev/sdb
	echo "done"
    fi

    if [[ $evacuate = 1 ]]; then
	echo -n "evacuating /dev/sdb... "
	bcachefs device evacuate	/dev/sdb
	echo "done"
    fi

    bcachefs fs usage -h /mnt

    echo -n "removing /dev/sdb... "
    if [[ $evacuate = 1 ]]; then
	#bcachefs device remove		/dev/sdb
	bcachefs device remove		0 /mnt
    else
	#bcachefs device remove --force	/dev/sdb
	bcachefs device remove --force	0 /mnt
    fi
    echo "done"

    if ! wait $fiopid; then
	cat "$fioout"
	return 1
    fi

    bcachefs fs usage -h /mnt
    umount /mnt

    mount -t bcachefs /dev/sdb:/dev/sdc:/dev/sdd /mnt
    umount /mnt
}

test_device_remove_offline()
{
    do_remove_test 1 0
}

test_device_remove_online()
{
    do_remove_test 0 0
}

test_device_evacuate_offline()
{
    do_remove_test 1 1
}

test_device_evacuate_online()
{
    do_remove_test 0 1
}

test_device_add()
{
    set_watchdog 40
    #setup_tracing 'bcachefs:*'

    run_quiet "" bcachefs format		\
	--block_size=4k				\
	--no_initialize				\
	/dev/sdb

    mount /dev/sdb /mnt

    run_fio_randrw >"$ktest_out/fio-out" 2>&1 &
    local fiopid=$!

    sleep 1

    echo -n "adding /dev/sdc... "
    bcachefs device add /mnt /dev/sdc
    echo "done"

    if ! wait $fiopid; then
	cat "$fioout"
	return 1
    fi

    umount /mnt
    bcachefs fsck -n /dev/sdb /dev/sdc

    mount -t bcachefs /dev/sdb:/dev/sdc /mnt
    umount /mnt
}

test_device_set_state()
{
    set_watchdog 30

    run_quiet "" bcachefs format		\
	--replicas=2				\
	/dev/sd[bcd]

    mount -t bcachefs /dev/sdb:/dev/sdc:/dev/sdd /mnt

    run_fio_randrw >"$ktest_out/fio-out" 2>&1 &
    local fiopid=$!

    sleep 1

    for state in ro failed spare; do
	echo "setting state $state"
	bcachefs device set-state --force $state    /dev/sdc

	echo "setting state rw"
	#bcachefs device set-state	  rw /dev/sdc
	bcachefs device set-state	  rw 1 /mnt
    done

    if ! wait $fiopid; then
	cat "$fioout"
	return 1
    fi

    umount /mnt
}

test_device_ro()
{
    set_watchdog 30

    run_quiet "" bcachefs format		\
	--btree_node_size=128k			\
	/dev/sd[bcd]

    mount -t bcachefs /dev/sdb:/dev/sdc:/dev/sdd /mnt

    local fioout="$ktest_out/fio-out"
    run_fio_randrw >"$fioout" 2>&1 &
    local fiopid=$!

    sleep 4

    bcachefs device set-state ro /dev/sdc

    if ! wait $fiopid; then
	cat "$fioout"
	return 1
    fi

    umount /mnt

    mount -t bcachefs /dev/sdb:/dev/sdc:/dev/sdd /mnt
    umount /mnt
}

test_device_set_state_offline()
{
    set_watchdog 60
    run_quiet "" bcachefs format		\
	--replicas=2				\
	/dev/sd[bcd]

    mount -t bcachefs /dev/sdb:/dev/sdc:/dev/sdd /mnt

    local fioout="$ktest_out/fio-out"
    run_fio_randrw >"$fioout" 2>&1 &
    local fiopid=$!

    sleep 1

    bcachefs device set-state --force	    ro /dev/sdc

    if ! wait $fiopid; then
	cat "$fioout"
	return 1
    fi

    umount /mnt

    bcachefs device set-state --offline	    rw /dev/sdc

    mount -t bcachefs /dev/sdb:/dev/sdc:/dev/sdd /mnt
    cat /sys/block/sdc/bcachefs/state
    umount /mnt
}

test_device_readd()
{
    set_watchdog 40
    run_quiet "" bcachefs format		\
	--replicas=2				\
	/dev/sd[bc]

    mount -t bcachefs /dev/sdb:/dev/sdc /mnt

    local fioout="$ktest_out/fio-out"
    run_fio_randrw >"$fioout" 2>&1 &
    local fiopid=$!

    sleep 1

    echo -n "offlining /dev/sdb... "
    bcachefs device offline --force	/dev/sdb
    echo "done"

    echo -n "onlining /dev/sdb... "
    bcachefs device online /dev/sdb
    echo "done"

    if ! wait $fiopid; then
	cat "$fioout"
	return 1
    fi

    umount /mnt

    mount -t bcachefs /dev/sdb:/dev/sdc /mnt
    umount /mnt
}

do_replicas_errors_test()
{
    set_watchdog 180
    error_type=$1

    echo 1 > /sys/module/bcachefs/parameters/verify_all_btree_replicas
    #bcachefs_antagonist

    mdadm -B /dev/md0 -lfaulty -n1 /dev/sdc

    run_quiet "" bcachefs format		\
	--replicas=2				\
	/dev/md0 /dev/sdb

    mount -t bcachefs -o degraded /dev/sdb:/dev/md0 /mnt

    mdadm -G /dev/md0 -p$error_type

    run_fio_randrw

    # Check the read retry path for indirect extents:
    cp --reflink /mnt/fiotest /mnt/fiotest.reflinked
    dd if=/mnt/fiotest of=/dev/null bs=4k iflag=direct

    umount /mnt
    mdadm --stop /dev/md0

    #bcachefs fsck -n /dev/sdb /dev/sdc

    mount -t bcachefs -o fsck /dev/sdb:/dev/sdc /mnt
    umount /mnt
}

test_replicas_write_errors()
{
    do_replicas_errors_test write-all
}

test_replicas_read_errors()
{
    do_replicas_errors_test read-persistent1
}

test_cmd_fs_usage()
{
    set_watchdog 20

    run_quiet "" bcachefs format		\
	--replicas=2				\
	--bucket=128k				\
	/dev/sd[bcd]

    echo -n "mounting... "
    mount -t bcachefs /dev/sdb:/dev/sdc:/dev/sdd /mnt
    echo done

    echo "Before writing:"
    bcachefs fs usage  /mnt
    echo
    df -h /mnt
    echo

    run_fio_randrw --size=500M > /dev/null

    echo
    echo "After writing:"
    bcachefs fs usage -h /mnt
    echo
    df -h /mnt

    umount /mnt
}

test_rereplicate()
{
    set_watchdog 60
    run_quiet "" bcachefs format		\
	/dev/sd[bcd]
    mount -t bcachefs /dev/sdb:/dev/sdc:/dev/sdd /mnt

    dd if=/dev/zero of=/mnt/test bs=1M count=1024 oflag=direct

    umount /mnt
    mount -t bcachefs -o metadata_replicas=2,data_replicas=2 /dev/sdb:/dev/sdc:/dev/sdd /mnt

    #echo 2 > /sys/fs/bcachefs/*/options/data_replicas
    #echo 2 > /sys/fs/bcachefs/*/options/metadata_replicas

    bcachefs data rereplicate /mnt

    bcachefs fs usage -h /mnt
    umount /mnt

    mount -t bcachefs -o degraded /dev/sdb:/dev/sdc /mnt
    umount /mnt
}

disabled_test_device_add_faults()
{
    setup_tracing 'bcachefs:*'

    run_quiet "" bcachefs format		\
	--errors=remount-ro			\
	/dev/sdb
    mount /dev/sdb /mnt

    device_add_fault_fn()
    {
	set_faults "$1 enable"
	! bcachefs device add /mnt /dev/sdc
    }

    run_fault_injection_test bcachefs:add			device_add_fault_fn
    run_fault_injection_test bcachefs:bch_fs_init:dev_alloc	device_add_fault_fn
    run_fault_injection_test bcachefs:bch_fs_init:alloc_start	device_add_fault_fn
    run_fault_injection_test bcachefs:bch_fs_init:read_super	device_add_fault_fn
}

test_striping()
{
    set_watchdog 60
    setup_tracing 'bcachefs:*'

    if false; then
	run_quiet "" bcachefs format		\
	    --replicas=2			\
	    /dev/sd[bc]

	mount -t bcachefs /dev/sdb:/dev/sdc /mnt

	dd if=/dev/zero of=/mnt/test0 bs=1M count=1024 oflag=direct
	bcachefs fs usage -h /mnt

	bcachefs device add /mnt /dev/sdd

	dd if=/dev/zero of=/mnt/test1 bs=1M count=1024 oflag=direct
    else
	run_quiet "" bcachefs format		\
	    --data_checksum=none		\
	    --replicas=2			\
	    --label cache			\
	    --durability=1			\
	    --bucket=8M /dev/sd[b]		\
	    --label hdd				\
	    --durability=2			\
	    --bucket=8M /dev/sd[cd]		\
	    --foreground_target=cache		\
	    --background_target=hdd		\
	    --promote_target=cache

	mount -t bcachefs /dev/sdb:/dev/sdc:/dev/sdd /mnt

	dd if=/dev/zero of=/mnt/test0 bs=1M count=1024 oflag=direct
    fi

    bcachefs fs usage -h /mnt

    umount /mnt
}

disabled_change_replicas()
{
    run_quiet "" bcachefs format		\
	--replicas=2				\
	/dev/sd[bc]

    mount -t bcachefs /dev/sdb:/dev/sdc /mnt

    (
	cd /sys/fs/bcachefs/*
	while true; do
	    echo 1 > options/data_replicas
	    sleep 0.1
	    echo 2 > options/data_replicas
	    sleep 0.1
	done
    ) &

    run_fio_randrw

    while true; do
	dd if=/dev/zero of=/mnt/foo bs=1M count=1024 oflag=direct
	sync
	#rm /mnt/foo
    done

    kill %1
    umount /mnt
}

disabled_change_replicas2()
{
    run_quiet "" bcachefs format		\
	--replicas=2				\
	/dev/sd[bc]

    mount -t bcachefs /dev/sdb:/dev/sdc /mnt

    local opt=/sys/fs/bcachefs/*/options/data_replicas
    while true; do
	echo 1 > $opt
	dd if=/dev/zero of=/mnt/foo bs=1M count=1024
	echo 2 > $opt
	sync
	#rm /mnt/foo
    done

    umount /mnt
}

test_replicas_gc()
{
    set_watchdog 60

    echo 1 > /sys/module/bcachefs/parameters/force_reconstruct_read
    #echo 1 > /sys/module/bcachefs/parameters/debug_check_bkeys

    run_quiet "" bcachefs format -f		\
	--replicas=2				\
	/dev/sd[bcdef]
    devs=/dev/sdb:/dev/sdc:/dev/sdd:/dev/sde:/dev/sdf

    mount -t bcachefs $devs /mnt

    antagonist_trigger_gc &
    run_fio_randrw

    echo 1 > /sys/fs/bcachefs/*/internal/trigger_gc
    umount /mnt

    # test remount:
    mount -t bcachefs -o fix_errors,erasure_code $devs /mnt
    umount /mnt

    bcachefs fsck /dev/sd[bcdef]
}

test_replace_replica()
{
    set_watchdog 60

    bcachefs format --replicas=2 /dev/sd[bc]
    mount -t bcachefs -o degraded /dev/sdb /mnt

    bcachefs device add -f /mnt /dev/sdd

    bcachefs device set-state --force failed /dev/sdc

    bcachefs data rereplicate /mnt

    umount /mnt

    mount -t bcachefs -o degraded /dev/sdd /mnt
    umount /mnt
}

test_key_type_error()
{
    set_watchdog 20

    bcachefs format --replicas=2 /dev/sd[bc]

    mount -t bcachefs /dev/sdb:/dev/sdc /mnt
    bcachefs setattr --data_replicas=1 /mnt
    bcachefs setattr --foreground_target=sdc /mnt

    dd if=/dev/zero of=/mnt/foo bs=64k count=8 oflag=direct

    umount /mnt
    mount -t bcachefs -o very_degraded /dev/sdb /mnt

    bcachefs device remove --force /dev/sdc

    # should fail:
    ! dd if=/mnt/foo of=/dev/null bs=64k count=1 iflag=direct

    umount /mnt
}