Automatically run `creduce` in `csmith-fuzzing/driver.py`

Reduced test cases FTW \o/

3 files changed, 258 insertions, 37 deletions

diff --git a/csmith-fuzzing/README.md b/csmith-fuzzing/README.md
index 258ab5e7..c7e8eaf9 100644
--- a/csmith-fuzzing/README.md
+++ b/csmith-fuzzing/README.md
@@ -18,13 +18,13 @@ tests in the bindings fail, then we report an issue containing the test case!
 
 ## Prerequisites
 
-Requires `python3`, `csmith` and `bindgen` to be in `$PATH`.
+Requires `python3`, `csmith`, and `creduce` to be in `$PATH`.
 
-Many OS package managers have a `csmith` package:
+Many OS package managers have `csmith` and `creduce` packages:
 
 ```
-$ sudo apt install csmith
-$ brew install csmith
+$ sudo apt install csmith creduce
+$ brew install csmith creduce
 $ # Etc...
 ```
 
diff --git a/csmith-fuzzing/driver.py b/csmith-fuzzing/driver.py
index 7dc6086c..8bf76e0b 100755
--- a/csmith-fuzzing/driver.py
+++ b/csmith-fuzzing/driver.py
@@ -1,32 +1,44 @@
 #!/usr/bin/env python3
 
-import os, sys
+import argparse
+import os
+import re
+import shlex
+import sys
 from subprocess import run, SubprocessError, DEVNULL, PIPE
 from tempfile import NamedTemporaryFile
 
-csmith_command = [
-    "csmith",
-    "--no-checksum",
-    "--nomain",
-    "--max-block-size", "1",
-    "--max-block-depth", "1",
-]
+DESC = """
 
-def cat(path, title=None):
-    if not title:
-        title = path
-    print("-------------------- {} --------------------".format(title))
-    run(["cat", path])
+A `csmith` fuzzing driver for `bindgen`.
 
-def run_logged(cmd):
-    with NamedTemporaryFile() as stdout, NamedTemporaryFile() as stderr:
-        result = run(cmd, stdin=DEVNULL, stdout=stdout, stderr=stderr)
-        if result.returncode != 0:
-            print()
-            print("Error: '{}' exited with code {}".format(" ".join(cmd), result.returncode))
-            cat(stdout.name, title="stdout")
-            cat(stdout.name, title="stderr")
-        return result
+Generates random C source files with `csmith` and then passes them to `bindgen`
+(via `predicate.py`). If `bindgen` can't emit bindings, `rustc` can't compile
+those bindings, or the compiled bindings' layout tests fail, then the driver has
+found a bug, and will report the problematic test case to you.
+
+"""
+
+parser = argparse.ArgumentParser(
+    formatter_class=argparse.RawDescriptionHelpFormatter,
+    description=DESC.strip())
+
+parser.add_argument(
+    "--keep-going",
+    action="store_true",
+    help="Do not stop after finding a test case that exhibits a bug in `bindgen`. Instead, keep going.")
+
+CSMITH_ARGS="\
+--no-checksum \
+--nomain \
+--max-block-size 1 \
+--max-block-depth 1"
+
+parser.add_argument(
+    "--csmith-args",
+    type=str,
+    default=CSMITH_ARGS,
+    help="Pass this argument string to `csmith`. By default, very small functions are generated.")
 
 BINDGEN_ARGS = "--with-derive-partialeq \
 --with-derive-eq \
@@ -35,31 +47,234 @@ BINDGEN_ARGS = "--with-derive-partialeq \
 --with-derive-hash \
 --with-derive-default"
 
+parser.add_argument(
+    "--bindgen-args",
+    type=str,
+    default=BINDGEN_ARGS,
+    help="Pass this argument string to `bindgen`. By default, all traits are derived.")
+
+parser.add_argument(
+    "--no-creduce",
+    action="store_false",
+    dest="creduce",
+    help="Do not run `creduce` on any buggy test case(s) discovered.")
+
+################################################################################
+
+def cat(path, title=None):
+    if not title:
+        title = path
+    print("-------------------- {} --------------------".format(title))
+    print()
+    print()
+    run(["cat", path])
+
+def decode(f):
+    return f.decode(encoding="utf-8", errors="ignore")
+
+def run_logged(cmd):
+    result = run(cmd, stdin=DEVNULL, stdout=PIPE, stderr=PIPE)
+    result.stdout = decode(result.stdout)
+    result.stderr = decode(result.stderr)
+    if result.returncode != 0:
+        print()
+        print()
+        print("Error: {} exited with code {}".format(cmd, result.returncode))
+        print()
+        print()
+        for line in result.stdout.splitlines():
+            sys.stdout.write("+")
+            sys.stdout.write(line)
+            sys.stdout.write("\n")
+        for line in result.stderr.splitlines():
+            sys.stderr.write("+")
+            sys.stderr.write(line)
+            sys.stderr.write("\n")
+    return result
+
 def main():
-    print("Fuzzing `bindgen` with C-Smith...\n")
+    os.environ["RUST_BACKTRACE"] = "full"
+    args = parser.parse_args()
+
+    bindgen_args = args.bindgen_args
+    if bindgen_args.find(" -- ") == -1:
+        bindgen_args = bindgen_args + " -- "
+    bindgen_args = bindgen_args + " -I{}".format(os.path.abspath(os.path.dirname(sys.argv[0])))
+    args.bindgen_args = bindgen_args
+
+    print()
+    print()
+    print("Fuzzing `bindgen` with C-Smith...")
+    print()
+    print()
 
     iterations = 0
     while True:
         print("\rIteration: {}".format(iterations), end="", flush=True)
+        iterations += 1
 
         input = NamedTemporaryFile(delete=False, prefix="input-", suffix=".h")
         input.close()
-        result = run_logged(csmith_command + ["-o", input.name])
+        result = run_logged(["csmith", "-o", input.name] + shlex.split(args.csmith_args))
         if result.returncode != 0:
             exit(1)
 
-        result = run_logged([
+        predicate_command = [
             "./predicate.py",
             "--bindgen-args",
-            "{} -- -I{}".format(BINDGEN_ARGS, os.path.abspath(os.path.dirname(sys.argv[0]))),
+            args.bindgen_args,
             input.name
-        ])
+        ]
+        result = run_logged(predicate_command)
+
         if result.returncode != 0:
-            cat(input.name)
+            print()
+            print()
+            cat(input.name, title="Failing test case: {}".format(input.name))
+            print()
+            print()
+
+            if args.creduce:
+                creduce(args, input.name, result)
+
+            print_issue_template(args, input.name, predicate_command, result)
+
+            if args.keep_going:
+                continue
             exit(1)
 
         os.remove(input.name)
-        iterations += 1
+
+RUSTC_ERROR_REGEX = re.compile(r".*(error\[.*].*)")
+LAYOUT_TEST_FAILURE = re.compile(r".*(test bindgen_test_layout_.* \.\.\. FAILED)")
+
+def creduce(args, failing_test_case, result):
+    print()
+    print()
+    print("Reducing failing test case with `creduce`...")
+
+    match = re.search(RUSTC_ERROR_REGEX, result.stderr)
+    if match:
+        error_msg = match.group(1)
+        print("...searching for \"{}\".".format(error_msg))
+        return creduce_with_predicate_flags(
+            args,
+            failing_test_case,
+            "--bindgen-args '{}' --expect-compile-fail --rustc-grep '{}'".format(
+                args.bindgen_args,
+                re.escape(error_msg)
+            )
+        )
+
+    match = re.search(LAYOUT_TEST_FAILURE, result.stdout)
+    if match:
+        layout_failure = match.group(1)
+        struct_name = layout_failure[len("test bindgen_test_layout_"):layout_failure.rindex(" ... FAILED")]
+        print("...searching for \"{}\".".format(layout_failure))
+        return creduce_with_predicate_flags(
+            args,
+            failing_test_case,
+            "--bindgen-args '{}' --expect-layout-tests-fail --bindings-grep '{}' --layout-tests-grep '{}'".format(
+                args.bindgen_args,
+                re.escape(struct_name),
+                re.escape(layout_failure)
+            )
+        )
+
+    print("...nevermind, don't know how to `creduce` this bug. Skipping.")
+
+def creduce_with_predicate_flags(args, failing_test_case, predicate_flags):
+    predicate = """
+#!/usr/bin/env bash
+set -eu
+{} {} {}
+    """.format(
+        os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), "predicate.py")),
+        predicate_flags,
+        os.path.basename(failing_test_case)
+    )
+
+    print("...and reducing with this script:")
+    print()
+    print()
+    print(predicate)
+    print()
+    print()
+
+    predicate_path = failing_test_case + ".predicate.sh"
+    with open(predicate_path, "w") as p:
+        p.write(predicate)
+    os.chmod(predicate_path, 0o755)
+
+    creduce_command = ["creduce", "--n", str(os.cpu_count()), predicate_path, failing_test_case]
+    print("Running:", creduce_command)
+    result = run(creduce_command)
+    if result.returncode == 0:
+        print()
+        print()
+        print("`creduce` reduced the failing test case to:")
+        print()
+        print()
+        cat(failing_test_case)
+        print()
+        print()
+    else:
+        print()
+        print()
+        print("`creduce` failed!")
+        if not args.keep_going:
+            sys.exit(1)
+
+def print_issue_template(args, failing_test_case, predicate_command, result):
+    test_case_contents = None
+    with open(failing_test_case, "r") as f:
+        test_case_contents = f.read()
+
+    print("""
+
+! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !
+! File this issue at https://github.com/rust-lang-nursery/rust-bindgen/issues/new !
+! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! ! !
+
+   --------------- 8< --------------- 8< --------------- 8< ---------------
+
+This bug was found with `csmith` and `driver.py`.
+
+### Input Header
+
+```c
+{}
+```
+
+### `bindgen` Invocation
+
+```
+$ {}
+```
+
+### Actual Results
+
+<details>
+
+```
+{}
+```
+
+</details>
+
+### Expected Results
+
+`bindgen` emits bindings OK, then `rustc` compiles those bindings OK, then the
+compiled bindings' layout tests pass OK.
+
+   --------------- 8< --------------- 8< --------------- 8< ---------------
+
+                         <3 <3 <3 Thank you! <3 <3 <3
+    """.format(
+        test_case_contents,
+        " ".join(map(lambda s: "'{}'".format(s), predicate_command)),
+        result.stdout + result.stderr
+    ))
 
 if __name__ == "__main__":
     try:
diff --git a/csmith-fuzzing/predicate.py b/csmith-fuzzing/predicate.py
index 8896cd96..91e4e26d 100755
--- a/csmith-fuzzing/predicate.py
+++ b/csmith-fuzzing/predicate.py
@@ -110,10 +110,16 @@ def exit_1(msg, child=None):
     print(msg)
 
     if child:
-        print("---------- stdout ----------------------------------------------")
-        print(decode(child.stdout))
-        print("---------- stderr ----------------------------------------------")
-        print(decode(child.stderr))
+        stdout = decode(child.stdout)
+        for line in stdout.splitlines():
+            sys.stdout.write("+")
+            sys.stdout.write(line)
+            sys.stdout.write("\n")
+        stderr = decode(child.stderr)
+        for line in stderr.splitlines():
+            sys.stderr.write("+")
+            sys.stderr.write(line)
+            sys.stderr.write("\n")
 
     raise ExitOne()