New tool for generating logs of performance data

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

3 files changed, 304 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..ea8c4bf
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1 @@
+/target
diff --git a/Cargo.toml b/Cargo.toml
new file mode 100644
index 0000000..1898e7e
--- /dev/null
+++ b/Cargo.toml
@@ -0,0 +1,13 @@
+[package]
+name = "bisect-perf-regressions"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+git2 = "0.15.0"
+nom = "7.1.2"
+serde = "1.0.152"
+serde_derive = "1.0.152"
+toml = "0.5.10"
diff --git a/src/main.rs b/src/main.rs
new file mode 100644
index 0000000..24a478f
--- /dev/null
+++ b/src/main.rs
@@ -0,0 +1,290 @@
+use std::collections::HashMap;
+use std::collections::HashSet;
+use std::env;
+use std::io;
+use std::io::Write;
+use std::error::Error;
+use std::process::Command;
+use nom::number::complete::float;
+use serde_derive::{Serialize, Deserialize};
+use toml;
+
+const RESULTS:  &str = "perf-results.toml";
+
+type DataPoints = HashMap<String, f32>;
+
+struct TestResult {
+    commit:             String,
+    commitmsg:          String,
+    data_points:        DataPoints,
+}
+
+type TestResultsVec = Vec<TestResult>;
+
+#[derive(Serialize, Deserialize)]
+struct TestResultsMap {
+    d: HashMap<String, DataPoints>,
+}
+
+fn results_read(fname: &str) -> Result<TestResultsMap, Box<dyn Error>> {
+    let file_contents = std::fs::read_to_string(fname)?;
+    let r: TestResultsMap = toml::from_str(&file_contents)?;
+
+    Ok(r)
+}
+
+fn results_write(fname: &str, r: &TestResultsVec) -> Result<(), Box<dyn Error>> {
+    let r = results_mapped(&r);
+    let file_contents = toml::to_string(&r)?;
+    std::fs::write(fname, file_contents)?;
+
+    Ok(())
+}
+
+fn commit_subject(commit: &git2::Commit) -> String {
+    let msg = commit.message().unwrap().to_string();
+    let subject_len = msg.find('\n').unwrap_or(msg.len());
+
+    msg[..subject_len].to_string()
+}
+
+fn result_new(commit: &git2::Commit) -> TestResult {
+    TestResult {
+        commit:         commit.id().to_string(),
+        commitmsg:      commit_subject(commit),
+        data_points:    HashMap::new(),
+    }
+}
+
+fn results_new(repo: &git2::Repository, range: &str) -> Result<TestResultsVec, git2::Error> {
+    let mut walk = repo.revwalk().unwrap();
+
+    if let Err(e) = walk.push_range(range) {
+        eprintln!("Error walking {}: {}", range, e);
+        return Err(e);
+    }
+
+    let data_points = walk
+        .filter_map(|i| i.ok())
+        .filter_map(|i| repo.find_commit(i).ok())
+        .map(|i| result_new(&i))
+        .collect();
+
+    Ok(data_points)
+}
+
+fn results_insert(results: &mut TestResultsVec, commit: &str, test: &str, v: f32) {
+    for i in results.iter_mut() {
+        if i.commit == commit {
+            i.data_points.insert(test.to_string(), v);
+            return;
+        }
+    }
+
+    panic!("commit {} not found", commit);
+}
+
+fn results_mapped(r: &TestResultsVec) -> TestResultsMap {
+    let mut mapped = HashMap::new();
+
+    for i in r {
+        if i.data_points.len() != 0 {
+            mapped.insert(i.commit.clone(), i.data_points.clone());
+        }
+    }
+
+    TestResultsMap { d: mapped }
+}
+
+fn results_merge(r1: &mut TestResultsVec, r2: &TestResultsMap) {
+    for r1e in r1.iter_mut() {
+        let r2e = r2.d.get(&r1e.commit);
+
+        if let Some(r2e) = r2e {
+            r1e.data_points = r2e.clone();
+        }
+    }
+}
+
+fn pick_commit_to_test(results: &TestResultsVec, test: &str) -> Option<String> {
+    let i = results.first().unwrap();
+    if i.data_points.get(test).is_none() {
+        return Some(i.commit.clone());
+    }
+
+    let i = results.last().unwrap();
+    if i.data_points.get(test).is_none() {
+        return Some(i.commit.clone());
+    }
+
+    let mut idx:            i32 = 0;
+    let mut last_idx:       i32 = -1;
+    let mut last_val:       f32 = 0.0;
+
+    let mut missing_idx:    i32 = 0;
+    let mut missing_delta:  f32 = 0.0;
+
+    for i in results {
+        let v = i.data_points.get(test);
+
+        if let Some(v) = v {
+            if last_idx + 1 != idx {
+                let delta = v / last_val;
+
+                if delta > missing_delta {
+                    missing_idx     = (idx + last_idx) / 2;
+                    missing_delta   = delta;
+                }
+            }
+
+            last_idx = idx;
+            last_val = *v;
+        } else {
+        }
+
+        idx += 1;
+    }
+
+    if missing_idx != 0 {
+        Some(results[missing_idx as usize].commit.clone())
+    } else {
+        None
+    }
+}
+
+fn parse_test_output(output: &str) -> Option<f32> {
+    let to_float = float::<_, ()>;
+
+    for l in output.lines() {
+        if let Some(idx) = l.find("result") {
+            let idx = idx + 7;
+            return Some(to_float(&l[idx..]).unwrap().1)
+        }
+    }
+
+    None
+}
+
+fn run_tests(repo: &git2::Repository, range: &str, test: &str) {
+    let mut results = results_new(&repo, range).unwrap();
+
+    let existing = results_read(RESULTS);
+    if let Ok(existing) = existing {
+        results_merge(&mut results, &existing);
+    }
+
+    while let Some(commit) = pick_commit_to_test(&results, test) {
+        let output = Command::new("benchmark-git-commit.sh")
+            .arg(test)
+            .arg(&commit)
+            .output()
+            .unwrap()
+            .stdout;
+
+        io::stdout().write_all(&output).unwrap();
+
+        let output = std::str::from_utf8(&output).unwrap();
+        let result = parse_test_output(output);
+
+        if let Some(result) = result {
+            results_insert(&mut results, &commit, test, result);
+
+            results_write(RESULTS, &results).unwrap();
+        } else {
+            eprintln!("Error parsing test output");
+            break;
+        }
+    }
+
+    results_write(RESULTS, &results).unwrap();
+}
+
+fn log_with_results(repo: &git2::Repository,
+                    results: &TestResultsMap) -> Result<TestResultsVec, git2::Error> {
+    let mut walk = repo.revwalk().unwrap();
+
+    if let Err(e) = walk.push_head() {
+        eprintln!("Error walking: {}", e);
+        return Err(e);
+    }
+
+    let mut v = Vec::new();
+    let mut nr_empty = 0;
+
+    for i in walk
+        .filter_map(|i| i.ok())
+        .filter_map(|i| repo.find_commit(i).ok())
+        .map(|i| result_new(&i)) {
+        let r = results.d.get(&i.commit);
+        let mut i = i;
+
+        if let Some(r) = r {
+            i.data_points = r.clone();
+            nr_empty = 0;
+        } else {
+            nr_empty += 1;
+        }
+
+        if nr_empty > 2000 {
+            break;
+        }
+
+        v.push(i);
+    }
+
+    while let Some(l) = v.last() {
+        if l.data_points.len() != 0 {
+            break;
+        }
+        v.pop();
+    }
+
+    Ok(v)
+}
+
+fn list_tests(repo: &git2::Repository) {
+    let results = results_read(RESULTS).unwrap();
+    let log = log_with_results(repo, &results).unwrap();
+
+    let mut columns = HashSet::new();
+    for r in log.iter() {
+        for e in r.data_points.iter() {
+            columns.insert(e.0);
+        }
+    }
+
+    print!("{:100}", "");
+    for e in columns.iter() {
+        print!("{:20}", e);
+    }
+    println!("");
+
+    for i in log.iter() {
+        print!("{} {:89}", &i.commit[..10], i.commitmsg);
+
+        for e in columns.iter() {
+            if let Some(v) = i.data_points.get(e.clone()) {
+                print!("{:20}", v);
+            } else {
+                print!("{:20}", "");
+            }
+        }
+        println!("");
+    }
+}
+
+fn main() {
+    let args: Vec<String> = env::args().collect();
+    let repo = git2::Repository::open(".").unwrap();
+
+    let subcmd = args.get(1).unwrap();
+
+    if subcmd == "run" {
+        let range   = args.get(2).unwrap();
+        let test    = args.get(3).unwrap();
+
+        run_tests(&repo, range, test);
+    } else if subcmd == "log" {
+        list_tests(&repo);
+    }
+}