use std::collections::HashMap; use std::collections::BTreeSet; use std::io; use std::io::Write; use std::error::Error; use std::process::Command; use clap::{Parser, Subcommand}; use colored::*; use colored::control::ShouldColorize; use nom::number::complete::float; use pager::Pager; use serde_derive::{Serialize, Deserialize}; use toml; const RESULTS: &str = "perf-results.toml"; /* Results for each test, for a particular commit */ type DataPoints = HashMap; struct TestResult { commit: String, commitmsg: String, data_points: DataPoints, } type TestResultsVec = Vec; /* Test results and delta (ratio) from previous commit */ type DataPointsDeltas = HashMap; fn dps_to_dpdeltas(d: &DataPoints) -> DataPointsDeltas { d.iter().map(|x| (x.0.clone(), (*x.1, 0.0)) ).collect() } struct TestResultDeltas { commit: String, commitmsg: String, data_points: DataPointsDeltas, } type TestResultsDeltasVec = Vec; #[derive(Serialize, Deserialize)] struct TestResultsMap { #[serde(flatten)] d: HashMap, } fn results_read(fname: &str) -> Result> { let file_contents = std::fs::read_to_string(fname)?; let r: TestResultsMap = toml::from_str(&file_contents)?; Ok(r) } fn results_write(fname: &str, r: &TestResultsMap) -> Result<(), Box> { let file_contents = toml::to_string(&r)?; std::fs::write(fname, file_contents)?; Ok(()) } fn commit_subject(commit: &git2::Commit) -> String { let msg = commit.message().unwrap().to_string(); let subject_len = msg.find('\n').unwrap_or(msg.len()); msg[..subject_len].to_string() } fn result_new(commit: &git2::Commit) -> TestResult { TestResult { commit: commit.id().to_string(), commitmsg: commit_subject(commit), data_points: HashMap::new(), } } fn results_new(repo: &git2::Repository, range: &str) -> Result { let mut walk = repo.revwalk().unwrap(); if let Err(e) = walk.push_range(range) { eprintln!("Error walking {}: {}", range, e); return Err(e); } let data_points = walk .filter_map(|i| i.ok()) .filter_map(|i| repo.find_commit(i).ok()) .map(|i| result_new(&i)) .collect(); Ok(data_points) } fn results_merge(r1: &mut TestResultsVec, r2: &TestResultsMap) { for r1e in r1.iter_mut() { let r2e = r2.d.get(&r1e.commit); if let Some(r2e) = r2e { r1e.data_points = r2e.clone(); } } } fn pick_commit_to_test(results: &TestResultsVec, test: &str) -> Option { let i = results.first().unwrap(); if i.data_points.get(test).is_none() { let idx = 0; println!("Testing first commit"); println!("{} {} {:89}", idx, results[idx].commit[..10].yellow(), results[idx].commitmsg); return Some(idx); } let i = results.last().unwrap(); if i.data_points.get(test).is_none() { let idx = results.len() - 1; println!("Testing last commit"); println!("{} {} {:89}", idx, results[idx].commit[..10].yellow(), results[idx].commitmsg); return Some(idx); } let mut last_idx: usize = 0; let mut last_val: f32 = 0.0; let mut gap: Option<(usize, usize)> = None; let mut gap_delta: f32 = 0.0; for (idx, i) in results.iter().enumerate() { let v = i.data_points.get(test); if let Some(v) = v { if last_idx + 1 < idx { let delta = (last_val - v) / v; if delta < gap_delta { gap = Some((last_idx, idx)); gap_delta = delta; } } last_idx = idx; last_val = *v; } } if let Some(gap) = gap { let idx = (gap.0 + gap.1) / 2; println!("Bisecting regression by {:.1}%", gap_delta * 100.0); println!("{} {} {:89} {}", gap.0, results[gap.0].commit[..10].yellow(), results[gap.0].commitmsg, results[gap.0].data_points.get(test).unwrap()); println!("{} {} {:89}", idx, results[idx].commit[..10].yellow(), results[idx].commitmsg); println!("{} {} {:89} {}", gap.1, results[gap.1].commit[..10].yellow(), results[gap.1].commitmsg, results[gap.1].data_points.get(test).unwrap()); } gap.map(|x| (x.0 + x.1) / 2) } fn parse_test_output(output: &str) -> Option { let to_float = float::<_, ()>; for l in output.lines() { if let Some(idx) = l.find("result") { let idx = idx + 7; return Some(to_float(&l[idx..]).unwrap().1) } } None } fn cmd_run(repo: &git2::Repository, range: &str, test: &str) { let mut results = results_new(&repo, range).unwrap(); let mut existing = results_read(RESULTS).unwrap_or(TestResultsMap { d: HashMap::new() }); results_merge(&mut results, &existing); while let Some(idx) = pick_commit_to_test(&results, test) { let output = Command::new("benchmark-git-commit.sh") .arg(test) .arg(&results[idx].commit) .output() .unwrap() .stdout; io::stdout().write_all(&output).unwrap(); let output = std::str::from_utf8(&output).unwrap(); let result = parse_test_output(output); if let Some(result) = result { results[idx].data_points.insert(test.to_string(), result); let commit = results[idx].commit.to_string(); if existing.d.get(&commit).is_none() { existing.d.insert(commit.clone(), HashMap::new()); } existing.d.get_mut(&commit).unwrap().insert(test.to_string(), result); results_write(RESULTS, &existing).unwrap(); } else { eprintln!("Error parsing test output"); break; } } results_write(RESULTS, &existing).unwrap(); } fn log_with_results(repo: &git2::Repository, head: &Option, results: &TestResultsMap) -> Result { let mut walk = repo.revwalk().unwrap(); if let Some(head) = head { let object = repo.revparse_single(head)?; if let Err(e) = walk.push(object.id()) { eprintln!("Error walking {}: {}", head, e); return Err(e); } } else { if let Err(e) = walk.push_head() { eprintln!("Error walking: {}", e); return Err(e); } } let mut v = Vec::new(); let mut nr_empty = 0; let mut last_found_idx = 0; for i in walk .filter_map(|i| i.ok()) .filter_map(|i| repo.find_commit(i).ok()) .map(|i| result_new(&i)) { let r = results.d.get(&i.commit); let mut i = i; if let Some(r) = r { i.data_points = r.clone(); nr_empty = 0; last_found_idx = v.len(); } else { nr_empty += 1; } if nr_empty > 2000 { break; } v.push(i); } v.truncate(last_found_idx + 1); Ok(v) } fn results_to_results_with_deltas(results: TestResultsVec) -> TestResultsDeltasVec { let mut results: TestResultsDeltasVec = results.iter().map(|i| TestResultDeltas { commit: i.commit.clone(), commitmsg: i.commitmsg.clone(), data_points: dps_to_dpdeltas(&i.data_points) }).collect(); let mut last: DataPoints = HashMap::new(); for i in results.iter_mut().rev() { for c in i.data_points.iter_mut() { if let Some(l) = last.get(c.0) { c.1.1 = (c.1.0 - l) / l; } last.insert(c.0.clone(), c.1.0); } }; results } fn cmd_log(repo: &git2::Repository, head: &Option) -> io::Result<()> { /* We use write! to stdout instead of print! to avoid panicing on a broken pipe: */ let mut stdout = io::stdout(); colored::control::set_override(ShouldColorize::from_env().should_colorize()); Pager::with_pager("less -FRX").setup(); let results = results_read(RESULTS); if results.is_err() { eprintln!("No results found"); std::process::exit(1); } let results = results.unwrap(); let log = log_with_results(repo, head, &results).unwrap(); let log = results_to_results_with_deltas(log); let mut columns = BTreeSet::new(); for r in log.iter() { for e in r.data_points.iter() { columns.insert(e.0); } } let term_cols = termsize::get().map(|x| x.cols); let msg_width = if let Some(term_cols) = term_cols { term_cols as usize - 11 - columns.len() * 20 } else { 89 }; for (i, e) in columns.iter().enumerate() { if i != 0 && e.len() >= 20 { writeln!(stdout, "")?; } if i == 0 || e.len() >= 20 { write!(stdout, "{:>1$}", e, msg_width + 11 + (i + 1) * 20)?; } else { write!(stdout, "{:>1$}", e, 20)?; } } writeln!(stdout, "")?; for i in log.iter() { write!(stdout, "{} {:1$.*}", &i.commit[..10].yellow(), msg_width, i.commitmsg)?; for e in columns.iter() { if let Some(v) = i.data_points.get(e.clone()) { let f = format!("{:12} {:+6.1}%", v.0, v.1 * 100.0); let f = if v.1 < 0.0 { f.red() } else { f.normal() }; write!(stdout, "{}", f)?; } else { write!(stdout, "{:20}", "")?; } } writeln!(stdout, "")?; } Ok(()) } #[derive(Parser)] #[command(name = "bisect-perf-regressions")] #[command(author = "Kent Overstreet ")] #[command(version = "0.1")] #[command(about = "Bisect performance regressions and provide results in a git log view")] struct Args { #[command(subcommand)] command: Option, } #[derive(Subcommand)] enum Commands { Run { range: String, test: String, }, Log { head: Option }, } fn main() { let args = Args::parse(); let repo = git2::Repository::open(".").unwrap(); match &args.command { Some(Commands::Log { head } ) => { cmd_log(&repo, head).ok(); }, Some(Commands::Run { range, test} ) => cmd_run(&repo, range, test), None => {} } }