path: root/src/main.rs

use std::collections::HashMap;
use std::collections::BTreeSet;
use std::io;
use std::io::Write;
use std::error::Error;
use std::process::Command;
use clap::{Parser, Subcommand};
use colored::*;
use colored::control::ShouldColorize;
use nom::number::complete::float;
use pager::Pager;
use serde_derive::{Serialize, Deserialize};
use toml;

const RESULTS:  &str = "perf-results.toml";

/* Results for each test, for a particular commit */
type DataPoints = HashMap<String, f32>;

struct TestResult {
    commit:             String,
    commitmsg:          String,
    data_points:        DataPoints,
}

type TestResultsVec = Vec<TestResult>;

/* Test results and delta (ratio) from previous commit */
type DataPointsDeltas = HashMap<String, (f32, f32)>;

fn dps_to_dpdeltas(d: &DataPoints) -> DataPointsDeltas {
    d.iter().map(|x| (x.0.clone(), (*x.1, 0.0)) ).collect()
}

struct TestResultDeltas {
    commit:             String,
    commitmsg:          String,
    data_points:        DataPointsDeltas,
}

type TestResultsDeltasVec = Vec<TestResultDeltas>;

#[derive(Serialize, Deserialize)]
struct TestResultsMap {
    #[serde(flatten)]
    d: HashMap<String, DataPoints>,
}

fn results_read(fname: &str) -> Result<TestResultsMap, Box<dyn Error>> {
    let file_contents = std::fs::read_to_string(fname)?;
    let r: TestResultsMap = toml::from_str(&file_contents)?;

    Ok(r)
}

fn results_write(fname: &str, r: &TestResultsMap) -> Result<(), Box<dyn Error>> {
    let file_contents = toml::to_string(&r)?;
    std::fs::write(fname, file_contents)?;

    Ok(())
}

fn commit_subject(commit: &git2::Commit) -> String {
    let msg = commit.message().unwrap().to_string();
    let subject_len = msg.find('\n').unwrap_or(msg.len());

    msg[..subject_len].to_string()
}

fn result_new(commit: &git2::Commit) -> TestResult {
    TestResult {
        commit:         commit.id().to_string(),
        commitmsg:      commit_subject(commit),
        data_points:    HashMap::new(),
    }
}

fn results_new(repo: &git2::Repository, range: &str) -> Result<TestResultsVec, git2::Error> {
    let mut walk = repo.revwalk().unwrap();

    if let Err(e) = walk.push_range(range) {
        eprintln!("Error walking {}: {}", range, e);
        return Err(e);
    }

    let data_points = walk
        .filter_map(|i| i.ok())
        .filter_map(|i| repo.find_commit(i).ok())
        .map(|i| result_new(&i))
        .collect();

    Ok(data_points)
}

fn results_merge(r1: &mut TestResultsVec, r2: &TestResultsMap) {
    for r1e in r1.iter_mut() {
        let r2e = r2.d.get(&r1e.commit);

        if let Some(r2e) = r2e {
            r1e.data_points = r2e.clone();
        }
    }
}

fn pick_commit_to_test(results: &TestResultsVec, test: &str) -> Option<usize> {
    let i = results.first().unwrap();
    if i.data_points.get(test).is_none() {
        let idx = 0;

        println!("Testing first commit");
        println!("{} {} {:89}", idx,
                 results[idx].commit[..10].yellow(),
                 results[idx].commitmsg);
        return Some(idx);
    }

    let i = results.last().unwrap();
    if i.data_points.get(test).is_none() {
        let idx = results.len() - 1;

        println!("Testing last commit");
        println!("{} {} {:89}", idx,
                 results[idx].commit[..10].yellow(),
                 results[idx].commitmsg);
        return Some(idx);
    }

    let mut last_idx:   usize = 0;
    let mut last_val:   f32 = 0.0;

    let mut gap:        Option<(usize, usize)> = None;
    let mut gap_delta:  f32 = 0.0;

    for (idx, i) in results.iter().enumerate() {
        let v = i.data_points.get(test);

        if let Some(v) = v {
            if last_idx + 1 < idx {
                let delta = (last_val - v) / v;

                if delta < gap_delta {
                    gap         = Some((last_idx, idx));
                    gap_delta   = delta;
                }
            }

            last_idx = idx;
            last_val = *v;
        }
    }

    if let Some(gap) = gap {
        let idx = (gap.0 + gap.1) / 2;

        println!("Bisecting regression by {:.1}%", gap_delta * 100.0);

        println!("{} {} {:89} {}", gap.0,
                 results[gap.0].commit[..10].yellow(),
                 results[gap.0].commitmsg,
                 results[gap.0].data_points.get(test).unwrap());

        println!("{} {} {:89}", idx,
                 results[idx].commit[..10].yellow(),
                 results[idx].commitmsg);

        println!("{} {} {:89} {}", gap.1,
                 results[gap.1].commit[..10].yellow(),
                 results[gap.1].commitmsg,
                 results[gap.1].data_points.get(test).unwrap());
    }

    gap.map(|x| (x.0 + x.1) / 2)
}

fn parse_test_output(output: &str) -> Option<f32> {
    let to_float = float::<_, ()>;

    for l in output.lines() {
        if let Some(idx) = l.find("result") {
            let idx = idx + 7;
            return Some(to_float(&l[idx..]).unwrap().1)
        }
    }

    None
}

fn cmd_run(repo: &git2::Repository, range: &str, test: &str) {
    let mut results = results_new(&repo, range).unwrap();

    let mut existing = results_read(RESULTS).unwrap_or(TestResultsMap { d: HashMap::new() });
    results_merge(&mut results, &existing);

    while let Some(idx) = pick_commit_to_test(&results, test) {
        let output = Command::new("benchmark-git-commit.sh")
            .arg(test)
            .arg(&results[idx].commit)
            .output()
            .unwrap()
            .stdout;

        io::stdout().write_all(&output).unwrap();

        let output = std::str::from_utf8(&output).unwrap();
        let result = parse_test_output(output);

        if let Some(result) = result {
            results[idx].data_points.insert(test.to_string(), result);

            let commit = results[idx].commit.to_string();

            if existing.d.get(&commit).is_none() {
                existing.d.insert(commit.clone(), HashMap::new());
            }

            existing.d.get_mut(&commit).unwrap().insert(test.to_string(), result);

            results_write(RESULTS, &existing).unwrap();
        } else {
            eprintln!("Error parsing test output");
            break;
        }
    }

    results_write(RESULTS, &existing).unwrap();
}

fn log_with_results(repo: &git2::Repository,
                    head: &Option<String>,
                    results: &TestResultsMap) -> Result<TestResultsVec, git2::Error> {
    let mut walk = repo.revwalk().unwrap();

    if let Some(head) = head {
        let object = repo.revparse_single(head)?;

        if let Err(e) = walk.push(object.id()) {
            eprintln!("Error walking {}: {}", head, e);
            return Err(e);
        }
    } else {
        if let Err(e) = walk.push_head() {
            eprintln!("Error walking: {}", e);
            return Err(e);
        }
    }

    let mut v = Vec::new();
    let mut nr_empty = 0;
    let mut last_found_idx = 0;

    for i in walk
        .filter_map(|i| i.ok())
        .filter_map(|i| repo.find_commit(i).ok())
        .map(|i| result_new(&i)) {
        let r = results.d.get(&i.commit);
        let mut i = i;

        if let Some(r) = r {
            i.data_points = r.clone();
            nr_empty = 0;
            last_found_idx = v.len();
        } else {
            nr_empty += 1;
        }

        if nr_empty > 2000 {
            break;
        }

        v.push(i);
    }

    v.truncate(last_found_idx + 1);
    Ok(v)
}

fn results_to_results_with_deltas(results: TestResultsVec) -> TestResultsDeltasVec {
    let mut results: TestResultsDeltasVec = results.iter().map(|i| TestResultDeltas {
        commit:         i.commit.clone(),
        commitmsg:      i.commitmsg.clone(),
        data_points:    dps_to_dpdeltas(&i.data_points)
    }).collect();

    let mut last: DataPoints = HashMap::new();

    for i in results.iter_mut().rev() {
        for c in i.data_points.iter_mut() {
            if let Some(l) = last.get(c.0) {
                c.1.1 = (c.1.0 - l) / l;
            }

            last.insert(c.0.clone(), c.1.0);
        }
    };

    results
}

fn cmd_log(repo: &git2::Repository, head: &Option<String>) -> io::Result<()> {
    /* We use write! to stdout instead of print! to avoid panicing on a broken pipe: */
    let mut stdout = io::stdout();

    colored::control::set_override(ShouldColorize::from_env().should_colorize());
    Pager::with_pager("less -FRX").setup();

    let results = results_read(RESULTS);
    if results.is_err() {
        eprintln!("No results found");
        std::process::exit(1);
    }
    let results = results.unwrap();

    let log = log_with_results(repo, head, &results).unwrap();

    let log = results_to_results_with_deltas(log);

    let mut columns = BTreeSet::new();
    for r in log.iter() {
        for e in r.data_points.iter() {
            columns.insert(e.0);
        }
    }

    let term_cols = termsize::get().map(|x| x.cols);
    let msg_width = if let Some(term_cols) = term_cols {
        term_cols as usize - 11 - columns.len() * 20
    } else {
        89
    };

    for (i, e) in columns.iter().enumerate() {
        if i != 0 && e.len() >= 20 {
            writeln!(stdout, "")?;
        }

        if i == 0 || e.len() >= 20 {
            write!(stdout, "{:>1$}", e, msg_width + 11 + (i + 1) * 20)?;
        } else {
            write!(stdout, "{:>1$}", e, 20)?;
        }
    }
    writeln!(stdout, "")?;

    for i in log.iter() {
        write!(stdout, "{} {:1$.*}", &i.commit[..10].yellow(), msg_width, i.commitmsg)?;

        for e in columns.iter() {
            if let Some(v) = i.data_points.get(e.clone()) {
                let f = format!("{:12} {:+6.1}%", v.0, v.1 * 100.0);
                let f = if v.1 < 0.0 {
                    f.red()
                } else {
                    f.normal()
                };

                write!(stdout, "{}", f)?;
            } else {
                write!(stdout, "{:20}", "")?;
            }
        }
        writeln!(stdout, "")?;
    }

    Ok(())
}

#[derive(Parser)]
#[command(name      = "bisect-perf-regressions")]
#[command(author    = "Kent Overstreet <kent.overstreet@linux.dev>")]
#[command(version   = "0.1")]
#[command(about     = "Bisect performance regressions and provide results in a git log view")]
struct Args {
    #[command(subcommand)]
    command: Option<Commands>,
}

#[derive(Subcommand)]
enum Commands {
    Run {
        range:  String,
        test:   String,
    },
    Log {
        head:   Option<String>
    },
}

fn main() {
    let args = Args::parse();
    let repo = git2::Repository::open(".").unwrap();

    match &args.command {
        Some(Commands::Log { head } )        
            => { cmd_log(&repo, head).ok(); },
        Some(Commands::Run { range, test} )
            => cmd_run(&repo, range, test),
        None                        
            => {}
    }
}