From 0ebafa9068f4eff792824bc089bebcc22eac01f3 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Thu, 26 Jan 2023 12:21:17 -0500 Subject: ci-web: Reorganize to fix build warnings Signed-off-by: Kent Overstreet --- ci-web/Cargo.toml | 10 +- ci-web/src/bin/cgi.rs | 363 ++++++++++++++++++++++++++++++++++ ci-web/src/bin/gen-commit-summary.rs | 28 +++ ci-web/src/bin/get-test-job.rs | 249 ++++++++++++++++++++++++ ci-web/src/gen-commit-summary.rs | 29 --- ci-web/src/get-test-job.rs | 250 ------------------------ ci-web/src/lib.rs | 4 +- ci-web/src/main.rs | 364 ----------------------------------- 8 files changed, 643 insertions(+), 654 deletions(-) create mode 100644 ci-web/src/bin/cgi.rs create mode 100644 ci-web/src/bin/gen-commit-summary.rs create mode 100644 ci-web/src/bin/get-test-job.rs delete mode 100644 ci-web/src/gen-commit-summary.rs delete mode 100644 ci-web/src/get-test-job.rs delete mode 100644 ci-web/src/main.rs diff --git a/ci-web/Cargo.toml b/ci-web/Cargo.toml index 10108f0..6cadcf7 100644 --- a/ci-web/Cargo.toml +++ b/ci-web/Cargo.toml @@ -1,16 +1,8 @@ [package] -name = "ci-cgi" +name = "ci_cgi" version = "0.1.0" edition = "2021" -[[bin]] -name = "get-test-job" -path = "src/get-test-job.rs" - -[[bin]] -name = "gen-commit-summary" -path = "src/gen-commit-summary.rs" - #[workspace] #members = ["get-test-job", "ci-cgi"] diff --git a/ci-web/src/bin/cgi.rs b/ci-web/src/bin/cgi.rs new file mode 100644 index 0000000..e0776c5 --- /dev/null +++ b/ci-web/src/bin/cgi.rs @@ -0,0 +1,363 @@ +use std::collections::BTreeMap; +use std::fmt::Write; +use regex::Regex; +extern crate cgi; +extern crate querystring; + +use ci_cgi::{Ktestrc, ktestrc_read, TestResultsMap, TestStatus, read_lines, commitdir_get_results_toml, git_get_commit}; + +const COMMIT_FILTER: &str = include_str!("../../commit-filter"); +const STYLESHEET: &str = "bootstrap.min.css"; + +fn filter_results(r: TestResultsMap, tests_matching: &Regex) -> TestResultsMap { + r.iter() + .filter(|i| tests_matching.is_match(&i.0) ) + .map(|(k, v)| (k.clone(), *v)) + .collect() +} + +struct Ci { + ktestrc: Ktestrc, + repo: git2::Repository, + stylesheet: String, + script_name: String, + + branch: Option, + commit: Option, + tests_matching: Regex, +} + +fn commitdir_get_results_filtered(ci: &Ci, commit_id: &String) -> TestResultsMap { + let results = commitdir_get_results_toml(&ci.ktestrc, commit_id).unwrap_or(BTreeMap::new()); + + filter_results(results, &ci.tests_matching) +} + +struct CommitResults { + id: String, + message: String, + tests: TestResultsMap, +} + +fn commit_get_results(ci: &Ci, commit: &git2::Commit) -> CommitResults { + let id = commit.id().to_string(); + let tests = commitdir_get_results_filtered(ci, &id); + + CommitResults { + id: id, + message: commit.message().unwrap().to_string(), + tests: tests, + } +} + +fn branch_get_results(ci: &Ci) -> Result, String> { + let mut nr_empty = 0; + let mut nr_commits = 0; + let mut ret: Vec = Vec::new(); + + let branch = ci.branch.as_ref().unwrap(); + let mut walk = ci.repo.revwalk().unwrap(); + + let reference = git_get_commit(&ci.repo, branch.clone()); + if reference.is_err() { + /* XXX: return a 404 */ + return Err(format!("commit not found")); + } + let reference = reference.unwrap(); + + if let Err(e) = walk.push(reference.id()) { + return Err(format!("Error walking {}: {}", branch, e)); + } + + for commit in walk + .filter_map(|i| i.ok()) + .filter_map(|i| ci.repo.find_commit(i).ok()) { + let r = commit_get_results(ci, &commit); + + if !r.tests.is_empty() { + nr_empty = 0; + } else { + nr_empty += 1; + if nr_empty > 100 { + break; + } + } + + ret.push(r); + + nr_commits += 1; + if nr_commits > 50 { + break; + } + } + + + while !ret.is_empty() && ret[ret.len() - 1].tests.is_empty() { + ret.pop(); + } + + Ok(ret) +} + +fn ci_log(ci: &Ci) -> cgi::Response { + let mut out = String::new(); + let branch = ci.branch.as_ref().unwrap(); + + let commits = branch_get_results(ci); + if let Err(e) = commits { + return error_response(e); + } + + let commits = commits.unwrap(); + + let mut multiple_test_view = false; + for r in &commits { + if r.tests.len() > 1 { + multiple_test_view = true; + } + } + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "{}", branch).unwrap(); + writeln!(&mut out, "", ci.stylesheet).unwrap(); + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "
").unwrap(); + writeln!(&mut out, "").unwrap(); + + + if multiple_test_view { + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + + let mut nr_empty = 0; + for r in &commits { + if !r.tests.is_empty() { + if nr_empty != 0 { + writeln!(&mut out, "", nr_empty).unwrap(); + nr_empty = 0; + } + + fn count(r: &TestResultsMap, t: TestStatus) -> usize { + r.iter().filter(|x| x.1.status == t).count() + } + + let subject_len = r.message.find('\n').unwrap_or(r.message.len()); + + let duration: usize = r.tests.iter().map(|x| x.1.duration).sum(); + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "", + ci.script_name, branch, + r.id, &r.id.as_str()[..14]).unwrap(); + writeln!(&mut out, "", &r.message[..subject_len]).unwrap(); + writeln!(&mut out, "", count(&r.tests, TestStatus::Passed)).unwrap(); + writeln!(&mut out, "", count(&r.tests, TestStatus::Failed)).unwrap(); + writeln!(&mut out, "", count(&r.tests, TestStatus::NotStarted)).unwrap(); + writeln!(&mut out, "", count(&r.tests, TestStatus::NotRun)).unwrap(); + writeln!(&mut out, "", count(&r.tests, TestStatus::InProgress)).unwrap(); + writeln!(&mut out, "", count(&r.tests, TestStatus::Unknown)).unwrap(); + writeln!(&mut out, "", r.tests.len()).unwrap(); + writeln!(&mut out, "", duration).unwrap(); + writeln!(&mut out, "").unwrap(); + } else { + nr_empty += 1; + } + } + } else { + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + + let mut nr_empty = 0; + for r in &commits { + if let Some(t) = r.tests.first_key_value() { + if nr_empty != 0 { + writeln!(&mut out, "", nr_empty).unwrap(); + nr_empty = 0; + } + + let subject_len = r.message.find('\n').unwrap_or(r.message.len()); + + writeln!(&mut out, "", t.1.status.table_class()).unwrap(); + writeln!(&mut out, "", + ci.script_name, branch, + r.id, &r.id.as_str()[..14]).unwrap(); + writeln!(&mut out, "", &r.message[..subject_len]).unwrap(); + writeln!(&mut out, "", t.1.status.to_str()).unwrap(); + writeln!(&mut out, "", t.1.duration).unwrap(); + writeln!(&mut out, "", &r.id, t.0).unwrap(); + writeln!(&mut out, "", &r.id, t.0).unwrap(); + writeln!(&mut out, "", &r.id, t.0).unwrap(); + writeln!(&mut out, "").unwrap(); + } else { + nr_empty += 1; + } + } + } + + writeln!(&mut out, "
Commit Description Passed Failed Not started Not run In progress Unknown Total Duration
({} untested commits)
{} {} {} {} {} {} {} {} {} {}s
Commit Description Status Duration
({} untested commits)
{} {} {} {}s log full log output directory
").unwrap(); + writeln!(&mut out, "
").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + cgi::html_response(200, out) +} + +fn ci_commit(ci: &Ci) -> cgi::Response { + let commit_id = ci.commit.as_ref().unwrap(); + let mut out = String::new(); + let commit = git_get_commit(&ci.repo, commit_id.clone()); + if commit.is_err() { + /* XXX: return a 404 */ + return error_response(format!("commit not found")); + } + let commit = commit.unwrap(); + + let message = commit.message().unwrap(); + let subject_len = message.find('\n').unwrap_or(message.len()); + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "{}", &message[..subject_len]).unwrap(); + writeln!(&mut out, "", ci.stylesheet).unwrap(); + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "
").unwrap(); + + writeln!(&mut out, "

{}

", &message[..subject_len]).unwrap(); + + out.push_str(COMMIT_FILTER); + + writeln!(&mut out, "").unwrap(); + + for (name, result) in commitdir_get_results_filtered(ci, &commit_id) { + writeln!(&mut out, "", result.status.table_class()).unwrap(); + writeln!(&mut out, "", name).unwrap(); + writeln!(&mut out, "", result.status.to_str()).unwrap(); + writeln!(&mut out, "", result.duration).unwrap(); + writeln!(&mut out, "", &commit_id, name).unwrap(); + writeln!(&mut out, "", &commit_id, name).unwrap(); + writeln!(&mut out, "", &commit_id, name).unwrap(); + + if let Some(branch) = &ci.branch { + writeln!(&mut out, "", + ci.script_name, &branch, name).unwrap(); + } + + writeln!(&mut out, "").unwrap(); + } + + writeln!(&mut out, "
{} {} {}s log full log output directory git log
").unwrap(); + writeln!(&mut out, "
").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + cgi::html_response(200, out) +} + +fn ci_list_branches(ci: &Ci) -> cgi::Response { + let mut out = String::new(); + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "CI branch list").unwrap(); + writeln!(&mut out, "", ci.stylesheet).unwrap(); + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + + let lines = read_lines(&ci.ktestrc.ci_branches_to_test); + if let Err(e) = lines { + return error_response(format!("error opening ci_branches_to_test {:?}: {}", ci.ktestrc.ci_branches_to_test, e)); + } + let lines = lines.unwrap(); + + let branches: std::collections::HashSet<_> = lines + .filter_map(|i| i.ok()) + .map(|i| if let Some(w) = i.split_whitespace().nth(0) { Some(String::from(w)) } else { None }) + .filter_map(|i| i) + .collect(); + + let mut branches: Vec<_> = branches.iter().collect(); + branches.sort(); + + for b in branches { + writeln!(&mut out, "", ci.script_name, b, b).unwrap(); + } + + writeln!(&mut out, "
{}
").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + cgi::html_response(200, out) +} + +fn cgi_header_get(request: &cgi::Request, name: &str) -> String { + request.headers().get(name) + .map(|x| x.to_str()) + .transpose().ok().flatten() + .map(|x| x.to_string()) + .unwrap_or(String::new()) +} + +fn error_response(msg: String) -> cgi::Response { + let mut out = String::new(); + writeln!(&mut out, "{}", msg).unwrap(); + let env: Vec<_> = std::env::vars().collect(); + writeln!(&mut out, "env: {:?}", env).unwrap(); + cgi::text_response(200, out) +} + +cgi::cgi_main! {|request: cgi::Request| -> cgi::Response { + let ktestrc = ktestrc_read(); + if let Err(e) = ktestrc { + return error_response(format!("could not read config; {}", e)); + } + let ktestrc = ktestrc.unwrap(); + + if !ktestrc.ci_output_dir.exists() { + return error_response(format!("required file missing: JOBSERVER_OUTPUT_DIR (got {:?})", + ktestrc.ci_output_dir)); + } + + let repo = git2::Repository::open(&ktestrc.ci_linux_repo); + if let Err(e) = repo { + return error_response(format!("error opening repository {:?}: {}", ktestrc.ci_linux_repo, e)); + } + let repo = repo.unwrap(); + + let query = cgi_header_get(&request, "x-cgi-query-string"); + let query: std::collections::HashMap<_, _> = + querystring::querify(&query).into_iter().collect(); + + let tests_matching = query.get("test").unwrap_or(&""); + + let ci = Ci { + ktestrc: ktestrc, + repo: repo, + stylesheet: String::from(STYLESHEET), + script_name: cgi_header_get(&request, "x-cgi-script-name"), + + branch: query.get("branch").map(|x| x.to_string()), + commit: query.get("commit").map(|x| x.to_string()), + tests_matching: Regex::new(tests_matching).unwrap_or(Regex::new("").unwrap()), + }; + + if ci.commit.is_some() { + ci_commit(&ci) + } else if ci.branch.is_some() { + ci_log(&ci) + } else { + ci_list_branches(&ci) + } +} } diff --git a/ci-web/src/bin/gen-commit-summary.rs b/ci-web/src/bin/gen-commit-summary.rs new file mode 100644 index 0000000..5a3c3a4 --- /dev/null +++ b/ci-web/src/bin/gen-commit-summary.rs @@ -0,0 +1,28 @@ +use std::process; +use ci_cgi::{ktestrc_read, commitdir_get_results, TestResults}; +use clap::Parser; +use toml; + +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +struct Args { + commit: String, +} + +fn main() { + let args = Args::parse(); + + let ktestrc = ktestrc_read(); + if let Err(e) = ktestrc { + eprintln!("could not read config; {}", e); + process::exit(1); + } + let ktestrc = ktestrc.unwrap(); + + let results = TestResults { d: commitdir_get_results(&ktestrc, &args.commit) }; + + let file_contents = toml::to_string(&results).unwrap(); + + let commit_summary_fname = ktestrc.ci_output_dir.join(args.commit + ".toml"); + std::fs::write(commit_summary_fname, file_contents).unwrap(); +} diff --git a/ci-web/src/bin/get-test-job.rs b/ci-web/src/bin/get-test-job.rs new file mode 100644 index 0000000..d1ab61f --- /dev/null +++ b/ci-web/src/bin/get-test-job.rs @@ -0,0 +1,249 @@ +extern crate libc; +use std::collections::BTreeMap; +use std::fs::{OpenOptions, create_dir_all}; +use std::io::ErrorKind; +use std::path::{Path, PathBuf}; +use std::process; +use std::time::SystemTime; +use memoize::memoize; +use ci_cgi::{Ktestrc, ktestrc_read, read_lines, git_get_commit, commitdir_get_results_toml}; + +use multimap::MultiMap; +use die::die; + +use glob::glob; + +#[memoize] +fn get_subtests(test_path: PathBuf) -> Vec { + let output = std::process::Command::new(&test_path) + .arg("list-tests") + .output() + .expect(&format!("failed to execute process {:?} ", &test_path)) + .stdout; + let output = String::from_utf8_lossy(&output); + + output + .split_whitespace() + .map(|i| i.to_string()) + .collect() +} + +fn lockfile_exists(rc: &Ktestrc, commit: &str, test_name: &str, create: bool) -> bool { + let lockfile = rc.ci_output_dir.join(commit).join(test_name).join("status"); + + let timeout = std::time::Duration::from_secs(3600); + let metadata = std::fs::metadata(&lockfile); + + if let Ok(metadata) = metadata { + let elapsed = metadata.modified().unwrap() + .elapsed() + .unwrap_or(std::time::Duration::from_secs(0)); + + if metadata.is_file() && + metadata.len() == 0 && + elapsed > timeout && + std::fs::remove_file(&lockfile).is_ok() { + eprintln!("Deleted stale lock file {:?}, mtime {:?} now {:?} elapsed {:?})", + &lockfile, metadata.modified().unwrap(), + SystemTime::now(), + elapsed); + } + } + + if !create { + lockfile.exists() + } else { + let dir = lockfile.parent().unwrap(); + let r = create_dir_all(dir); + if let Err(e) = r { + if e.kind() != ErrorKind::AlreadyExists { + die!("error creating {:?}: {}", dir, e); + } + } + + let r = OpenOptions::new() + .write(true) + .create_new(true) + .open(&lockfile); + if let Err(ref e) = r { + if e.kind() != ErrorKind::AlreadyExists { + die!("error creating {:?}: {}", lockfile, e); + } + } + + r.is_ok() + } +} + +struct TestJob { + branch: String, + commit: String, + age: usize, + test: PathBuf, + subtests: Vec, +} + +fn subtest_full_name(test_path: &Path, subtest: &String) -> String { + format!("{}.{}", + test_path.file_stem().unwrap().to_string_lossy(), + subtest.replace("/", ".")) +} + +fn branch_get_next_test_job(rc: &Ktestrc, repo: &git2::Repository, + branch: &str, test_path: &Path) -> Option { + let mut ret = TestJob { + branch: branch.to_string(), + commit: String::new(), + age: 0, + test: test_path.to_path_buf(), + subtests: Vec::new(), + }; + + let subtests = get_subtests(PathBuf::from(test_path)); + + let mut walk = repo.revwalk().unwrap(); + let reference = git_get_commit(&repo, branch.to_string()); + if reference.is_err() { + eprintln!("branch {} not found", branch); + return None; + } + let reference = reference.unwrap(); + + if let Err(e) = walk.push(reference.id()) { + eprintln!("Error walking {}: {}", branch, e); + return None; + } + + for commit in walk + .filter_map(|i| i.ok()) + .filter_map(|i| repo.find_commit(i).ok()) { + let commit = commit.id().to_string(); + ret.commit = commit.clone(); + + let results = commitdir_get_results_toml(rc, &commit).unwrap_or(BTreeMap::new()); + + for subtest in subtests.iter() { + let full_subtest_name = subtest_full_name(test_path, &subtest); + + if results.get(&full_subtest_name).is_none() && + !lockfile_exists(rc, &commit, &full_subtest_name, false) { + ret.subtests.push(subtest.to_string()); + if ret.subtests.len() > 20 { + break; + } + } + } + + if !ret.subtests.is_empty() { + return Some(ret); + } + + ret.age += 1; + if ret.age > 50 { + break; + } + } + + None +} + +fn get_best_test_job(rc: &Ktestrc, repo: &git2::Repository, + branch_tests: &MultiMap) -> Option { + let mut ret: Option = None; + + for (branch, testvec) in branch_tests.iter_all() { + for test in testvec { + let job = branch_get_next_test_job(rc, repo, branch, test); + + let ret_age = ret.as_ref().map_or(std::usize::MAX, |x| x.age); + let job_age = job.as_ref().map_or(std::usize::MAX, |x| x.age); + + if job_age < ret_age { + ret = job; + } + } + } + + ret +} + +fn create_job_lockfiles(rc: &Ktestrc, mut job: TestJob) -> Option { + job.subtests = job.subtests.iter() + .filter(|i| lockfile_exists(rc, &job.commit, + &subtest_full_name(&Path::new(&job.test), &i), true)) + .map(|i| i.to_string()) + .collect(); + + if !job.subtests.is_empty() { Some(job) } else { None } +} + +fn main() { + let ktestrc = ktestrc_read(); + if let Err(e) = ktestrc { + eprintln!("could not read config; {}", e); + process::exit(1); + } + let ktestrc = ktestrc.unwrap(); + + let repo = git2::Repository::open(&ktestrc.ci_linux_repo); + if let Err(e) = repo { + eprintln!("Error opening {:?}: {}", ktestrc.ci_linux_repo, e); + eprintln!("Please specify correct ci_linux_repo"); + process::exit(1); + } + let repo = repo.unwrap(); + + let _r = std::process::Command::new("flock") + .arg("--nonblock") + .arg(".git_fetch.lock") + .arg("git").arg("fetch").arg("--all") + .current_dir(&ktestrc.ci_linux_repo) + .output(); + + let lines = read_lines(&ktestrc.ci_branches_to_test); + if let Err(e) = lines { + eprintln!("Error opening {:?}: {}", ktestrc.ci_branches_to_test, e); + eprintln!("Please specify correct ci_branches_to_test"); + process::exit(1); + } + let lines = lines.unwrap(); + + let lines = lines.filter_map(|i| i.ok()); + + let mut branch_tests: MultiMap = MultiMap::new(); + + for l in lines { + let l: Vec<_> = l.split_whitespace().take(2).collect(); + + if l.len() == 2 { + let branch = l[0]; + let test = l[1]; + + for i in glob(test).expect(&format!("No tests matching {}", test)) + .filter_map(|i| i.ok()) { + branch_tests.insert(branch.to_string(), i); + } + } + } + + let mut job: Option; + + loop { + job = get_best_test_job(&ktestrc, &repo, &branch_tests); + + if job.is_none() { + break; + } + + job = create_job_lockfiles(&ktestrc, job.unwrap()); + if let Some(job) = job { + print!("{} {} {}", job.branch, job.commit, job.test.display()); + for t in job.subtests { + print!(" {}", t); + } + println!(""); + break; + } + } + +} diff --git a/ci-web/src/gen-commit-summary.rs b/ci-web/src/gen-commit-summary.rs deleted file mode 100644 index 1da81f0..0000000 --- a/ci-web/src/gen-commit-summary.rs +++ /dev/null @@ -1,29 +0,0 @@ -use std::process; -mod lib; -use lib::{ktestrc_read, commitdir_get_results, TestResults}; -use clap::Parser; -use toml; - -#[derive(Parser)] -#[command(author, version, about, long_about = None)] -struct Args { - commit: String, -} - -fn main() { - let args = Args::parse(); - - let ktestrc = ktestrc_read(); - if let Err(e) = ktestrc { - eprintln!("could not read config; {}", e); - process::exit(1); - } - let ktestrc = ktestrc.unwrap(); - - let results = TestResults { d: commitdir_get_results(&ktestrc, &args.commit) }; - - let file_contents = toml::to_string(&results).unwrap(); - - let commit_summary_fname = ktestrc.ci_output_dir.join(args.commit + ".toml"); - std::fs::write(commit_summary_fname, file_contents).unwrap(); -} diff --git a/ci-web/src/get-test-job.rs b/ci-web/src/get-test-job.rs deleted file mode 100644 index 85e6feb..0000000 --- a/ci-web/src/get-test-job.rs +++ /dev/null @@ -1,250 +0,0 @@ -extern crate libc; -use std::collections::BTreeMap; -use std::fs::{OpenOptions, create_dir_all}; -use std::io::ErrorKind; -use std::path::{Path, PathBuf}; -use std::process; -use std::time::SystemTime; -use memoize::memoize; -mod lib; -use lib::*; - -use multimap::MultiMap; -use die::die; - -use glob::glob; - -#[memoize] -fn get_subtests(test_path: PathBuf) -> Vec { - let output = std::process::Command::new(&test_path) - .arg("list-tests") - .output() - .expect(&format!("failed to execute process {:?} ", &test_path)) - .stdout; - let output = String::from_utf8_lossy(&output); - - output - .split_whitespace() - .map(|i| i.to_string()) - .collect() -} - -fn lockfile_exists(rc: &Ktestrc, commit: &str, test_name: &str, create: bool) -> bool { - let lockfile = rc.ci_output_dir.join(commit).join(test_name).join("status"); - - let timeout = std::time::Duration::from_secs(3600); - let metadata = std::fs::metadata(&lockfile); - - if let Ok(metadata) = metadata { - let elapsed = metadata.modified().unwrap() - .elapsed() - .unwrap_or(std::time::Duration::from_secs(0)); - - if metadata.is_file() && - metadata.len() == 0 && - elapsed > timeout && - std::fs::remove_file(&lockfile).is_ok() { - eprintln!("Deleted stale lock file {:?}, mtime {:?} now {:?} elapsed {:?})", - &lockfile, metadata.modified().unwrap(), - SystemTime::now(), - elapsed); - } - } - - if !create { - lockfile.exists() - } else { - let dir = lockfile.parent().unwrap(); - let r = create_dir_all(dir); - if let Err(e) = r { - if e.kind() != ErrorKind::AlreadyExists { - die!("error creating {:?}: {}", dir, e); - } - } - - let r = OpenOptions::new() - .write(true) - .create_new(true) - .open(&lockfile); - if let Err(ref e) = r { - if e.kind() != ErrorKind::AlreadyExists { - die!("error creating {:?}: {}", lockfile, e); - } - } - - r.is_ok() - } -} - -struct TestJob { - branch: String, - commit: String, - age: usize, - test: PathBuf, - subtests: Vec, -} - -fn subtest_full_name(test_path: &Path, subtest: &String) -> String { - format!("{}.{}", - test_path.file_stem().unwrap().to_string_lossy(), - subtest.replace("/", ".")) -} - -fn branch_get_next_test_job(rc: &Ktestrc, repo: &git2::Repository, - branch: &str, test_path: &Path) -> Option { - let mut ret = TestJob { - branch: branch.to_string(), - commit: String::new(), - age: 0, - test: test_path.to_path_buf(), - subtests: Vec::new(), - }; - - let subtests = get_subtests(PathBuf::from(test_path)); - - let mut walk = repo.revwalk().unwrap(); - let reference = git_get_commit(&repo, branch.to_string()); - if reference.is_err() { - eprintln!("branch {} not found", branch); - return None; - } - let reference = reference.unwrap(); - - if let Err(e) = walk.push(reference.id()) { - eprintln!("Error walking {}: {}", branch, e); - return None; - } - - for commit in walk - .filter_map(|i| i.ok()) - .filter_map(|i| repo.find_commit(i).ok()) { - let commit = commit.id().to_string(); - ret.commit = commit.clone(); - - let results = commitdir_get_results_toml(rc, &commit).unwrap_or(BTreeMap::new()); - - for subtest in subtests.iter() { - let full_subtest_name = subtest_full_name(test_path, &subtest); - - if results.get(&full_subtest_name).is_none() && - !lockfile_exists(rc, &commit, &full_subtest_name, false) { - ret.subtests.push(subtest.to_string()); - if ret.subtests.len() > 20 { - break; - } - } - } - - if !ret.subtests.is_empty() { - return Some(ret); - } - - ret.age += 1; - if ret.age > 50 { - break; - } - } - - None -} - -fn get_best_test_job(rc: &Ktestrc, repo: &git2::Repository, - branch_tests: &MultiMap) -> Option { - let mut ret: Option = None; - - for (branch, testvec) in branch_tests.iter_all() { - for test in testvec { - let job = branch_get_next_test_job(rc, repo, branch, test); - - let ret_age = ret.as_ref().map_or(std::usize::MAX, |x| x.age); - let job_age = job.as_ref().map_or(std::usize::MAX, |x| x.age); - - if job_age < ret_age { - ret = job; - } - } - } - - ret -} - -fn create_job_lockfiles(rc: &Ktestrc, mut job: TestJob) -> Option { - job.subtests = job.subtests.iter() - .filter(|i| lockfile_exists(rc, &job.commit, - &subtest_full_name(&Path::new(&job.test), &i), true)) - .map(|i| i.to_string()) - .collect(); - - if !job.subtests.is_empty() { Some(job) } else { None } -} - -fn main() { - let ktestrc = ktestrc_read(); - if let Err(e) = ktestrc { - eprintln!("could not read config; {}", e); - process::exit(1); - } - let ktestrc = ktestrc.unwrap(); - - let repo = git2::Repository::open(&ktestrc.ci_linux_repo); - if let Err(e) = repo { - eprintln!("Error opening {:?}: {}", ktestrc.ci_linux_repo, e); - eprintln!("Please specify correct ci_linux_repo"); - process::exit(1); - } - let repo = repo.unwrap(); - - let _r = std::process::Command::new("flock") - .arg("--nonblock") - .arg(".git_fetch.lock") - .arg("git").arg("fetch").arg("--all") - .current_dir(&ktestrc.ci_linux_repo) - .output(); - - let lines = read_lines(&ktestrc.ci_branches_to_test); - if let Err(e) = lines { - eprintln!("Error opening {:?}: {}", ktestrc.ci_branches_to_test, e); - eprintln!("Please specify correct ci_branches_to_test"); - process::exit(1); - } - let lines = lines.unwrap(); - - let lines = lines.filter_map(|i| i.ok()); - - let mut branch_tests: MultiMap = MultiMap::new(); - - for l in lines { - let l: Vec<_> = l.split_whitespace().take(2).collect(); - - if l.len() == 2 { - let branch = l[0]; - let test = l[1]; - - for i in glob(test).expect(&format!("No tests matching {}", test)) - .filter_map(|i| i.ok()) { - branch_tests.insert(branch.to_string(), i); - } - } - } - - let mut job: Option; - - loop { - job = get_best_test_job(&ktestrc, &repo, &branch_tests); - - if job.is_none() { - break; - } - - job = create_job_lockfiles(&ktestrc, job.unwrap()); - if let Some(job) = job { - print!("{} {} {}", job.branch, job.commit, job.test.display()); - for t in job.subtests { - print!(" {}", t); - } - println!(""); - break; - } - } - -} diff --git a/ci-web/src/lib.rs b/ci-web/src/lib.rs index 6819a0d..b4bd65c 100644 --- a/ci-web/src/lib.rs +++ b/ci-web/src/lib.rs @@ -53,7 +53,7 @@ pub enum TestStatus { } impl TestStatus { - pub fn from_str(status: &str) -> TestStatus { + fn from_str(status: &str) -> TestStatus { if status.is_empty() { TestStatus::InProgress } else if status.contains("IN PROGRESS") { @@ -107,7 +107,7 @@ pub struct TestResults { pub d: TestResultsMap } -pub fn read_test_result(testdir: &std::fs::DirEntry) -> Option { +fn read_test_result(testdir: &std::fs::DirEntry) -> Option { Some(TestResult { status: TestStatus::from_str(&read_to_string(&testdir.path().join("status")).ok()?), duration: read_to_string(&testdir.path().join("duration")).unwrap_or("0".to_string()).parse().unwrap_or(0), diff --git a/ci-web/src/main.rs b/ci-web/src/main.rs deleted file mode 100644 index 30ea932..0000000 --- a/ci-web/src/main.rs +++ /dev/null @@ -1,364 +0,0 @@ -use std::collections::BTreeMap; -use std::fmt::Write; -use regex::Regex; -extern crate cgi; -extern crate querystring; - -mod lib; -use lib::*; - -const COMMIT_FILTER: &str = include_str!("../commit-filter"); -const STYLESHEET: &str = "bootstrap.min.css"; - -fn filter_results(r: TestResultsMap, tests_matching: &Regex) -> TestResultsMap { - r.iter() - .filter(|i| tests_matching.is_match(&i.0) ) - .map(|(k, v)| (k.clone(), *v)) - .collect() -} - -struct Ci { - ktestrc: Ktestrc, - repo: git2::Repository, - stylesheet: String, - script_name: String, - - branch: Option, - commit: Option, - tests_matching: Regex, -} - -fn commitdir_get_results_filtered(ci: &Ci, commit_id: &String) -> TestResultsMap { - let results = commitdir_get_results_toml(&ci.ktestrc, commit_id).unwrap_or(BTreeMap::new()); - - filter_results(results, &ci.tests_matching) -} - -struct CommitResults { - id: String, - message: String, - tests: TestResultsMap, -} - -fn commit_get_results(ci: &Ci, commit: &git2::Commit) -> CommitResults { - let id = commit.id().to_string(); - let tests = commitdir_get_results_filtered(ci, &id); - - CommitResults { - id: id, - message: commit.message().unwrap().to_string(), - tests: tests, - } -} - -fn branch_get_results(ci: &Ci) -> Result, String> { - let mut nr_empty = 0; - let mut nr_commits = 0; - let mut ret: Vec = Vec::new(); - - let branch = ci.branch.as_ref().unwrap(); - let mut walk = ci.repo.revwalk().unwrap(); - - let reference = git_get_commit(&ci.repo, branch.clone()); - if reference.is_err() { - /* XXX: return a 404 */ - return Err(format!("commit not found")); - } - let reference = reference.unwrap(); - - if let Err(e) = walk.push(reference.id()) { - return Err(format!("Error walking {}: {}", branch, e)); - } - - for commit in walk - .filter_map(|i| i.ok()) - .filter_map(|i| ci.repo.find_commit(i).ok()) { - let r = commit_get_results(ci, &commit); - - if !r.tests.is_empty() { - nr_empty = 0; - } else { - nr_empty += 1; - if nr_empty > 100 { - break; - } - } - - ret.push(r); - - nr_commits += 1; - if nr_commits > 50 { - break; - } - } - - - while !ret.is_empty() && ret[ret.len() - 1].tests.is_empty() { - ret.pop(); - } - - Ok(ret) -} - -fn ci_log(ci: &Ci) -> cgi::Response { - let mut out = String::new(); - let branch = ci.branch.as_ref().unwrap(); - - let commits = branch_get_results(ci); - if let Err(e) = commits { - return error_response(e); - } - - let commits = commits.unwrap(); - - let mut multiple_test_view = false; - for r in &commits { - if r.tests.len() > 1 { - multiple_test_view = true; - } - } - - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "{}", branch).unwrap(); - writeln!(&mut out, "", ci.stylesheet).unwrap(); - - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "
").unwrap(); - writeln!(&mut out, "").unwrap(); - - - if multiple_test_view { - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - - let mut nr_empty = 0; - for r in &commits { - if !r.tests.is_empty() { - if nr_empty != 0 { - writeln!(&mut out, "", nr_empty).unwrap(); - nr_empty = 0; - } - - fn count(r: &TestResultsMap, t: TestStatus) -> usize { - r.iter().filter(|x| x.1.status == t).count() - } - - let subject_len = r.message.find('\n').unwrap_or(r.message.len()); - - let duration: usize = r.tests.iter().map(|x| x.1.duration).sum(); - - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "", - ci.script_name, branch, - r.id, &r.id.as_str()[..14]).unwrap(); - writeln!(&mut out, "", &r.message[..subject_len]).unwrap(); - writeln!(&mut out, "", count(&r.tests, TestStatus::Passed)).unwrap(); - writeln!(&mut out, "", count(&r.tests, TestStatus::Failed)).unwrap(); - writeln!(&mut out, "", count(&r.tests, TestStatus::NotStarted)).unwrap(); - writeln!(&mut out, "", count(&r.tests, TestStatus::NotRun)).unwrap(); - writeln!(&mut out, "", count(&r.tests, TestStatus::InProgress)).unwrap(); - writeln!(&mut out, "", count(&r.tests, TestStatus::Unknown)).unwrap(); - writeln!(&mut out, "", r.tests.len()).unwrap(); - writeln!(&mut out, "", duration).unwrap(); - writeln!(&mut out, "").unwrap(); - } else { - nr_empty += 1; - } - } - } else { - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - - let mut nr_empty = 0; - for r in &commits { - if let Some(t) = r.tests.first_key_value() { - if nr_empty != 0 { - writeln!(&mut out, "", nr_empty).unwrap(); - nr_empty = 0; - } - - let subject_len = r.message.find('\n').unwrap_or(r.message.len()); - - writeln!(&mut out, "", t.1.status.table_class()).unwrap(); - writeln!(&mut out, "", - ci.script_name, branch, - r.id, &r.id.as_str()[..14]).unwrap(); - writeln!(&mut out, "", &r.message[..subject_len]).unwrap(); - writeln!(&mut out, "", t.1.status.to_str()).unwrap(); - writeln!(&mut out, "", t.1.duration).unwrap(); - writeln!(&mut out, "", &r.id, t.0).unwrap(); - writeln!(&mut out, "", &r.id, t.0).unwrap(); - writeln!(&mut out, "", &r.id, t.0).unwrap(); - writeln!(&mut out, "").unwrap(); - } else { - nr_empty += 1; - } - } - } - - writeln!(&mut out, "
Commit Description Passed Failed Not started Not run In progress Unknown Total Duration
({} untested commits)
{} {} {} {} {} {} {} {} {} {}s
Commit Description Status Duration
({} untested commits)
{} {} {} {}s log full log output directory
").unwrap(); - writeln!(&mut out, "
").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - cgi::html_response(200, out) -} - -fn ci_commit(ci: &Ci) -> cgi::Response { - let commit_id = ci.commit.as_ref().unwrap(); - let mut out = String::new(); - let commit = git_get_commit(&ci.repo, commit_id.clone()); - if commit.is_err() { - /* XXX: return a 404 */ - return error_response(format!("commit not found")); - } - let commit = commit.unwrap(); - - let message = commit.message().unwrap(); - let subject_len = message.find('\n').unwrap_or(message.len()); - - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "{}", &message[..subject_len]).unwrap(); - writeln!(&mut out, "", ci.stylesheet).unwrap(); - - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "
").unwrap(); - - writeln!(&mut out, "

{}

", &message[..subject_len]).unwrap(); - - out.push_str(COMMIT_FILTER); - - writeln!(&mut out, "").unwrap(); - - for (name, result) in commitdir_get_results_filtered(ci, &commit_id) { - writeln!(&mut out, "", result.status.table_class()).unwrap(); - writeln!(&mut out, "", name).unwrap(); - writeln!(&mut out, "", result.status.to_str()).unwrap(); - writeln!(&mut out, "", result.duration).unwrap(); - writeln!(&mut out, "", &commit_id, name).unwrap(); - writeln!(&mut out, "", &commit_id, name).unwrap(); - writeln!(&mut out, "", &commit_id, name).unwrap(); - - if let Some(branch) = &ci.branch { - writeln!(&mut out, "", - ci.script_name, &branch, name).unwrap(); - } - - writeln!(&mut out, "").unwrap(); - } - - writeln!(&mut out, "
{} {} {}s log full log output directory git log
").unwrap(); - writeln!(&mut out, "
").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - cgi::html_response(200, out) -} - -fn ci_list_branches(ci: &Ci) -> cgi::Response { - let mut out = String::new(); - - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "CI branch list").unwrap(); - writeln!(&mut out, "", ci.stylesheet).unwrap(); - - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - - let lines = read_lines(&ci.ktestrc.ci_branches_to_test); - if let Err(e) = lines { - return error_response(format!("error opening ci_branches_to_test {:?}: {}", ci.ktestrc.ci_branches_to_test, e)); - } - let lines = lines.unwrap(); - - let branches: std::collections::HashSet<_> = lines - .filter_map(|i| i.ok()) - .map(|i| if let Some(w) = i.split_whitespace().nth(0) { Some(String::from(w)) } else { None }) - .filter_map(|i| i) - .collect(); - - let mut branches: Vec<_> = branches.iter().collect(); - branches.sort(); - - for b in branches { - writeln!(&mut out, "", ci.script_name, b, b).unwrap(); - } - - writeln!(&mut out, "
{}
").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - cgi::html_response(200, out) -} - -fn cgi_header_get(request: &cgi::Request, name: &str) -> String { - request.headers().get(name) - .map(|x| x.to_str()) - .transpose().ok().flatten() - .map(|x| x.to_string()) - .unwrap_or(String::new()) -} - -fn error_response(msg: String) -> cgi::Response { - let mut out = String::new(); - writeln!(&mut out, "{}", msg).unwrap(); - let env: Vec<_> = std::env::vars().collect(); - writeln!(&mut out, "env: {:?}", env).unwrap(); - cgi::text_response(200, out) -} - -cgi::cgi_main! {|request: cgi::Request| -> cgi::Response { - let ktestrc = ktestrc_read(); - if let Err(e) = ktestrc { - return error_response(format!("could not read config; {}", e)); - } - let ktestrc = ktestrc.unwrap(); - - if !ktestrc.ci_output_dir.exists() { - return error_response(format!("required file missing: JOBSERVER_OUTPUT_DIR (got {:?})", - ktestrc.ci_output_dir)); - } - - let repo = git2::Repository::open(&ktestrc.ci_linux_repo); - if let Err(e) = repo { - return error_response(format!("error opening repository {:?}: {}", ktestrc.ci_linux_repo, e)); - } - let repo = repo.unwrap(); - - let query = cgi_header_get(&request, "x-cgi-query-string"); - let query: std::collections::HashMap<_, _> = - querystring::querify(&query).into_iter().collect(); - - let tests_matching = query.get("test").unwrap_or(&""); - - let ci = Ci { - ktestrc: ktestrc, - repo: repo, - stylesheet: String::from(STYLESHEET), - script_name: cgi_header_get(&request, "x-cgi-script-name"), - - branch: query.get("branch").map(|x| x.to_string()), - commit: query.get("commit").map(|x| x.to_string()), - tests_matching: Regex::new(tests_matching).unwrap_or(Regex::new("").unwrap()), - }; - - if ci.commit.is_some() { - ci_commit(&ci) - } else if ci.branch.is_some() { - ci_log(&ci) - } else { - ci_list_branches(&ci) - } -} } -- cgit v1.2.3