diff options
| -rw-r--r-- | src/bin/cgi.rs | 73 | ||||
| -rw-r--r-- | src/bin/gc-results.rs | 32 | ||||
| -rw-r--r-- | src/bin/gen-job-list.rs | 112 | ||||
| -rw-r--r-- | src/bin/get-test-job.rs | 5 | ||||
| -rw-r--r-- | src/lib.rs | 40 | ||||
| -rw-r--r-- | src/users.rs | 32 |
6 files changed, 199 insertions, 95 deletions
diff --git a/src/bin/cgi.rs b/src/bin/cgi.rs index 31137ca..9bac1c7 100644 --- a/src/bin/cgi.rs +++ b/src/bin/cgi.rs @@ -5,7 +5,7 @@ use chrono::Duration; extern crate cgi; extern crate querystring; -use ci_cgi::{Ktestrc, ktestrc_read, TestResultsMap, TestStatus, commitdir_get_results, git_get_commit, workers_get, update_lcov}; +use ci_cgi::{CiConfig, Userrc, ciconfig_read, TestResultsMap, TestStatus, commitdir_get_results, git_get_commit, workers_get, update_lcov}; const COMMIT_FILTER: &str = include_str!("../../commit-filter"); const STYLESHEET: &str = "bootstrap.min.css"; @@ -18,18 +18,19 @@ fn filter_results(r: TestResultsMap, tests_matching: &Regex) -> TestResultsMap { } struct Ci { - ktestrc: Ktestrc, + rc: CiConfig, repo: git2::Repository, stylesheet: String, script_name: String, + user: Option<String>, branch: Option<String>, commit: Option<String>, tests_matching: Regex, } fn commitdir_get_results_filtered(ci: &Ci, commit_id: &String) -> TestResultsMap { - let results = commitdir_get_results(&ci.ktestrc, commit_id).unwrap_or(BTreeMap::new()); + let results = commitdir_get_results(&ci.rc.ktest, commit_id).unwrap_or(BTreeMap::new()); filter_results(results, &ci.tests_matching) } @@ -240,9 +241,9 @@ fn ci_commit(ci: &Ci) -> cgi::Response { writeln!(&mut out, "<h3><th>{}</th></h3>", &message[..subject_len]).unwrap(); - update_lcov(&ci.ktestrc, &commit_id); + update_lcov(&ci.rc.ktest, &commit_id); - if ci.ktestrc.output_dir.join(&commit_id).join("lcov").exists() { + if ci.rc.ktest.output_dir.join(&commit_id).join("lcov").exists() { writeln!(&mut out, "<p> <a href=c/{}/lcov> Code coverage </a> </p>", &commit_id).unwrap(); } @@ -274,20 +275,53 @@ fn ci_commit(ci: &Ci) -> cgi::Response { cgi::html_response(200, out) } -fn ci_list_branches(ci: &Ci, out: &mut String) { +fn ci_list_branches(ci: &Ci, user: &Userrc, out: &mut String) { writeln!(out, "<div> <table class=\"table\">").unwrap(); - for (b, _) in &ci.ktestrc.branch { + for (b, _) in &user.branch { writeln!(out, "<tr> <th> <a href={}?branch={}>{}</a> </th> </tr>", ci.script_name, b, b).unwrap(); } writeln!(out, "</table> </div>").unwrap(); } +fn ci_user(ci: &Ci) -> cgi::Response { + let username = ci.user.as_ref().unwrap(); + let u = ci.rc.users.get(username); + + if u.is_none() { + return error_response(format!("User {} not found", &username)); + } + let u = u.unwrap(); + + let mut out = String::new(); + + writeln!(&mut out, "<!DOCTYPE HTML>").unwrap(); + writeln!(&mut out, "<html><head><title>CI branch list</title></head>").unwrap(); + writeln!(&mut out, "<link href=\"{}\" rel=\"stylesheet\">", ci.stylesheet).unwrap(); + + writeln!(&mut out, "<body>").unwrap(); + + ci_list_branches(ci, &u, &mut out); + + writeln!(&mut out, "</body>").unwrap(); + writeln!(&mut out, "</html>").unwrap(); + + cgi::html_response(200, out) +} + +fn ci_list_users(ci: &Ci, out: &mut String) { + writeln!(out, "<div> <table class=\"table\">").unwrap(); + for (i, _) in &ci.rc.users { + writeln!(out, "<tr> <th> <a href={}?user={}>{}</a> </th> </tr>", ci.script_name, i, i).unwrap(); + } + writeln!(out, "</table> </div>").unwrap(); +} + fn ci_worker_status(ci: &Ci, out: &mut String) -> Option<()>{ use chrono::prelude::Utc; - let workers = workers_get(&ci.ktestrc).ok()?; + let workers = workers_get(&ci.rc.ktest).ok()?; writeln!(out, "<div> <table class=\"table\">").unwrap(); @@ -299,7 +333,7 @@ fn ci_worker_status(ci: &Ci, out: &mut String) -> Option<()>{ writeln!(out, "</tr>").unwrap(); let now = Utc::now(); - let tests_dir = ci.ktestrc.ktest_dir.clone().into_os_string().into_string().unwrap() + "/tests/"; + let tests_dir = ci.rc.ktest.ktest_dir.clone().into_os_string().into_string().unwrap() + "/tests/"; for w in workers { let elapsed = (now - w.starttime).max(Duration::zero()); @@ -330,7 +364,7 @@ fn ci_home(ci: &Ci) -> cgi::Response { writeln!(&mut out, "<body>").unwrap(); - ci_list_branches(ci, &mut out); + ci_list_users(ci, &mut out); ci_worker_status(ci, &mut out); @@ -357,15 +391,15 @@ fn error_response(msg: String) -> cgi::Response { } cgi::cgi_main! {|request: cgi::Request| -> cgi::Response { - let ktestrc = ktestrc_read(); - if let Err(e) = ktestrc { + let rc = ciconfig_read(); + if let Err(e) = rc { return error_response(format!("could not read config; {}", e)); } - let ktestrc = ktestrc.unwrap(); + let rc = rc.unwrap(); - if !ktestrc.output_dir.exists() { + if !rc.ktest.output_dir.exists() { return error_response(format!("required file missing: JOBSERVER_OUTPUT_DIR (got {:?})", - ktestrc.output_dir)); + rc.ktest.output_dir)); } unsafe { @@ -373,9 +407,9 @@ cgi::cgi_main! {|request: cgi::Request| -> cgi::Response { .expect("set_verify_owner_validation should never fail"); } - let repo = git2::Repository::open(&ktestrc.linux_repo); + let repo = git2::Repository::open(&rc.ktest.linux_repo); if let Err(e) = repo { - return error_response(format!("error opening repository {:?}: {}", ktestrc.linux_repo, e)); + return error_response(format!("error opening repository {:?}: {}", rc.ktest.linux_repo, e)); } let repo = repo.unwrap(); @@ -386,11 +420,12 @@ cgi::cgi_main! {|request: cgi::Request| -> cgi::Response { let tests_matching = query.get("test").unwrap_or(&""); let ci = Ci { - ktestrc: ktestrc, + rc: rc, repo: repo, stylesheet: String::from(STYLESHEET), script_name: cgi_header_get(&request, "x-cgi-script-name"), + user: query.get("user").map(|x| x.to_string()), branch: query.get("branch").map(|x| x.to_string()), commit: query.get("commit").map(|x| x.to_string()), tests_matching: Regex::new(tests_matching).unwrap_or(Regex::new("").unwrap()), @@ -400,6 +435,8 @@ cgi::cgi_main! {|request: cgi::Request| -> cgi::Response { ci_commit(&ci) } else if ci.branch.is_some() { ci_log(&ci) + } else if ci.user.is_some() { + ci_user(&ci) } else { ci_home(&ci) } diff --git a/src/bin/gc-results.rs b/src/bin/gc-results.rs index 006004c..1b60a17 100644 --- a/src/bin/gc-results.rs +++ b/src/bin/gc-results.rs @@ -2,7 +2,7 @@ extern crate libc; use std::process; use std::collections::HashSet; use std::fs::DirEntry; -use ci_cgi::{Ktestrc, ktestrc_read, git_get_commit}; +use ci_cgi::{CiConfig, ciconfig_read, git_get_commit}; use clap::Parser; #[derive(Parser)] @@ -36,22 +36,30 @@ fn branch_get_commits(repo: &git2::Repository, .collect() } -fn get_live_commits(rc: &Ktestrc) -> HashSet<String> +fn get_live_commits(rc: &CiConfig) -> HashSet<String> { - let repo = git2::Repository::open(&rc.linux_repo); + let repo = git2::Repository::open(&rc.ktest.linux_repo); if let Err(e) = repo { - eprintln!("Error opening {:?}: {}", rc.linux_repo, e); + eprintln!("Error opening {:?}: {}", rc.ktest.linux_repo, e); eprintln!("Please specify correct linux_repo"); process::exit(1); } let repo = repo.unwrap(); - rc.branch.iter() - .flat_map(move |(branch, branchconfig)| branchconfig.tests.iter() - .filter_map(|i| rc.test_group.get(i)).map(move |test_group| (branch, test_group))) - .map(|(branch, test_group)| branch_get_commits(&repo, &branch, test_group.max_commits)) - .flatten() - .collect() + let mut ret: HashSet<String> = HashSet::new(); + + for (_, user) in rc.users.iter() { + for (branch, branch_config) in user.branch.iter() { + for test_group in branch_config.tests.iter() { + let max_commits = user.test_group.get(test_group).map(|x| x.max_commits).unwrap_or(0); + for commit in branch_get_commits(&repo, &branch, max_commits) { + ret.insert(commit); + } + } + } + } + + ret } fn result_is_live(commits: &HashSet<String>, d: &DirEntry) -> bool { @@ -72,7 +80,7 @@ fn result_is_live(commits: &HashSet<String>, d: &DirEntry) -> bool { fn main() { let args = Args::parse(); - let rc = ktestrc_read(); + let rc = ciconfig_read(); if let Err(e) = rc { eprintln!("could not read config; {}", e); process::exit(1); @@ -81,7 +89,7 @@ fn main() { let commits = get_live_commits(&rc); - for d in rc.output_dir.read_dir().unwrap() + for d in rc.ktest.output_dir.read_dir().unwrap() .filter_map(|d| d.ok()) .filter(|d| !result_is_live(&commits, &d)) .map(|d| d.path()) { diff --git a/src/bin/gen-job-list.rs b/src/bin/gen-job-list.rs index 28cad45..81212bf 100644 --- a/src/bin/gen-job-list.rs +++ b/src/bin/gen-job-list.rs @@ -6,7 +6,7 @@ use std::io::prelude::*; use std::path::{Path, PathBuf}; use std::process; use std::process::Stdio; -use ci_cgi::{Ktestrc, KtestrcTestGroup, ktestrc_read, git_get_commit, commitdir_get_results, lockfile_exists}; +use ci_cgi::{CiConfig, Userrc, RcTestGroup, ciconfig_read, git_get_commit, commitdir_get_results, lockfile_exists}; use ci_cgi::TestResultsMap; use file_lock::{FileLock, FileOptions}; use memoize::memoize; @@ -80,12 +80,12 @@ fn have_result(results: &TestResultsMap, subtest: &str) -> bool { } } -fn branch_test_jobs(rc: &Ktestrc, repo: &git2::Repository, +fn branch_test_jobs(rc: &CiConfig, repo: &git2::Repository, branch: &str, - test_group: &KtestrcTestGroup, + test_group: &RcTestGroup, test_path: &Path, verbose: bool) -> Vec<TestJob> { - let test_path = rc.ktest_dir.join("tests").join(test_path); + let test_path = rc.ktest.ktest_dir.join("tests").join(test_path); let mut ret = Vec::new(); let subtests = get_subtests(test_path.clone()); @@ -113,7 +113,7 @@ fn branch_test_jobs(rc: &Ktestrc, repo: &git2::Repository, .enumerate() { let commit = commit.id().to_string(); - let results = commitdir_get_results(rc, &commit).unwrap_or(BTreeMap::new()); + let results = commitdir_get_results(&rc.ktest, &commit).unwrap_or(BTreeMap::new()); if verbose { eprintln!("at commit {} age {}\nresults {:?}", &commit, age, results) } @@ -124,7 +124,7 @@ fn branch_test_jobs(rc: &Ktestrc, repo: &git2::Repository, let full_subtest_name = subtest_full_name(&test_path, &i); !have_result(&results, &full_subtest_name) && - !lockfile_exists(rc, &commit, &full_subtest_name, false) + !lockfile_exists(&rc.ktest, &commit, &full_subtest_name, false) }) .map(|i| i.clone()) .collect(); @@ -144,11 +144,12 @@ fn branch_test_jobs(rc: &Ktestrc, repo: &git2::Repository, ret } -fn rc_test_jobs(rc: &Ktestrc, repo: &git2::Repository, - verbose: bool) -> Vec<TestJob> { - let mut ret: Vec<_> = rc.branch.iter() +fn user_test_jobs(rc: &CiConfig, repo: &git2::Repository, + user: &Userrc, + verbose: bool) -> Vec<TestJob> { + let mut ret: Vec<_> = user.branch.iter() .flat_map(move |(branch, branchconfig)| branchconfig.tests.iter() - .filter_map(|i| rc.test_group.get(i)).map(move |testgroup| (branch, testgroup))) + .filter_map(|i| user.test_group.get(i)).map(move |testgroup| (branch, testgroup))) .flat_map(move |(branch, testgroup)| testgroup.tests.iter() .flat_map(move |test| branch_test_jobs(rc, repo, &branch, &testgroup, &test, verbose))) .collect(); @@ -160,9 +161,22 @@ fn rc_test_jobs(rc: &Ktestrc, repo: &git2::Repository, ret } -fn write_test_jobs(rc: &Ktestrc, jobs_in: Vec<TestJob>) -> anyhow::Result<()> { - let jobs_fname = rc.output_dir.join("jobs"); - let jobs_fname_new = rc.output_dir.join("jobs.new"); +fn rc_test_jobs(rc: &CiConfig, repo: &git2::Repository, + verbose: bool) -> Vec<TestJob> { + let mut ret: Vec<_> = rc.users.iter() + .flat_map(|(_, user)| user_test_jobs(rc, repo, &user, verbose)) + .collect(); + + /* sort by commit, dedup */ + + ret.sort(); + ret.reverse(); + ret +} + +fn write_test_jobs(rc: &CiConfig, jobs_in: Vec<TestJob>) -> anyhow::Result<()> { + let jobs_fname = rc.ktest.output_dir.join("jobs"); + let jobs_fname_new = rc.ktest.output_dir.join("jobs.new"); let mut jobs_out = std::io::BufWriter::new(File::create(&jobs_fname_new)?); for job in jobs_in.iter() { @@ -185,39 +199,41 @@ fn write_test_jobs(rc: &Ktestrc, jobs_in: Vec<TestJob>) -> anyhow::Result<()> { Ok(()) } -fn fetch_remotes(rc: &Ktestrc, repo: &git2::Repository) -> anyhow::Result<bool> { - fn fetch_remotes_locked(rc: &Ktestrc, repo: &git2::Repository) -> Result<(), git2::Error> { - for (branch, branchconfig) in &rc.branch { - let fetch = branchconfig.fetch - .split_whitespace() - .map(|i| OsStr::new(i)); - - let status = std::process::Command::new("git") - .arg("-C") - .arg(&rc.linux_repo) - .arg("fetch") - .args(fetch) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .status() - .expect(&format!("failed to execute fetch")); - if !status.success() { - eprintln!("fetch error: {}", status); - return Ok(()); +fn fetch_remotes(rc: &CiConfig, repo: &git2::Repository) -> anyhow::Result<bool> { + fn fetch_remotes_locked(rc: &CiConfig, repo: &git2::Repository) -> Result<(), git2::Error> { + for (_, userconfig) in &rc.users { + for (branch, branchconfig) in &userconfig.branch { + let fetch = branchconfig.fetch + .split_whitespace() + .map(|i| OsStr::new(i)); + + let status = std::process::Command::new("git") + .arg("-C") + .arg(&rc.ktest.linux_repo) + .arg("fetch") + .args(fetch) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .status() + .expect(&format!("failed to execute fetch")); + if !status.success() { + eprintln!("fetch error: {}", status); + return Ok(()); + } + + let fetch_head = repo.revparse_single("FETCH_HEAD") + .map_err(|e| { eprintln!("error parsing FETCH_HEAD: {}", e); e})? + .peel_to_commit() + .map_err(|e| { eprintln!("error getting FETCH_HEAD: {}", e); e})?; + + repo.branch(branch, &fetch_head, true)?; } - - let fetch_head = repo.revparse_single("FETCH_HEAD") - .map_err(|e| { eprintln!("error parsing FETCH_HEAD: {}", e); e})? - .peel_to_commit() - .map_err(|e| { eprintln!("error getting FETCH_HEAD: {}", e); e})?; - - repo.branch(branch, &fetch_head, true)?; } Ok(()) } - let lockfile = rc.output_dir.join("fetch.lock"); + let lockfile = rc.ktest.output_dir.join("fetch.lock"); let metadata = std::fs::metadata(&lockfile); if let Ok(metadata) = metadata { let elapsed = metadata.modified().unwrap() @@ -243,12 +259,12 @@ fn fetch_remotes(rc: &Ktestrc, repo: &git2::Repository) -> anyhow::Result<bool> Ok(true) } -fn update_jobs(rc: &Ktestrc, repo: &git2::Repository) -> anyhow::Result<()> { +fn update_jobs(rc: &CiConfig, repo: &git2::Repository) -> anyhow::Result<()> { if !fetch_remotes(rc, repo)? { return Ok(()); } - let lockfile = rc.output_dir.join("jobs.lock"); + let lockfile = rc.ktest.output_dir.join("jobs.lock"); let filelock = FileLock::lock(lockfile, true, FileOptions::new().create(true).write(true))?; let jobs_in = rc_test_jobs(rc, repo, false); @@ -260,20 +276,20 @@ fn update_jobs(rc: &Ktestrc, repo: &git2::Repository) -> anyhow::Result<()> { } fn main() { - let ktestrc = ktestrc_read(); - if let Err(e) = ktestrc { + let rc = ciconfig_read(); + if let Err(e) = rc { eprintln!("could not read config; {}", e); process::exit(1); } - let ktestrc = ktestrc.unwrap(); + let rc = rc.unwrap(); - let repo = git2::Repository::open(&ktestrc.linux_repo); + let repo = git2::Repository::open(&rc.ktest.linux_repo); if let Err(e) = repo { - eprintln!("Error opening {:?}: {}", ktestrc.linux_repo, e); + eprintln!("Error opening {:?}: {}", rc.ktest.linux_repo, e); eprintln!("Please specify correct linux_repo"); process::exit(1); } let repo = repo.unwrap(); - update_jobs(&ktestrc, &repo).ok(); + update_jobs(&rc, &repo).ok(); } diff --git a/src/bin/get-test-job.rs b/src/bin/get-test-job.rs index c394110..11d1817 100644 --- a/src/bin/get-test-job.rs +++ b/src/bin/get-test-job.rs @@ -1,7 +1,7 @@ extern crate libc; use std::path::Path; use std::process; -use ci_cgi::{Ktestrc, ktestrc_read, lockfile_exists}; +use ci_cgi::{Ktestrc, ciconfig_read, lockfile_exists}; use ci_cgi::{Worker, workers_update}; use file_lock::{FileLock, FileOptions}; use clap::Parser; @@ -137,12 +137,13 @@ fn main() { let args = Args::parse(); - let rc = ktestrc_read(); + let rc = ciconfig_read(); if let Err(e) = rc { eprintln!("could not read config; {}", e); process::exit(1); } let rc = rc.unwrap(); + let rc = rc.ktest; let lockfile = rc.output_dir.join("jobs.lock"); let filelock = FileLock::lock(lockfile, true, FileOptions::new().create(true).write(true)).unwrap(); @@ -11,6 +11,9 @@ use anyhow; pub mod testresult_capnp; pub mod worker_capnp; +pub mod users; +pub use users::Userrc; +pub use users::RcTestGroup; pub fn git_get_commit(repo: &git2::Repository, reference: String) -> Result<git2::Commit, git2::Error> { let r = repo.revparse_single(&reference); @@ -28,25 +31,11 @@ pub fn git_get_commit(repo: &git2::Repository, reference: String) -> Result<git2 } #[derive(Deserialize)] -pub struct KtestrcTestGroup { - pub max_commits: u64, - pub priority: u64, - pub tests: Vec<PathBuf>, -} - -#[derive(Deserialize)] -pub struct KtestrcBranch { - pub fetch: String, - pub tests: Vec<String>, -} - -#[derive(Deserialize)] pub struct Ktestrc { pub linux_repo: PathBuf, pub output_dir: PathBuf, pub ktest_dir: PathBuf, - pub test_group: BTreeMap<String, KtestrcTestGroup>, - pub branch: BTreeMap<String, KtestrcBranch>, + pub users_dir: PathBuf, } pub fn ktestrc_read() -> anyhow::Result<Ktestrc> { @@ -56,6 +45,27 @@ pub fn ktestrc_read() -> anyhow::Result<Ktestrc> { Ok(ktestrc) } +pub struct CiConfig { + pub ktest: Ktestrc, + pub users: BTreeMap<String, Userrc>, +} + +pub fn ciconfig_read() -> anyhow::Result<CiConfig> { + let mut rc = CiConfig { + ktest: ktestrc_read()?, + users: BTreeMap::new(), + }; + + for i in std::fs::read_dir(&rc.ktest.users_dir)? + .filter_map(|x| x.ok()) + .map(|i| i.path()){ + rc.users.insert(i.file_stem().unwrap().to_string_lossy().to_string(), + users::userrc_read(&i)?); + } + + Ok(rc) +} + pub use testresult_capnp::test_result::Status as TestStatus; impl TestStatus { diff --git a/src/users.rs b/src/users.rs new file mode 100644 index 0000000..b7c11bd --- /dev/null +++ b/src/users.rs @@ -0,0 +1,32 @@ +use std::collections::BTreeMap; +use std::fs::read_to_string; +use std::path::PathBuf; +use serde_derive::Deserialize; +use toml; +use anyhow; + +#[derive(Deserialize)] +pub struct RcTestGroup { + pub max_commits: u64, + pub priority: u64, + pub tests: Vec<PathBuf>, +} + +#[derive(Deserialize)] +pub struct RcBranch { + pub fetch: String, + pub tests: Vec<String>, +} + +#[derive(Deserialize)] +pub struct Userrc { + pub test_group: BTreeMap<String, RcTestGroup>, + pub branch: BTreeMap<String, RcBranch>, +} + +pub fn userrc_read(path: &PathBuf) -> anyhow::Result<Userrc> { + let config = read_to_string(path)?; + let rc: Userrc = toml::from_str(&config)?; + + Ok(rc) +} |