From a4c7c97110bc51346ee0409e620c9e0f81fbda24 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Mon, 19 Sep 2022 18:46:16 -0400 Subject: ci: New CGI frontend, written in Rust Signed-off-by: Kent Overstreet --- ci-web/.gitignore | 1 + ci-web/Cargo.toml | 12 ++ ci-web/src/main.rs | 406 +++++++++++++++++++++++++++++++++++++++++++++++++ ci/_test-git-branch.sh | 5 +- ci/get-test-job.sh | 2 +- ci/test-job-done.sh | 151 ------------------ 6 files changed, 421 insertions(+), 156 deletions(-) create mode 100644 ci-web/.gitignore create mode 100644 ci-web/Cargo.toml create mode 100644 ci-web/src/main.rs delete mode 100755 ci/test-job-done.sh diff --git a/ci-web/.gitignore b/ci-web/.gitignore new file mode 100644 index 0000000..eb5a316 --- /dev/null +++ b/ci-web/.gitignore @@ -0,0 +1 @@ +target diff --git a/ci-web/Cargo.toml b/ci-web/Cargo.toml new file mode 100644 index 0000000..1b8b1d6 --- /dev/null +++ b/ci-web/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ci-web" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +cgi = "0.6" +git2 = "0.14" +querystring = "1.1.0" +dirs = "4.0.0" diff --git a/ci-web/src/main.rs b/ci-web/src/main.rs new file mode 100644 index 0000000..230689a --- /dev/null +++ b/ci-web/src/main.rs @@ -0,0 +1,406 @@ +use git2::Repository; +use std::fmt::Write; +use std::fs::File; +use std::io::{self, Read, BufRead}; +use std::path::{Path, PathBuf}; +use std::process::Command; +extern crate cgi; +extern crate dirs; +extern crate querystring; + +const COMMIT_FILTER: &str = include_str!("../../ci/commit-filter"); +const STYLESHEET: &str = "/bootstrap.min.css"; + +fn read_file(f: &Path) -> Option { + let mut ret = String::new(); + let mut file = File::open(f).ok()?; + file.read_to_string(&mut ret).ok()?; + Some(ret) +} + +fn read_lines

(filename: P) -> io::Result>> +where P: AsRef, { + let file = File::open(filename)?; + Ok(io::BufReader::new(file).lines()) +} + +fn git_get_commit(repo: &git2::Repository, reference: String) -> Result { + let r = repo.revparse_single(&reference); + if let Err(e) = r { + eprintln!("Error from resolve_reference_from_short_name {} in {}: {}", reference, repo.path().display(), e); + return Err(e); + } + + let r = r.unwrap().peel_to_commit(); + if let Err(e) = r { + eprintln!("Error from peel_to_commit {} in {}: {}", reference, repo.path().display(), e); + return Err(e); + } + r +} + +struct Ktestrc { + ci_linux_repo: PathBuf, + ci_output_dir: PathBuf, + ci_branches_to_test: PathBuf, +} + +fn ktestrc_read() -> Ktestrc { + let mut ktestrc = Ktestrc { + ci_linux_repo: PathBuf::new(), + ci_output_dir: PathBuf::new(), + ci_branches_to_test: PathBuf::new(), + }; + + if let Some(home) = dirs::home_dir() { + ktestrc.ci_branches_to_test = home.join("BRANCHES-TO-TEST"); + } + + fn ktestrc_get(rc: &'static str, var: &'static str) -> Option { + let cmd = format!(". {}; echo -n ${}", rc, var); + + let output = Command::new("sh") + .arg("-c") + .arg(&cmd) + .output() + .expect("failed to execute process /bin/sh") + .stdout; + + let output = String::from_utf8_lossy(&output); + let output = output.trim(); + + if !output.is_empty() { + Some(PathBuf::from(output)) + } else { + None + } + } + + if let Some(v) = ktestrc_get("/etc/ktestrc", "JOBSERVER_LINUX_DIR") { + ktestrc.ci_linux_repo = v; + } + + if let Some(v) = ktestrc_get("/etc/ktestrc", "JOBSERVER_OUTPUT_DIR") { + ktestrc.ci_output_dir = v; + } + + if let Some(v) = ktestrc_get("/etc/ktestrc", "JOBSERVER_BRANCHES_TO_TEST") { + ktestrc.ci_branches_to_test = v; + } + + /* + if let Some(v) = ktestrc_get("$HOME/.ktestrc", "JOBSERVER_LINUX_DIR") { + ktestrc.ci_linux_repo = v; + } + + if let Some(v) = ktestrc_get("$HOME/.ktestrc", "JOBSERVER_OUTPUT_DIR") { + ktestrc.ci_output_dir = v; + } + + if let Some(v) = ktestrc_get("$HOME/.ktestrc", "JOBSERVER_BRANCHES_TO_TEST") { + ktestrc.ci_branches_to_test = v; + } + */ + + ktestrc +} + +#[derive(PartialEq)] +enum TestStatus { + InProgress, + Passed, + Failed, + NotRun, + NotStarted, + Unknown, +} + +impl TestStatus { + fn from_str(status: &str) -> TestStatus { + if status.is_empty() { + TestStatus::InProgress + } else if status.contains("PASSED") { + TestStatus::Passed + } else if status.contains("FAILED") { + TestStatus::Failed + } else if status.contains("NOTRUN") { + TestStatus::NotRun + } else if status.contains("NOT STARTED") { + TestStatus::NotStarted + } else { + TestStatus::Unknown + } + } + + fn to_str(&self) -> &'static str { + match self { + TestStatus::InProgress => "In progress", + TestStatus::Passed => "Passed", + TestStatus::Failed => "Failed", + TestStatus::NotRun => "Not run", + TestStatus::NotStarted => "Not started", + TestStatus::Unknown => "Unknown", + } + } + + fn table_class(&self) -> &'static str { + match self { + TestStatus::InProgress => "table-secondary", + TestStatus::Passed => "table-success", + TestStatus::Failed => "table-danger", + TestStatus::NotRun => "table-secondary", + TestStatus::NotStarted => "table-secondary", + TestStatus::Unknown => "table-secondary", + } + } +} + +struct TestResult { + name: String, + status: TestStatus, + duration: usize, +} + +fn read_test_result(testdir: &std::fs::DirEntry) -> Option { + Some(TestResult { + name: testdir.file_name().into_string().unwrap(), + status: TestStatus::from_str(&read_file(&testdir.path().join("status"))?), + duration: read_file(&testdir.path().join("duration"))?.parse().ok()? + }) +} + +struct Ci { + ktestrc: Ktestrc, + repo: git2::Repository, + stylesheet: String, + script_name: String, +} + +fn commit_get_results(ci: &Ci, commit_id: &String) -> Vec { + let mut dirents: Vec<_> = ci.ktestrc.ci_output_dir.join(commit_id) + .read_dir() + .expect("read_dir call failed") + .filter_map(|i| i.ok()) + .collect(); + + dirents.sort_by_key(|x| x.file_name()); + + dirents.iter().map(|x| read_test_result(x)).filter_map(|i| i).collect() +} + +fn ci_log(ci: &Ci, branch: String) -> cgi::Response { + let mut out = String::new(); + let mut walk = ci.repo.revwalk().unwrap(); + + let reference = git_get_commit(&ci.repo, branch.clone()); + if reference.is_err() { + /* XXX: return a 404 */ + return error_response(format!("commit not found")); + } + let reference = reference.unwrap(); + + if let Err(e) = walk.push(reference.id()) { + return error_response(format!("Error walking {}: {}", branch, e)); + } + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "{}", branch).unwrap(); + writeln!(&mut out, "", ci.stylesheet).unwrap(); + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "

").unwrap(); + writeln!(&mut out, "").unwrap(); + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + + let mut nr_empty = 0; + for commit in walk + .filter_map(|i| i.ok()) + .filter_map(|i| ci.repo.find_commit(i).ok()) { + let id = commit.id().to_string(); + let r = commit_get_results(ci, &id); + + if !r.is_empty() { + if nr_empty != 0 { + writeln!(&mut out, " ... ").unwrap(); + nr_empty = 0; + } + + fn count(r: &Vec, t: TestStatus) -> usize { + r.iter().filter(|x| x.status == t).count() + } + + let message = commit.message().unwrap(); + let subject_len = message.find('\n').unwrap_or(message.len()); + + let duration: usize = r.iter().map(|x| x.duration).sum(); + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "", ci.script_name, id, &id.as_str()[..14]).unwrap(); + writeln!(&mut out, "", &message[..subject_len]).unwrap(); + writeln!(&mut out, "", count(&r, TestStatus::Passed)).unwrap(); + writeln!(&mut out, "", count(&r, TestStatus::Failed)).unwrap(); + writeln!(&mut out, "", count(&r, TestStatus::NotStarted)).unwrap(); + writeln!(&mut out, "", count(&r, TestStatus::NotRun)).unwrap(); + writeln!(&mut out, "", count(&r, TestStatus::InProgress)).unwrap(); + writeln!(&mut out, "", count(&r, TestStatus::Unknown)).unwrap(); + writeln!(&mut out, "", r.len()).unwrap(); + writeln!(&mut out, "", duration).unwrap(); + writeln!(&mut out, "").unwrap(); + } else { + nr_empty += 1; + if nr_empty > 100 { + break; + } + } + } + writeln!(&mut out, "
Commit Description Passed Failed Not started Not run In progress Unknown Total Duration
{} {} {} {} {} {} {} {} {} {}s
").unwrap(); + writeln!(&mut out, "
").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + cgi::html_response(200, out) +} + +fn ci_commit(ci: &Ci, commit_id: String) -> cgi::Response { + let mut out = String::new(); + let commit = git_get_commit(&ci.repo, commit_id.clone()); + if commit.is_err() { + /* XXX: return a 404 */ + return error_response(format!("commit not found")); + } + let commit = commit.unwrap(); + + let message = commit.message().unwrap(); + let subject_len = message.find('\n').unwrap_or(message.len()); + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "{}", &message[..subject_len]).unwrap(); + writeln!(&mut out, "", ci.stylesheet).unwrap(); + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "
").unwrap(); + + writeln!(&mut out, "

{}

", &message[..subject_len]).unwrap(); + + out.push_str(COMMIT_FILTER); + + writeln!(&mut out, "").unwrap(); + + for result in commit_get_results(ci, &commit_id) { + writeln!(&mut out, "", result.status.table_class()).unwrap(); + writeln!(&mut out, "", result.name).unwrap(); + writeln!(&mut out, "", result.status.to_str()).unwrap(); + writeln!(&mut out, "", result.duration).unwrap(); + writeln!(&mut out, "", &commit_id, result.name).unwrap(); + writeln!(&mut out, "", &commit_id, result.name).unwrap(); + writeln!(&mut out, "", &commit_id, result.name).unwrap(); + writeln!(&mut out, "").unwrap(); + } + + writeln!(&mut out, "
{} {} {}s log full log output directory
").unwrap(); + writeln!(&mut out, "
").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + cgi::html_response(200, out) +} + +fn ci_list_branches(ci: &Ci) -> cgi::Response { + let mut out = String::new(); + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "CI branch list").unwrap(); + writeln!(&mut out, "", ci.stylesheet).unwrap(); + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + + if let Ok(lines) = read_lines(&ci.ktestrc.ci_branches_to_test) { + let branches: std::collections::HashSet<_> = lines + .filter_map(|i| i.ok()) + .map(|i| if let Some(w) = i.split_whitespace().nth(0) { Some(String::from(w)) } else { None }) + .filter_map(|i| i) + .collect(); + + let mut branches: Vec<_> = branches.iter().collect(); + branches.sort(); + + for b in branches { + writeln!(&mut out, "", ci.script_name, b, b).unwrap(); + } + } else { + writeln!(&mut out, "(BRANCHES-TO-TEST not found)").unwrap(); + } + + writeln!(&mut out, "
{}
").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + cgi::html_response(200, out) +} + +fn cgi_header_get(request: &cgi::Request, name: &str) -> String { + request.headers().get(name) + .map(|x| x.to_str()) + .transpose().ok().flatten() + .map(|x| x.to_string()) + .unwrap_or(String::new()) +} + +fn error_response(msg: String) -> cgi::Response { + let mut out = String::new(); + writeln!(&mut out, "{}", msg).unwrap(); + let env: Vec<_> = std::env::vars().collect(); + writeln!(&mut out, "env: {:?}", env).unwrap(); + cgi::text_response(200, out) +} + +cgi::cgi_main! {|request: cgi::Request| -> cgi::Response { + let ktestrc = ktestrc_read(); + + if !ktestrc.ci_linux_repo.exists() { + return error_response(format!("required file missing: JOBSERVER_LINUX_DIR (got {:?})", + ktestrc.ci_linux_repo.as_os_str())); + } + + if !ktestrc.ci_output_dir.exists() { + return error_response(format!("required file missing: JOBSERVER_OUTPUT_DIR (got {:?})", + ktestrc.ci_output_dir.as_os_str())); + } + + if !ktestrc.ci_branches_to_test.exists() { + return error_response(format!("required file missing: JOBSERVER_BRANCHES_TO_TEST (got {:?})", + ktestrc.ci_branches_to_test.as_os_str())); + } + + let repo = Repository::open(&ktestrc.ci_linux_repo).unwrap(); + + let ci = Ci { + ktestrc: ktestrc, + repo: repo, + stylesheet: String::from(STYLESHEET), + script_name: cgi_header_get(&request, "x-cgi-script-name"), + }; + + let query = cgi_header_get(&request, "x-cgi-query-string"); + let query: std::collections::HashMap<_, _> = + querystring::querify(&query).into_iter().collect(); + + if let Some(commit) = query.get("commit") { + ci_commit(&ci, commit.to_string()) + } else if let Some(log) = query.get("log") { + ci_log(&ci, log.to_string()) + } else { + ci_list_branches(&ci) + } +} } diff --git a/ci/_test-git-branch.sh b/ci/_test-git-branch.sh index 59094ac..05886a7 100644 --- a/ci/_test-git-branch.sh +++ b/ci/_test-git-branch.sh @@ -37,6 +37,7 @@ COMMIT=${TEST_JOB[1]} TEST_PATH=${TEST_JOB[2]} TEST_NAME=$(basename -s .ktest $TEST_PATH) SUBTESTS=( "${TEST_JOB[@]:3}" ) +OUTPUT=$JOBSERVER_OUTPUT_DIR/$COMMIT if [[ -z $BRANCH ]]; then echo "Error getting test job: need git branch" @@ -108,12 +109,8 @@ done echo "Compressing output" find ktest-out/out -type f -name \*log -print0|xargs -0 brotli --rm -9 -OUTPUT=$JOBSERVER_OUTPUT_DIR/c/$COMMIT ssh $JOBSERVER mkdir -p $OUTPUT echo "Sending results to jobserver" (cd ktest-out/out; tar --create --file - *)| ssh $JOBSERVER "(cd $OUTPUT; tar --extract --file -)" - -echo "Running test-job-done.sh" -ssh $JOBSERVER test-job-done.sh $BRANCH $COMMIT diff --git a/ci/get-test-job.sh b/ci/get-test-job.sh index 8347b08..bdcb888 100755 --- a/ci/get-test-job.sh +++ b/ci/get-test-job.sh @@ -7,4 +7,4 @@ flock --nonblock .git_fetch.lock git fetch --all > /dev/null make -C ~/ktest/lib get-test-job 1>&2 -~/ktest/lib/get-test-job -b ~/BRANCHES-TO-TEST -o $JOBSERVER_OUTPUT_DIR/c +~/ktest/lib/get-test-job -b ~/BRANCHES-TO-TEST -o $JOBSERVER_OUTPUT_DIR diff --git a/ci/test-job-done.sh b/ci/test-job-done.sh deleted file mode 100755 index 33dba57..0000000 --- a/ci/test-job-done.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/bash - -set -o nounset -set -o errexit -set -o errtrace - -[[ -f ~/.ktestrc ]] && . ~/.ktestrc - -CI_DIR=$(dirname "$(readlink -f "$0")") - -cd $JOBSERVER_LINUX_DIR - -BRANCH=$1 -COMMIT=$2 -OUTPUT=$JOBSERVER_OUTPUT_DIR/c/$COMMIT -COMMIT_SUBJECT=$(git log -n1 --pretty=format:%s $COMMIT) - -echo "Generating summary for branch $BRANCH commit $COMMIT" - -set +e -STATUSES=$(find "$OUTPUT" -name status) - -if [[ -n $STATUSES ]]; then - cat $STATUSES|grep -c PASSED > $OUTPUT/nr_passed - cat $STATUSES|grep -c FAILED > $OUTPUT/nr_failed - cat $STATUSES|grep -c NOTRUN > $OUTPUT/nr_notrun - cat $STATUSES|grep -c "NOT STARTED" > $OUTPUT/nr_notstarted - cat $STATUSES|grep -cvE '(PASSED|FAILED|NOTRUN)' > $OUTPUT/nr_unknown - echo $STATUSES|wc -w > $OUTPUT/nr_tests -fi -set -o errexit - -git_commit_html() -{ - echo '' - echo "$COMMIT_SUBJECT" - echo '' - - echo '' - echo '
' - - echo "

" - echo "$COMMIT_SUBJECT" - echo "

" - - cat $CI_DIR/commit-filter - - echo '' - - for STATUSFILE in $(find $OUTPUT -name status|sort); do - STATUS=$(<$STATUSFILE) - TESTNAME=$(basename $(dirname $STATUSFILE)) - STATUSMSG=Unknown - TABLECLASS=table-secondary - - if [[ -f $TESTNAME/duration ]]; then - DURATION=$(<$TESTNAME/duration) - else - DURATION=$(echo $STATUS|grep -Eo '[0-9]+s' || true) - fi - - case $STATUS in - *PASSED*) - STATUSMSG=Passed - TABLECLASS=table-success - ;; - *FAILED*) - STATUSMSG=Failed - TABLECLASS=table-danger - ;; - *NOTRUN*) - STATUSMSG="Not Run" - ;; - *"NOT STARTED"*) - STATUSMSG="Not Started" - ;; - esac - - echo "" - echo "" - echo "" - echo "" - echo "" - echo "" - echo "" - echo "" - done - - echo "
$TESTNAME $STATUSMSG $DURATION log full log output directory
" - echo "
" - echo "" - echo "" -} - -git_commit_html > $OUTPUT/index.html - -git_log_html() -{ - echo '' - echo "$BRANCH" - echo '' - - echo '' - echo '
' - echo '' - - echo "" - echo "" - echo "" - echo "" - echo "" - echo "" - echo "" - echo "" - echo "" - echo "" - - git log --pretty=oneline $BRANCH| - while read LINE; do - COMMIT=$(echo $LINE|cut -d\ -f1) - COMMIT_SHORT=$(echo $LINE|cut -b1-14) - DESCRIPTION=$(echo $LINE|cut -d\ -f2-) - RESULTS=$JOBSERVER_OUTPUT_DIR/c/$COMMIT - - [[ ! -d $RESULTS ]] && break - - if [[ -f $RESULTS/nr_tests ]]; then - echo "" - echo "" - echo "" - echo "" - echo "" - echo "" - echo "" - echo "" - echo "" - echo "" - fi - done - - echo "
Commit Description Passed Failed Not started Not run Unknown Total
$COMMIT_SHORT $DESCRIPTION $(<$RESULTS/nr_passed) $(<$RESULTS/nr_failed) $(<$RESULTS/nr_notstarted) $(<$RESULTS/nr_notrun) $(<$RESULTS/nr_unknown) $(<$RESULTS/nr_tests)
" - echo "
" - echo "" - echo "" -} - -echo "Creating log for $BRANCH" -BRANCH_LOG=$(echo "$BRANCH"|tr / _).html -git_log_html > "$JOBSERVER_OUTPUT_DIR/$BRANCH_LOG" - -echo "Success" -- cgit v1.2.3