diff options
author | Kent Overstreet <kent.overstreet@linux.dev> | 2022-09-19 18:46:16 -0400 |
---|---|---|
committer | Kent Overstreet <kent.overstreet@linux.dev> | 2022-09-21 01:57:24 -0400 |
commit | a4c7c97110bc51346ee0409e620c9e0f81fbda24 (patch) | |
tree | 9a57c891ae4dcf25ce2e37c7e736590bcc0a16c0 | |
parent | 50af205983b3a037a29796a10e7cb14a32cf282e (diff) |
ci: New CGI frontend, written in Rust
Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>
-rw-r--r-- | ci-web/.gitignore | 1 | ||||
-rw-r--r-- | ci-web/Cargo.toml | 12 | ||||
-rw-r--r-- | ci-web/src/main.rs | 406 | ||||
-rw-r--r-- | ci/_test-git-branch.sh | 5 | ||||
-rwxr-xr-x | ci/get-test-job.sh | 2 | ||||
-rwxr-xr-x | ci/test-job-done.sh | 151 |
6 files changed, 421 insertions, 156 deletions
diff --git a/ci-web/.gitignore b/ci-web/.gitignore new file mode 100644 index 0000000..eb5a316 --- /dev/null +++ b/ci-web/.gitignore @@ -0,0 +1 @@ +target diff --git a/ci-web/Cargo.toml b/ci-web/Cargo.toml new file mode 100644 index 0000000..1b8b1d6 --- /dev/null +++ b/ci-web/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ci-web" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +cgi = "0.6" +git2 = "0.14" +querystring = "1.1.0" +dirs = "4.0.0" diff --git a/ci-web/src/main.rs b/ci-web/src/main.rs new file mode 100644 index 0000000..230689a --- /dev/null +++ b/ci-web/src/main.rs @@ -0,0 +1,406 @@ +use git2::Repository; +use std::fmt::Write; +use std::fs::File; +use std::io::{self, Read, BufRead}; +use std::path::{Path, PathBuf}; +use std::process::Command; +extern crate cgi; +extern crate dirs; +extern crate querystring; + +const COMMIT_FILTER: &str = include_str!("../../ci/commit-filter"); +const STYLESHEET: &str = "/bootstrap.min.css"; + +fn read_file(f: &Path) -> Option<String> { + let mut ret = String::new(); + let mut file = File::open(f).ok()?; + file.read_to_string(&mut ret).ok()?; + Some(ret) +} + +fn read_lines<P>(filename: P) -> io::Result<io::Lines<io::BufReader<File>>> +where P: AsRef<Path>, { + let file = File::open(filename)?; + Ok(io::BufReader::new(file).lines()) +} + +fn git_get_commit(repo: &git2::Repository, reference: String) -> Result<git2::Commit, git2::Error> { + let r = repo.revparse_single(&reference); + if let Err(e) = r { + eprintln!("Error from resolve_reference_from_short_name {} in {}: {}", reference, repo.path().display(), e); + return Err(e); + } + + let r = r.unwrap().peel_to_commit(); + if let Err(e) = r { + eprintln!("Error from peel_to_commit {} in {}: {}", reference, repo.path().display(), e); + return Err(e); + } + r +} + +struct Ktestrc { + ci_linux_repo: PathBuf, + ci_output_dir: PathBuf, + ci_branches_to_test: PathBuf, +} + +fn ktestrc_read() -> Ktestrc { + let mut ktestrc = Ktestrc { + ci_linux_repo: PathBuf::new(), + ci_output_dir: PathBuf::new(), + ci_branches_to_test: PathBuf::new(), + }; + + if let Some(home) = dirs::home_dir() { + ktestrc.ci_branches_to_test = home.join("BRANCHES-TO-TEST"); + } + + fn ktestrc_get(rc: &'static str, var: &'static str) -> Option<PathBuf> { + let cmd = format!(". {}; echo -n ${}", rc, var); + + let output = Command::new("sh") + .arg("-c") + .arg(&cmd) + .output() + .expect("failed to execute process /bin/sh") + .stdout; + + let output = String::from_utf8_lossy(&output); + let output = output.trim(); + + if !output.is_empty() { + Some(PathBuf::from(output)) + } else { + None + } + } + + if let Some(v) = ktestrc_get("/etc/ktestrc", "JOBSERVER_LINUX_DIR") { + ktestrc.ci_linux_repo = v; + } + + if let Some(v) = ktestrc_get("/etc/ktestrc", "JOBSERVER_OUTPUT_DIR") { + ktestrc.ci_output_dir = v; + } + + if let Some(v) = ktestrc_get("/etc/ktestrc", "JOBSERVER_BRANCHES_TO_TEST") { + ktestrc.ci_branches_to_test = v; + } + + /* + if let Some(v) = ktestrc_get("$HOME/.ktestrc", "JOBSERVER_LINUX_DIR") { + ktestrc.ci_linux_repo = v; + } + + if let Some(v) = ktestrc_get("$HOME/.ktestrc", "JOBSERVER_OUTPUT_DIR") { + ktestrc.ci_output_dir = v; + } + + if let Some(v) = ktestrc_get("$HOME/.ktestrc", "JOBSERVER_BRANCHES_TO_TEST") { + ktestrc.ci_branches_to_test = v; + } + */ + + ktestrc +} + +#[derive(PartialEq)] +enum TestStatus { + InProgress, + Passed, + Failed, + NotRun, + NotStarted, + Unknown, +} + +impl TestStatus { + fn from_str(status: &str) -> TestStatus { + if status.is_empty() { + TestStatus::InProgress + } else if status.contains("PASSED") { + TestStatus::Passed + } else if status.contains("FAILED") { + TestStatus::Failed + } else if status.contains("NOTRUN") { + TestStatus::NotRun + } else if status.contains("NOT STARTED") { + TestStatus::NotStarted + } else { + TestStatus::Unknown + } + } + + fn to_str(&self) -> &'static str { + match self { + TestStatus::InProgress => "In progress", + TestStatus::Passed => "Passed", + TestStatus::Failed => "Failed", + TestStatus::NotRun => "Not run", + TestStatus::NotStarted => "Not started", + TestStatus::Unknown => "Unknown", + } + } + + fn table_class(&self) -> &'static str { + match self { + TestStatus::InProgress => "table-secondary", + TestStatus::Passed => "table-success", + TestStatus::Failed => "table-danger", + TestStatus::NotRun => "table-secondary", + TestStatus::NotStarted => "table-secondary", + TestStatus::Unknown => "table-secondary", + } + } +} + +struct TestResult { + name: String, + status: TestStatus, + duration: usize, +} + +fn read_test_result(testdir: &std::fs::DirEntry) -> Option<TestResult> { + Some(TestResult { + name: testdir.file_name().into_string().unwrap(), + status: TestStatus::from_str(&read_file(&testdir.path().join("status"))?), + duration: read_file(&testdir.path().join("duration"))?.parse().ok()? + }) +} + +struct Ci { + ktestrc: Ktestrc, + repo: git2::Repository, + stylesheet: String, + script_name: String, +} + +fn commit_get_results(ci: &Ci, commit_id: &String) -> Vec<TestResult> { + let mut dirents: Vec<_> = ci.ktestrc.ci_output_dir.join(commit_id) + .read_dir() + .expect("read_dir call failed") + .filter_map(|i| i.ok()) + .collect(); + + dirents.sort_by_key(|x| x.file_name()); + + dirents.iter().map(|x| read_test_result(x)).filter_map(|i| i).collect() +} + +fn ci_log(ci: &Ci, branch: String) -> cgi::Response { + let mut out = String::new(); + let mut walk = ci.repo.revwalk().unwrap(); + + let reference = git_get_commit(&ci.repo, branch.clone()); + if reference.is_err() { + /* XXX: return a 404 */ + return error_response(format!("commit not found")); + } + let reference = reference.unwrap(); + + if let Err(e) = walk.push(reference.id()) { + return error_response(format!("Error walking {}: {}", branch, e)); + } + + writeln!(&mut out, "<!DOCTYPE HTML>").unwrap(); + writeln!(&mut out, "<html><head><title>{}</title></head>", branch).unwrap(); + writeln!(&mut out, "<link href=\"{}\" rel=\"stylesheet\">", ci.stylesheet).unwrap(); + + writeln!(&mut out, "<body>").unwrap(); + writeln!(&mut out, "<div class=\"container\">").unwrap(); + writeln!(&mut out, "<table class=\"table\">").unwrap(); + + writeln!(&mut out, "<tr>").unwrap(); + writeln!(&mut out, "<th> Commit </th>").unwrap(); + writeln!(&mut out, "<th> Description </th>").unwrap(); + writeln!(&mut out, "<th> Passed </th>").unwrap(); + writeln!(&mut out, "<th> Failed </th>").unwrap(); + writeln!(&mut out, "<th> Not started </th>").unwrap(); + writeln!(&mut out, "<th> Not run </th>").unwrap(); + writeln!(&mut out, "<th> In progress </th>").unwrap(); + writeln!(&mut out, "<th> Unknown </th>").unwrap(); + writeln!(&mut out, "<th> Total </th>").unwrap(); + writeln!(&mut out, "<th> Duration </th>").unwrap(); + writeln!(&mut out, "</tr>").unwrap(); + + let mut nr_empty = 0; + for commit in walk + .filter_map(|i| i.ok()) + .filter_map(|i| ci.repo.find_commit(i).ok()) { + let id = commit.id().to_string(); + let r = commit_get_results(ci, &id); + + if !r.is_empty() { + if nr_empty != 0 { + writeln!(&mut out, "<tr> ... </tr>").unwrap(); + nr_empty = 0; + } + + fn count(r: &Vec<TestResult>, t: TestStatus) -> usize { + r.iter().filter(|x| x.status == t).count() + } + + let message = commit.message().unwrap(); + let subject_len = message.find('\n').unwrap_or(message.len()); + + let duration: usize = r.iter().map(|x| x.duration).sum(); + + writeln!(&mut out, "<tr>").unwrap(); + writeln!(&mut out, "<td> <a href=\"{}?commit={}\">{}</a> </td>", ci.script_name, id, &id.as_str()[..14]).unwrap(); + writeln!(&mut out, "<td> {} </td>", &message[..subject_len]).unwrap(); + writeln!(&mut out, "<td> {} </td>", count(&r, TestStatus::Passed)).unwrap(); + writeln!(&mut out, "<td> {} </td>", count(&r, TestStatus::Failed)).unwrap(); + writeln!(&mut out, "<td> {} </td>", count(&r, TestStatus::NotStarted)).unwrap(); + writeln!(&mut out, "<td> {} </td>", count(&r, TestStatus::NotRun)).unwrap(); + writeln!(&mut out, "<td> {} </td>", count(&r, TestStatus::InProgress)).unwrap(); + writeln!(&mut out, "<td> {} </td>", count(&r, TestStatus::Unknown)).unwrap(); + writeln!(&mut out, "<td> {} </td>", r.len()).unwrap(); + writeln!(&mut out, "<td> {}s </td>", duration).unwrap(); + writeln!(&mut out, "</tr>").unwrap(); + } else { + nr_empty += 1; + if nr_empty > 100 { + break; + } + } + } + writeln!(&mut out, "</table>").unwrap(); + writeln!(&mut out, "</div>").unwrap(); + writeln!(&mut out, "</body>").unwrap(); + writeln!(&mut out, "</html>").unwrap(); + cgi::html_response(200, out) +} + +fn ci_commit(ci: &Ci, commit_id: String) -> cgi::Response { + let mut out = String::new(); + let commit = git_get_commit(&ci.repo, commit_id.clone()); + if commit.is_err() { + /* XXX: return a 404 */ + return error_response(format!("commit not found")); + } + let commit = commit.unwrap(); + + let message = commit.message().unwrap(); + let subject_len = message.find('\n').unwrap_or(message.len()); + + writeln!(&mut out, "<!DOCTYPE HTML>").unwrap(); + writeln!(&mut out, "<html><head><title>{}</title></head>", &message[..subject_len]).unwrap(); + writeln!(&mut out, "<link href=\"{}\" rel=\"stylesheet\">", ci.stylesheet).unwrap(); + + writeln!(&mut out, "<body>").unwrap(); + writeln!(&mut out, "<div class=\"container\">").unwrap(); + + writeln!(&mut out, "<h3><th>{}</th></h3>", &message[..subject_len]).unwrap(); + + out.push_str(COMMIT_FILTER); + + writeln!(&mut out, "<table class=\"table\">").unwrap(); + + for result in commit_get_results(ci, &commit_id) { + writeln!(&mut out, "<tr class={}>", result.status.table_class()).unwrap(); + writeln!(&mut out, "<td> {} </td>", result.name).unwrap(); + writeln!(&mut out, "<td> {} </td>", result.status.to_str()).unwrap(); + writeln!(&mut out, "<td> {}s </td>", result.duration).unwrap(); + writeln!(&mut out, "<td> <a href=/c/{}/{}/log.br> log </a> </td>", &commit_id, result.name).unwrap(); + writeln!(&mut out, "<td> <a href=/c/{}/{}/full_log.br> full log </a> </td>", &commit_id, result.name).unwrap(); + writeln!(&mut out, "<td> <a href=/c/{}/{}> output directory </a> </td>", &commit_id, result.name).unwrap(); + writeln!(&mut out, "</tr>").unwrap(); + } + + writeln!(&mut out, "</table>").unwrap(); + writeln!(&mut out, "</div>").unwrap(); + writeln!(&mut out, "</body>").unwrap(); + writeln!(&mut out, "</html>").unwrap(); + cgi::html_response(200, out) +} + +fn ci_list_branches(ci: &Ci) -> cgi::Response { + let mut out = String::new(); + + writeln!(&mut out, "<!DOCTYPE HTML>").unwrap(); + writeln!(&mut out, "<html><head><title>CI branch list</title></head>").unwrap(); + writeln!(&mut out, "<link href=\"{}\" rel=\"stylesheet\">", ci.stylesheet).unwrap(); + + writeln!(&mut out, "<body>").unwrap(); + writeln!(&mut out, "<table class=\"table\">").unwrap(); + + if let Ok(lines) = read_lines(&ci.ktestrc.ci_branches_to_test) { + let branches: std::collections::HashSet<_> = lines + .filter_map(|i| i.ok()) + .map(|i| if let Some(w) = i.split_whitespace().nth(0) { Some(String::from(w)) } else { None }) + .filter_map(|i| i) + .collect(); + + let mut branches: Vec<_> = branches.iter().collect(); + branches.sort(); + + for b in branches { + writeln!(&mut out, "<tr> <th> <a href={}?log={}>{}</a> </th> </tr>", ci.script_name, b, b).unwrap(); + } + } else { + writeln!(&mut out, "(BRANCHES-TO-TEST not found)").unwrap(); + } + + writeln!(&mut out, "</table>").unwrap(); + writeln!(&mut out, "</div>").unwrap(); + writeln!(&mut out, "</body>").unwrap(); + writeln!(&mut out, "</html>").unwrap(); + cgi::html_response(200, out) +} + +fn cgi_header_get(request: &cgi::Request, name: &str) -> String { + request.headers().get(name) + .map(|x| x.to_str()) + .transpose().ok().flatten() + .map(|x| x.to_string()) + .unwrap_or(String::new()) +} + +fn error_response(msg: String) -> cgi::Response { + let mut out = String::new(); + writeln!(&mut out, "{}", msg).unwrap(); + let env: Vec<_> = std::env::vars().collect(); + writeln!(&mut out, "env: {:?}", env).unwrap(); + cgi::text_response(200, out) +} + +cgi::cgi_main! {|request: cgi::Request| -> cgi::Response { + let ktestrc = ktestrc_read(); + + if !ktestrc.ci_linux_repo.exists() { + return error_response(format!("required file missing: JOBSERVER_LINUX_DIR (got {:?})", + ktestrc.ci_linux_repo.as_os_str())); + } + + if !ktestrc.ci_output_dir.exists() { + return error_response(format!("required file missing: JOBSERVER_OUTPUT_DIR (got {:?})", + ktestrc.ci_output_dir.as_os_str())); + } + + if !ktestrc.ci_branches_to_test.exists() { + return error_response(format!("required file missing: JOBSERVER_BRANCHES_TO_TEST (got {:?})", + ktestrc.ci_branches_to_test.as_os_str())); + } + + let repo = Repository::open(&ktestrc.ci_linux_repo).unwrap(); + + let ci = Ci { + ktestrc: ktestrc, + repo: repo, + stylesheet: String::from(STYLESHEET), + script_name: cgi_header_get(&request, "x-cgi-script-name"), + }; + + let query = cgi_header_get(&request, "x-cgi-query-string"); + let query: std::collections::HashMap<_, _> = + querystring::querify(&query).into_iter().collect(); + + if let Some(commit) = query.get("commit") { + ci_commit(&ci, commit.to_string()) + } else if let Some(log) = query.get("log") { + ci_log(&ci, log.to_string()) + } else { + ci_list_branches(&ci) + } +} } diff --git a/ci/_test-git-branch.sh b/ci/_test-git-branch.sh index 59094ac..05886a7 100644 --- a/ci/_test-git-branch.sh +++ b/ci/_test-git-branch.sh @@ -37,6 +37,7 @@ COMMIT=${TEST_JOB[1]} TEST_PATH=${TEST_JOB[2]} TEST_NAME=$(basename -s .ktest $TEST_PATH) SUBTESTS=( "${TEST_JOB[@]:3}" ) +OUTPUT=$JOBSERVER_OUTPUT_DIR/$COMMIT if [[ -z $BRANCH ]]; then echo "Error getting test job: need git branch" @@ -108,12 +109,8 @@ done echo "Compressing output" find ktest-out/out -type f -name \*log -print0|xargs -0 brotli --rm -9 -OUTPUT=$JOBSERVER_OUTPUT_DIR/c/$COMMIT ssh $JOBSERVER mkdir -p $OUTPUT echo "Sending results to jobserver" (cd ktest-out/out; tar --create --file - *)| ssh $JOBSERVER "(cd $OUTPUT; tar --extract --file -)" - -echo "Running test-job-done.sh" -ssh $JOBSERVER test-job-done.sh $BRANCH $COMMIT diff --git a/ci/get-test-job.sh b/ci/get-test-job.sh index 8347b08..bdcb888 100755 --- a/ci/get-test-job.sh +++ b/ci/get-test-job.sh @@ -7,4 +7,4 @@ flock --nonblock .git_fetch.lock git fetch --all > /dev/null make -C ~/ktest/lib get-test-job 1>&2 -~/ktest/lib/get-test-job -b ~/BRANCHES-TO-TEST -o $JOBSERVER_OUTPUT_DIR/c +~/ktest/lib/get-test-job -b ~/BRANCHES-TO-TEST -o $JOBSERVER_OUTPUT_DIR diff --git a/ci/test-job-done.sh b/ci/test-job-done.sh deleted file mode 100755 index 33dba57..0000000 --- a/ci/test-job-done.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/bash - -set -o nounset -set -o errexit -set -o errtrace - -[[ -f ~/.ktestrc ]] && . ~/.ktestrc - -CI_DIR=$(dirname "$(readlink -f "$0")") - -cd $JOBSERVER_LINUX_DIR - -BRANCH=$1 -COMMIT=$2 -OUTPUT=$JOBSERVER_OUTPUT_DIR/c/$COMMIT -COMMIT_SUBJECT=$(git log -n1 --pretty=format:%s $COMMIT) - -echo "Generating summary for branch $BRANCH commit $COMMIT" - -set +e -STATUSES=$(find "$OUTPUT" -name status) - -if [[ -n $STATUSES ]]; then - cat $STATUSES|grep -c PASSED > $OUTPUT/nr_passed - cat $STATUSES|grep -c FAILED > $OUTPUT/nr_failed - cat $STATUSES|grep -c NOTRUN > $OUTPUT/nr_notrun - cat $STATUSES|grep -c "NOT STARTED" > $OUTPUT/nr_notstarted - cat $STATUSES|grep -cvE '(PASSED|FAILED|NOTRUN)' > $OUTPUT/nr_unknown - echo $STATUSES|wc -w > $OUTPUT/nr_tests -fi -set -o errexit - -git_commit_html() -{ - echo '<!DOCTYPE HTML>' - echo "<html><head><title>$COMMIT_SUBJECT</title></head>" - echo '<link href="../../bootstrap.min.css" rel="stylesheet">' - - echo '<body>' - echo '<div class="container">' - - echo "<h3>" - echo "<th>$COMMIT_SUBJECT</th>" - echo "</h3>" - - cat $CI_DIR/commit-filter - - echo '<table class="table">' - - for STATUSFILE in $(find $OUTPUT -name status|sort); do - STATUS=$(<$STATUSFILE) - TESTNAME=$(basename $(dirname $STATUSFILE)) - STATUSMSG=Unknown - TABLECLASS=table-secondary - - if [[ -f $TESTNAME/duration ]]; then - DURATION=$(<$TESTNAME/duration) - else - DURATION=$(echo $STATUS|grep -Eo '[0-9]+s' || true) - fi - - case $STATUS in - *PASSED*) - STATUSMSG=Passed - TABLECLASS=table-success - ;; - *FAILED*) - STATUSMSG=Failed - TABLECLASS=table-danger - ;; - *NOTRUN*) - STATUSMSG="Not Run" - ;; - *"NOT STARTED"*) - STATUSMSG="Not Started" - ;; - esac - - echo "<tr class=$TABLECLASS>" - echo "<td> $TESTNAME </td>" - echo "<td> $STATUSMSG </td>" - echo "<td> $DURATION </td>" - echo "<td> <a href=$TESTNAME/log.br> log </a> </td>" - echo "<td> <a href=$TESTNAME/full_log.br> full log </a> </td>" - echo "<td> <a href=$TESTNAME> output directory </a> </td>" - echo "</tr>" - done - - echo "</table>" - echo "</div>" - echo "</body>" - echo "</html>" -} - -git_commit_html > $OUTPUT/index.html - -git_log_html() -{ - echo '<!DOCTYPE HTML>' - echo "<html><head><title>$BRANCH</title></head>" - echo '<link href="bootstrap.min.css" rel="stylesheet">' - - echo '<body>' - echo '<div class="container">' - echo '<table class="table">' - - echo "<tr>" - echo "<th> Commit </th>" - echo "<th> Description </th>" - echo "<th> Passed </th>" - echo "<th> Failed </th>" - echo "<th> Not started </th>" - echo "<th> Not run </th>" - echo "<th> Unknown </th>" - echo "<th> Total </th>" - echo "</tr>" - - git log --pretty=oneline $BRANCH| - while read LINE; do - COMMIT=$(echo $LINE|cut -d\ -f1) - COMMIT_SHORT=$(echo $LINE|cut -b1-14) - DESCRIPTION=$(echo $LINE|cut -d\ -f2-) - RESULTS=$JOBSERVER_OUTPUT_DIR/c/$COMMIT - - [[ ! -d $RESULTS ]] && break - - if [[ -f $RESULTS/nr_tests ]]; then - echo "<tr>" - echo "<td> <a href=\"c/$COMMIT\">$COMMIT_SHORT</a> </td>" - echo "<td> $DESCRIPTION </td>" - echo "<td> $(<$RESULTS/nr_passed) </td>" - echo "<td> $(<$RESULTS/nr_failed) </td>" - echo "<td> $(<$RESULTS/nr_notstarted) </td>" - echo "<td> $(<$RESULTS/nr_notrun) </td>" - echo "<td> $(<$RESULTS/nr_unknown) </td>" - echo "<td> $(<$RESULTS/nr_tests) </td>" - echo "</tr>" - fi - done - - echo "</table>" - echo "</div>" - echo "</body>" - echo "</html>" -} - -echo "Creating log for $BRANCH" -BRANCH_LOG=$(echo "$BRANCH"|tr / _).html -git_log_html > "$JOBSERVER_OUTPUT_DIR/$BRANCH_LOG" - -echo "Success" |