From 9b196227dc807aad9069d02a30bb197036328520 Mon Sep 17 00:00:00 2001 From: Kent Overstreet Date: Wed, 3 Jul 2024 12:16:31 -0400 Subject: ci-web: move to toplevel Signed-off-by: Kent Overstreet --- Cargo.lock | 856 +++++++++++++++++++++++++++++++++++ Cargo.toml | 31 ++ ci-web/Cargo.lock | 856 ----------------------------------- ci-web/Cargo.toml | 31 -- ci-web/commit-filter | 70 --- ci-web/src/.gitignore | 1 - ci-web/src/bin/cgi.rs | 406 ----------------- ci-web/src/bin/gc-results.rs | 98 ---- ci-web/src/bin/gen-commit-summary.rs | 22 - ci-web/src/bin/gen-job-list.rs | 279 ------------ ci-web/src/bin/get-test-job.rs | 183 -------- ci-web/src/bin/update-lcov.rs | 22 - ci-web/src/build.rs | 15 - ci-web/src/lib.rs | 399 ---------------- ci-web/src/testresult.capnp | 22 - ci-web/src/worker.capnp | 18 - commit-filter | 70 +++ src/.gitignore | 1 + src/bin/cgi.rs | 406 +++++++++++++++++ src/bin/gc-results.rs | 98 ++++ src/bin/gen-commit-summary.rs | 22 + src/bin/gen-job-list.rs | 279 ++++++++++++ src/bin/get-test-job.rs | 183 ++++++++ src/bin/update-lcov.rs | 22 + src/build.rs | 15 + src/lib.rs | 399 ++++++++++++++++ src/testresult.capnp | 22 + src/worker.capnp | 18 + 28 files changed, 2422 insertions(+), 2422 deletions(-) create mode 100644 Cargo.lock create mode 100644 Cargo.toml delete mode 100644 ci-web/Cargo.lock delete mode 100644 ci-web/Cargo.toml delete mode 100644 ci-web/commit-filter delete mode 100644 ci-web/src/.gitignore delete mode 100644 ci-web/src/bin/cgi.rs delete mode 100644 ci-web/src/bin/gc-results.rs delete mode 100644 ci-web/src/bin/gen-commit-summary.rs delete mode 100644 ci-web/src/bin/gen-job-list.rs delete mode 100644 ci-web/src/bin/get-test-job.rs delete mode 100644 ci-web/src/bin/update-lcov.rs delete mode 100644 ci-web/src/build.rs delete mode 100644 ci-web/src/lib.rs delete mode 100644 ci-web/src/testresult.capnp delete mode 100644 ci-web/src/worker.capnp create mode 100644 commit-filter create mode 100644 src/.gitignore create mode 100644 src/bin/cgi.rs create mode 100644 src/bin/gc-results.rs create mode 100644 src/bin/gen-commit-summary.rs create mode 100644 src/bin/gen-job-list.rs create mode 100644 src/bin/get-test-job.rs create mode 100644 src/bin/update-lcov.rs create mode 100644 src/build.rs create mode 100644 src/lib.rs create mode 100644 src/testresult.capnp create mode 100644 src/worker.capnp diff --git a/Cargo.lock b/Cargo.lock new file mode 100644 index 0000000..c4e4d2d --- /dev/null +++ b/Cargo.lock @@ -0,0 +1,856 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + +[[package]] +name = "anstream" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" + +[[package]] +name = "anstyle-parse" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" +dependencies = [ + "anstyle", + "windows-sys", +] + +[[package]] +name = "anyhow" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" + +[[package]] +name = "autocfg" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" + +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + +[[package]] +name = "bytes" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" + +[[package]] +name = "capnp" +version = "0.19.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de71387912cac7dd3cb7c219e09628411620a18061bba58c71453c26ae7bf66a" +dependencies = [ + "embedded-io", +] + +[[package]] +name = "capnpc" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ba30e0f08582d53c2f3710cf4bb65ff562614b1ba86906d7391adffe189ec" +dependencies = [ + "capnp", +] + +[[package]] +name = "cc" +version = "1.0.104" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74b6a57f98764a267ff415d50a25e6e166f3831a5071af4995296ea97d210490" +dependencies = [ + "jobserver", + "libc", + "once_cell", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cgi" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a71d44b66aa4d9229907c0126271215210037fc179437a2e92746ba6bc6e0a00" +dependencies = [ + "http", +] + +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "js-sys", + "num-traits", + "wasm-bindgen", + "windows-targets", +] + +[[package]] +name = "ci_cgi" +version = "0.1.0" +dependencies = [ + "anyhow", + "capnp", + "capnpc", + "cgi", + "chrono", + "clap", + "die", + "file-lock", + "git2", + "glob", + "libc", + "memmap", + "memoize", + "multimap", + "querystring", + "regex", + "serde", + "serde_derive", + "toml", +] + +[[package]] +name = "clap" +version = "4.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84b3edb18336f4df585bc9aa31dd99c036dfa5dc5e9a2939a722a188f3a8970d" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1c09dd5ada6c6c78075d6fd0da3f90d8080651e2d6cc8eb2f1aaa4034ced708" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.68", +] + +[[package]] +name = "clap_lex" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" + +[[package]] +name = "colorchoice" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" + +[[package]] +name = "core-foundation-sys" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" + +[[package]] +name = "die" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8634d5e6139f7364a4e99bd718b2f511f2f25863146360e70909bc45a016290" + +[[package]] +name = "embedded-io" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" + +[[package]] +name = "file-lock" +version = "2.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "040b48f80a749da50292d0f47a1e2d5bf1d772f52836c07f64bfccc62ba6e664" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "git2" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf7f68c2995f392c49fffb4f95ae2c873297830eb25c6bc4c114ce8f4562acc" +dependencies = [ + "bitflags", + "libc", + "libgit2-sys", + "log", + "openssl-probe", + "openssl-sys", + "url", +] + +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + +[[package]] +name = "idna" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + +[[package]] +name = "is_terminal_polyfill" +version = "1.70.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" + +[[package]] +name = "itoa" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" + +[[package]] +name = "jobserver" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" +dependencies = [ + "libc", +] + +[[package]] +name = "js-sys" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" +dependencies = [ + "wasm-bindgen", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.155" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" + +[[package]] +name = "libgit2-sys" +version = "0.14.2+1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f3d95f6b51075fe9810a7ae22c7095f12b98005ab364d8544797a825ce946a4" +dependencies = [ + "cc", + "libc", + "libssh2-sys", + "libz-sys", + "openssl-sys", + "pkg-config", +] + +[[package]] +name = "libssh2-sys" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b094a36eb4b8b8c8a7b4b8ae43b2944502be3e59cd87687595cf6b0a71b3f4ca" +dependencies = [ + "cc", + "libc", + "libz-sys", + "openssl-sys", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "libz-sys" +version = "1.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "log" +version = "0.4.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" + +[[package]] +name = "memchr" +version = "2.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" + +[[package]] +name = "memmap" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "memoize" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c25d125e4063f313300d87c8f658e5b3d69257095df9a4221c12ba50b0421bff" +dependencies = [ + "lazy_static", + "memoize-inner", +] + +[[package]] +name = "memoize-inner" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8b7d5160e6ffcc59d4c571c38238ec5b7065bc91a5a24f511988dabcddda723" +dependencies = [ + "lazy_static", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" +dependencies = [ + "serde", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "openssl-probe" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" + +[[package]] +name = "openssl-sys" +version = "0.9.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "percent-encoding" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" + +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + +[[package]] +name = "proc-macro2" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "querystring" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9318ead08c799aad12a55a3e78b82e0b6167271ffd1f627b758891282f739187" + +[[package]] +name = "quote" +version = "1.0.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" + +[[package]] +name = "serde" +version = "1.0.203" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.203" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.68", +] + +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.68" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tinyvec" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c55115c6fbe2d2bef26eb09ad74bde02d8255476fc0c7b515ef09fbb35742d82" +dependencies = [ + "tinyvec_macros", +] + +[[package]] +name = "tinyvec_macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" + +[[package]] +name = "toml" +version = "0.5.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" +dependencies = [ + "serde", +] + +[[package]] +name = "unicode-bidi" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "unicode-normalization" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" +dependencies = [ + "tinyvec", +] + +[[package]] +name = "url" +version = "2.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" +dependencies = [ + "form_urlencoded", + "idna", + "percent-encoding", +] + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + +[[package]] +name = "wasm-bindgen" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" +dependencies = [ + "cfg-if", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn 2.0.68", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.68", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.92" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-core" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..a03d115 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,31 @@ +[package] +name = "ci_cgi" +version = "0.1.0" +edition = "2021" +build = "src/build.rs" + +#[workspace] +#members = ["get-test-job", "ci-cgi"] + +[dependencies] +cgi = "0.6" +git2 = "0.16" +querystring = "1.1.0" +multimap = "0.8.3" +die = "0.2.0" +libc = "0.2" +toml = "0.5.9" +serde = "1.0.145" +serde_derive = "1.0.145" +regex = "1" +memoize = "0.3.1" +glob = "0.3.0" +clap = { version = "4.0.32", features = ["derive"] } +file-lock = "2.1.6" +capnp = "0.19.*" +anyhow = "1.0.71" +chrono = "0.4.26" +memmap = "0.7.0" + +[build-dependencies] +capnpc = "0.19.*" diff --git a/ci-web/Cargo.lock b/ci-web/Cargo.lock deleted file mode 100644 index c4e4d2d..0000000 --- a/ci-web/Cargo.lock +++ /dev/null @@ -1,856 +0,0 @@ -# This file is automatically @generated by Cargo. -# It is not intended for manual editing. -version = 3 - -[[package]] -name = "aho-corasick" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" -dependencies = [ - "memchr", -] - -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - -[[package]] -name = "anstream" -version = "0.6.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "is_terminal_polyfill", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" - -[[package]] -name = "anstyle-parse" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" -dependencies = [ - "windows-sys", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" -dependencies = [ - "anstyle", - "windows-sys", -] - -[[package]] -name = "anyhow" -version = "1.0.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" - -[[package]] -name = "autocfg" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" - -[[package]] -name = "bitflags" -version = "1.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" - -[[package]] -name = "bumpalo" -version = "3.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" - -[[package]] -name = "bytes" -version = "1.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" - -[[package]] -name = "capnp" -version = "0.19.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de71387912cac7dd3cb7c219e09628411620a18061bba58c71453c26ae7bf66a" -dependencies = [ - "embedded-io", -] - -[[package]] -name = "capnpc" -version = "0.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c75ba30e0f08582d53c2f3710cf4bb65ff562614b1ba86906d7391adffe189ec" -dependencies = [ - "capnp", -] - -[[package]] -name = "cc" -version = "1.0.104" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74b6a57f98764a267ff415d50a25e6e166f3831a5071af4995296ea97d210490" -dependencies = [ - "jobserver", - "libc", - "once_cell", -] - -[[package]] -name = "cfg-if" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - -[[package]] -name = "cgi" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a71d44b66aa4d9229907c0126271215210037fc179437a2e92746ba6bc6e0a00" -dependencies = [ - "http", -] - -[[package]] -name = "chrono" -version = "0.4.38" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" -dependencies = [ - "android-tzdata", - "iana-time-zone", - "js-sys", - "num-traits", - "wasm-bindgen", - "windows-targets", -] - -[[package]] -name = "ci_cgi" -version = "0.1.0" -dependencies = [ - "anyhow", - "capnp", - "capnpc", - "cgi", - "chrono", - "clap", - "die", - "file-lock", - "git2", - "glob", - "libc", - "memmap", - "memoize", - "multimap", - "querystring", - "regex", - "serde", - "serde_derive", - "toml", -] - -[[package]] -name = "clap" -version = "4.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84b3edb18336f4df585bc9aa31dd99c036dfa5dc5e9a2939a722a188f3a8970d" -dependencies = [ - "clap_builder", - "clap_derive", -] - -[[package]] -name = "clap_builder" -version = "4.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1c09dd5ada6c6c78075d6fd0da3f90d8080651e2d6cc8eb2f1aaa4034ced708" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim", -] - -[[package]] -name = "clap_derive" -version = "4.5.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "syn 2.0.68", -] - -[[package]] -name = "clap_lex" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" - -[[package]] -name = "colorchoice" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" - -[[package]] -name = "core-foundation-sys" -version = "0.8.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f" - -[[package]] -name = "die" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8634d5e6139f7364a4e99bd718b2f511f2f25863146360e70909bc45a016290" - -[[package]] -name = "embedded-io" -version = "0.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edd0f118536f44f5ccd48bcb8b111bdc3de888b58c74639dfb034a357d0f206d" - -[[package]] -name = "file-lock" -version = "2.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "040b48f80a749da50292d0f47a1e2d5bf1d772f52836c07f64bfccc62ba6e664" -dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "form_urlencoded" -version = "1.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" -dependencies = [ - "percent-encoding", -] - -[[package]] -name = "git2" -version = "0.16.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf7f68c2995f392c49fffb4f95ae2c873297830eb25c6bc4c114ce8f4562acc" -dependencies = [ - "bitflags", - "libc", - "libgit2-sys", - "log", - "openssl-probe", - "openssl-sys", - "url", -] - -[[package]] -name = "glob" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" - -[[package]] -name = "heck" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - -[[package]] -name = "http" -version = "0.2.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "iana-time-zone" -version = "0.1.60" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows-core", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - -[[package]] -name = "idna" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6" -dependencies = [ - "unicode-bidi", - "unicode-normalization", -] - -[[package]] -name = "is_terminal_polyfill" -version = "1.70.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" - -[[package]] -name = "itoa" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" - -[[package]] -name = "jobserver" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" -dependencies = [ - "libc", -] - -[[package]] -name = "js-sys" -version = "0.3.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" -dependencies = [ - "wasm-bindgen", -] - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "libc" -version = "0.2.155" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" - -[[package]] -name = "libgit2-sys" -version = "0.14.2+1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f3d95f6b51075fe9810a7ae22c7095f12b98005ab364d8544797a825ce946a4" -dependencies = [ - "cc", - "libc", - "libssh2-sys", - "libz-sys", - "openssl-sys", - "pkg-config", -] - -[[package]] -name = "libssh2-sys" -version = "0.2.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b094a36eb4b8b8c8a7b4b8ae43b2944502be3e59cd87687595cf6b0a71b3f4ca" -dependencies = [ - "cc", - "libc", - "libz-sys", - "openssl-sys", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "libz-sys" -version = "1.1.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c15da26e5af7e25c90b37a2d75cdbf940cf4a55316de9d84c679c9b8bfabf82e" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "log" -version = "0.4.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24" - -[[package]] -name = "memchr" -version = "2.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" - -[[package]] -name = "memmap" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "memoize" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c25d125e4063f313300d87c8f658e5b3d69257095df9a4221c12ba50b0421bff" -dependencies = [ - "lazy_static", - "memoize-inner", -] - -[[package]] -name = "memoize-inner" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8b7d5160e6ffcc59d4c571c38238ec5b7065bc91a5a24f511988dabcddda723" -dependencies = [ - "lazy_static", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "multimap" -version = "0.8.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" -dependencies = [ - "serde", -] - -[[package]] -name = "num-traits" -version = "0.2.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" -dependencies = [ - "autocfg", -] - -[[package]] -name = "once_cell" -version = "1.19.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" - -[[package]] -name = "openssl-probe" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" - -[[package]] -name = "openssl-sys" -version = "0.9.102" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - -[[package]] -name = "percent-encoding" -version = "2.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" - -[[package]] -name = "pkg-config" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" - -[[package]] -name = "proc-macro2" -version = "1.0.86" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" -dependencies = [ - "unicode-ident", -] - -[[package]] -name = "querystring" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9318ead08c799aad12a55a3e78b82e0b6167271ffd1f627b758891282f739187" - -[[package]] -name = "quote" -version = "1.0.36" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" -dependencies = [ - "proc-macro2", -] - -[[package]] -name = "regex" -version = "1.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" -dependencies = [ - "aho-corasick", - "memchr", - "regex-automata", - "regex-syntax", -] - -[[package]] -name = "regex-automata" -version = "0.4.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" -dependencies = [ - "aho-corasick", - "memchr", - "regex-syntax", -] - -[[package]] -name = "regex-syntax" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" - -[[package]] -name = "serde" -version = "1.0.203" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7253ab4de971e72fb7be983802300c30b5a7f0c2e56fab8abfc6a214307c0094" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.203" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "500cbc0ebeb6f46627f50f3f5811ccf6bf00643be300b4c3eabc0ef55dc5b5ba" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.68", -] - -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "syn" -version = "2.0.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - -[[package]] -name = "tinyvec" -version = "1.6.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c55115c6fbe2d2bef26eb09ad74bde02d8255476fc0c7b515ef09fbb35742d82" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" - -[[package]] -name = "toml" -version = "0.5.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4f7f0dd8d50a853a531c426359045b1998f04219d88799810762cd4ad314234" -dependencies = [ - "serde", -] - -[[package]] -name = "unicode-bidi" -version = "0.3.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75" - -[[package]] -name = "unicode-ident" -version = "1.0.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" - -[[package]] -name = "unicode-normalization" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5" -dependencies = [ - "tinyvec", -] - -[[package]] -name = "url" -version = "2.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c" -dependencies = [ - "form_urlencoded", - "idna", - "percent-encoding", -] - -[[package]] -name = "utf8parse" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" - -[[package]] -name = "vcpkg" -version = "0.2.15" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" - -[[package]] -name = "wasm-bindgen" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" -dependencies = [ - "cfg-if", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" -dependencies = [ - "bumpalo", - "log", - "once_cell", - "proc-macro2", - "quote", - "syn 2.0.68", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.68", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.92" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" - -[[package]] -name = "winapi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" -dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", -] - -[[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "windows-core" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-sys" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" -dependencies = [ - "windows-targets", -] - -[[package]] -name = "windows-targets" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" -dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_gnullvm", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", -] - -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" - -[[package]] -name = "windows_aarch64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" - -[[package]] -name = "windows_i686_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" - -[[package]] -name = "windows_i686_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" - -[[package]] -name = "windows_i686_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" - -[[package]] -name = "windows_x86_64_gnu" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" - -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" - -[[package]] -name = "windows_x86_64_msvc" -version = "0.52.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" diff --git a/ci-web/Cargo.toml b/ci-web/Cargo.toml deleted file mode 100644 index a03d115..0000000 --- a/ci-web/Cargo.toml +++ /dev/null @@ -1,31 +0,0 @@ -[package] -name = "ci_cgi" -version = "0.1.0" -edition = "2021" -build = "src/build.rs" - -#[workspace] -#members = ["get-test-job", "ci-cgi"] - -[dependencies] -cgi = "0.6" -git2 = "0.16" -querystring = "1.1.0" -multimap = "0.8.3" -die = "0.2.0" -libc = "0.2" -toml = "0.5.9" -serde = "1.0.145" -serde_derive = "1.0.145" -regex = "1" -memoize = "0.3.1" -glob = "0.3.0" -clap = { version = "4.0.32", features = ["derive"] } -file-lock = "2.1.6" -capnp = "0.19.*" -anyhow = "1.0.71" -chrono = "0.4.26" -memmap = "0.7.0" - -[build-dependencies] -capnpc = "0.19.*" diff --git a/ci-web/commit-filter b/ci-web/commit-filter deleted file mode 100644 index c2abea9..0000000 --- a/ci-web/commit-filter +++ /dev/null @@ -1,70 +0,0 @@ - - - -
- Filter by: -
- - \ No newline at end of file diff --git a/ci-web/src/.gitignore b/ci-web/src/.gitignore deleted file mode 100644 index 83d49a1..0000000 --- a/ci-web/src/.gitignore +++ /dev/null @@ -1 +0,0 @@ -*_capnp.rs diff --git a/ci-web/src/bin/cgi.rs b/ci-web/src/bin/cgi.rs deleted file mode 100644 index 31137ca..0000000 --- a/ci-web/src/bin/cgi.rs +++ /dev/null @@ -1,406 +0,0 @@ -use std::collections::BTreeMap; -use std::fmt::Write; -use regex::Regex; -use chrono::Duration; -extern crate cgi; -extern crate querystring; - -use ci_cgi::{Ktestrc, ktestrc_read, TestResultsMap, TestStatus, commitdir_get_results, git_get_commit, workers_get, update_lcov}; - -const COMMIT_FILTER: &str = include_str!("../../commit-filter"); -const STYLESHEET: &str = "bootstrap.min.css"; - -fn filter_results(r: TestResultsMap, tests_matching: &Regex) -> TestResultsMap { - r.iter() - .filter(|i| tests_matching.is_match(&i.0) ) - .map(|(k, v)| (k.clone(), *v)) - .collect() -} - -struct Ci { - ktestrc: Ktestrc, - repo: git2::Repository, - stylesheet: String, - script_name: String, - - branch: Option, - commit: Option, - tests_matching: Regex, -} - -fn commitdir_get_results_filtered(ci: &Ci, commit_id: &String) -> TestResultsMap { - let results = commitdir_get_results(&ci.ktestrc, commit_id).unwrap_or(BTreeMap::new()); - - filter_results(results, &ci.tests_matching) -} - -struct CommitResults { - id: String, - message: String, - tests: TestResultsMap, -} - -fn commit_get_results(ci: &Ci, commit: &git2::Commit) -> CommitResults { - let id = commit.id().to_string(); - let tests = commitdir_get_results_filtered(ci, &id); - - CommitResults { - id: id, - message: commit.message().unwrap().to_string(), - tests: tests, - } -} - -fn branch_get_results(ci: &Ci) -> Result, String> { - let mut nr_empty = 0; - let mut nr_commits = 0; - let mut ret: Vec = Vec::new(); - - let branch = ci.branch.as_ref().unwrap(); - let mut walk = ci.repo.revwalk().unwrap(); - - let reference = git_get_commit(&ci.repo, branch.clone()); - if reference.is_err() { - /* XXX: return a 404 */ - return Err(format!("commit not found")); - } - let reference = reference.unwrap(); - - if let Err(e) = walk.push(reference.id()) { - return Err(format!("Error walking {}: {}", branch, e)); - } - - for commit in walk - .filter_map(|i| i.ok()) - .filter_map(|i| ci.repo.find_commit(i).ok()) { - let r = commit_get_results(ci, &commit); - - if !r.tests.is_empty() { - nr_empty = 0; - } else { - nr_empty += 1; - if nr_empty > 100 { - break; - } - } - - ret.push(r); - - nr_commits += 1; - if nr_commits > 50 { - break; - } - } - - - while !ret.is_empty() && ret[ret.len() - 1].tests.is_empty() { - ret.pop(); - } - - Ok(ret) -} - -fn ci_log(ci: &Ci) -> cgi::Response { - let mut out = String::new(); - let branch = ci.branch.as_ref().unwrap(); - - let commits = branch_get_results(ci); - if let Err(e) = commits { - return error_response(e); - } - - let commits = commits.unwrap(); - - let mut multiple_test_view = false; - for r in &commits { - if r.tests.len() > 1 { - multiple_test_view = true; - } - } - - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "{}", branch).unwrap(); - writeln!(&mut out, "", ci.stylesheet).unwrap(); - - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "
").unwrap(); - writeln!(&mut out, "").unwrap(); - - - if multiple_test_view { - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - - let mut nr_empty = 0; - for r in &commits { - if !r.tests.is_empty() { - if nr_empty != 0 { - writeln!(&mut out, "", nr_empty).unwrap(); - nr_empty = 0; - } - - fn count(r: &TestResultsMap, t: TestStatus) -> usize { - r.iter().filter(|x| x.1.status == t).count() - } - - let subject_len = r.message.find('\n').unwrap_or(r.message.len()); - - let duration: u64 = r.tests.iter().map(|x| x.1.duration).sum(); - - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "", - ci.script_name, branch, - r.id, &r.id.as_str()[..14]).unwrap(); - writeln!(&mut out, "", &r.message[..subject_len]).unwrap(); - writeln!(&mut out, "", count(&r.tests, TestStatus::Passed)).unwrap(); - writeln!(&mut out, "", count(&r.tests, TestStatus::Failed)).unwrap(); - writeln!(&mut out, "", count(&r.tests, TestStatus::Notstarted)).unwrap(); - writeln!(&mut out, "", count(&r.tests, TestStatus::Notrun)).unwrap(); - writeln!(&mut out, "", count(&r.tests, TestStatus::Inprogress)).unwrap(); - writeln!(&mut out, "", count(&r.tests, TestStatus::Unknown)).unwrap(); - writeln!(&mut out, "", r.tests.len()).unwrap(); - writeln!(&mut out, "", duration).unwrap(); - writeln!(&mut out, "").unwrap(); - } else { - nr_empty += 1; - } - } - } else { - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - - let mut nr_empty = 0; - for r in &commits { - if let Some(t) = r.tests.first_key_value() { - if nr_empty != 0 { - writeln!(&mut out, "", nr_empty).unwrap(); - nr_empty = 0; - } - - let subject_len = r.message.find('\n').unwrap_or(r.message.len()); - - writeln!(&mut out, "", t.1.status.table_class()).unwrap(); - writeln!(&mut out, "", - ci.script_name, branch, - r.id, &r.id.as_str()[..14]).unwrap(); - writeln!(&mut out, "", &r.message[..subject_len]).unwrap(); - writeln!(&mut out, "", t.1.status.to_str()).unwrap(); - writeln!(&mut out, "", t.1.duration).unwrap(); - writeln!(&mut out, "", &r.id, t.0).unwrap(); - writeln!(&mut out, "", &r.id, t.0).unwrap(); - writeln!(&mut out, "", &r.id, t.0).unwrap(); - writeln!(&mut out, "").unwrap(); - } else { - nr_empty += 1; - } - } - } - - writeln!(&mut out, "
Commit Description Passed Failed Not started Not run In progress Unknown Total Duration
({} untested commits)
{} {} {} {} {} {} {} {} {} {}s
Commit Description Status Duration
({} untested commits)
{} {} {} {}s log full log output directory
").unwrap(); - writeln!(&mut out, "
").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - cgi::html_response(200, out) -} - -fn ci_commit(ci: &Ci) -> cgi::Response { - let commit_id = ci.commit.as_ref().unwrap(); - let mut out = String::new(); - let commit = git_get_commit(&ci.repo, commit_id.clone()); - if commit.is_err() { - /* XXX: return a 404 */ - return error_response(format!("commit not found")); - } - let commit = commit.unwrap(); - let commit_id = commit.id().to_string(); // normalize - - let message = commit.message().unwrap(); - let subject_len = message.find('\n').unwrap_or(message.len()); - - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "{}", &message[..subject_len]).unwrap(); - writeln!(&mut out, "", ci.stylesheet).unwrap(); - - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "
").unwrap(); - - writeln!(&mut out, "

{}

", &message[..subject_len]).unwrap(); - - update_lcov(&ci.ktestrc, &commit_id); - - if ci.ktestrc.output_dir.join(&commit_id).join("lcov").exists() { - writeln!(&mut out, "

Code coverage

", &commit_id).unwrap(); - } - - out.push_str(COMMIT_FILTER); - - writeln!(&mut out, "").unwrap(); - - for (name, result) in commitdir_get_results_filtered(ci, &commit_id) { - writeln!(&mut out, "", result.status.table_class()).unwrap(); - writeln!(&mut out, "", name).unwrap(); - writeln!(&mut out, "", result.status.to_str()).unwrap(); - writeln!(&mut out, "", result.duration).unwrap(); - writeln!(&mut out, "", &commit_id, name).unwrap(); - writeln!(&mut out, "", &commit_id, name).unwrap(); - writeln!(&mut out, "", &commit_id, name).unwrap(); - - if let Some(branch) = &ci.branch { - writeln!(&mut out, "", - ci.script_name, &branch, name).unwrap(); - } - - writeln!(&mut out, "").unwrap(); - } - - writeln!(&mut out, "
{} {} {}s log full log output directory git log
").unwrap(); - writeln!(&mut out, "
").unwrap(); - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - cgi::html_response(200, out) -} - -fn ci_list_branches(ci: &Ci, out: &mut String) { - writeln!(out, "
").unwrap(); - - for (b, _) in &ci.ktestrc.branch { - writeln!(out, "", ci.script_name, b, b).unwrap(); - } - - writeln!(out, "
{}
").unwrap(); -} - -fn ci_worker_status(ci: &Ci, out: &mut String) -> Option<()>{ - use chrono::prelude::Utc; - - let workers = workers_get(&ci.ktestrc).ok()?; - - writeln!(out, "
").unwrap(); - - writeln!(out, "").unwrap(); - writeln!(out, "").unwrap(); - writeln!(out, "").unwrap(); - writeln!(out, "").unwrap(); - writeln!(out, "").unwrap(); - writeln!(out, "").unwrap(); - - let now = Utc::now(); - let tests_dir = ci.ktestrc.ktest_dir.clone().into_os_string().into_string().unwrap() + "/tests/"; - - for w in workers { - let elapsed = (now - w.starttime).max(Duration::zero()); - let tests = w.tests.strip_prefix(&tests_dir).unwrap_or(&w.tests); - - writeln!(out, "").unwrap(); - writeln!(out, "", w.hostname, w.workdir).unwrap(); - writeln!(out, "", w.branch, w.age).unwrap(); - writeln!(out, "", tests).unwrap(); - writeln!(out, "", - elapsed.num_hours(), - elapsed.num_minutes() % 60, - elapsed.num_seconds() % 60).unwrap(); - writeln!(out, "").unwrap(); - } - - writeln!(out, "
Host.workdir Commit Tests Elapsed time
{}.{} {}~{} {} {}:{:02}:{:02}
").unwrap(); - - Some(()) -} - -fn ci_home(ci: &Ci) -> cgi::Response { - let mut out = String::new(); - - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "CI branch list").unwrap(); - writeln!(&mut out, "", ci.stylesheet).unwrap(); - - writeln!(&mut out, "").unwrap(); - - ci_list_branches(ci, &mut out); - - ci_worker_status(ci, &mut out); - - writeln!(&mut out, "").unwrap(); - writeln!(&mut out, "").unwrap(); - - cgi::html_response(200, out) -} - -fn cgi_header_get(request: &cgi::Request, name: &str) -> String { - request.headers().get(name) - .map(|x| x.to_str()) - .transpose().ok().flatten() - .map(|x| x.to_string()) - .unwrap_or(String::new()) -} - -fn error_response(msg: String) -> cgi::Response { - let mut out = String::new(); - writeln!(&mut out, "{}", msg).unwrap(); - let env: Vec<_> = std::env::vars().collect(); - writeln!(&mut out, "env: {:?}", env).unwrap(); - cgi::text_response(200, out) -} - -cgi::cgi_main! {|request: cgi::Request| -> cgi::Response { - let ktestrc = ktestrc_read(); - if let Err(e) = ktestrc { - return error_response(format!("could not read config; {}", e)); - } - let ktestrc = ktestrc.unwrap(); - - if !ktestrc.output_dir.exists() { - return error_response(format!("required file missing: JOBSERVER_OUTPUT_DIR (got {:?})", - ktestrc.output_dir)); - } - - unsafe { - git2::opts::set_verify_owner_validation(false) - .expect("set_verify_owner_validation should never fail"); - } - - let repo = git2::Repository::open(&ktestrc.linux_repo); - if let Err(e) = repo { - return error_response(format!("error opening repository {:?}: {}", ktestrc.linux_repo, e)); - } - let repo = repo.unwrap(); - - let query = cgi_header_get(&request, "x-cgi-query-string"); - let query: std::collections::HashMap<_, _> = - querystring::querify(&query).into_iter().collect(); - - let tests_matching = query.get("test").unwrap_or(&""); - - let ci = Ci { - ktestrc: ktestrc, - repo: repo, - stylesheet: String::from(STYLESHEET), - script_name: cgi_header_get(&request, "x-cgi-script-name"), - - branch: query.get("branch").map(|x| x.to_string()), - commit: query.get("commit").map(|x| x.to_string()), - tests_matching: Regex::new(tests_matching).unwrap_or(Regex::new("").unwrap()), - }; - - if ci.commit.is_some() { - ci_commit(&ci) - } else if ci.branch.is_some() { - ci_log(&ci) - } else { - ci_home(&ci) - } -} } diff --git a/ci-web/src/bin/gc-results.rs b/ci-web/src/bin/gc-results.rs deleted file mode 100644 index 006004c..0000000 --- a/ci-web/src/bin/gc-results.rs +++ /dev/null @@ -1,98 +0,0 @@ -extern crate libc; -use std::process; -use std::collections::HashSet; -use std::fs::DirEntry; -use ci_cgi::{Ktestrc, ktestrc_read, git_get_commit}; -use clap::Parser; - -#[derive(Parser)] -#[command(author, version, about, long_about = None)] -struct Args { - #[arg(short, long)] - dry_run: bool, -} - -fn branch_get_commits(repo: &git2::Repository, - branch: &str, - max_commits: u64) -> Vec { - let max_commits = max_commits.try_into().unwrap(); - let mut walk = repo.revwalk().unwrap(); - let reference = git_get_commit(&repo, branch.to_string()); - if reference.is_err() { - eprintln!("branch {} not found", branch); - return Vec::new(); - } - let reference = reference.unwrap(); - - if let Err(e) = walk.push(reference.id()) { - eprintln!("Error walking {}: {}", branch, e); - return Vec::new(); - } - - walk.filter_map(|i| i.ok()) - .take(max_commits) - .filter_map(|i| repo.find_commit(i).ok()) - .map(|i| i.id().to_string()) - .collect() -} - -fn get_live_commits(rc: &Ktestrc) -> HashSet -{ - let repo = git2::Repository::open(&rc.linux_repo); - if let Err(e) = repo { - eprintln!("Error opening {:?}: {}", rc.linux_repo, e); - eprintln!("Please specify correct linux_repo"); - process::exit(1); - } - let repo = repo.unwrap(); - - rc.branch.iter() - .flat_map(move |(branch, branchconfig)| branchconfig.tests.iter() - .filter_map(|i| rc.test_group.get(i)).map(move |test_group| (branch, test_group))) - .map(|(branch, test_group)| branch_get_commits(&repo, &branch, test_group.max_commits)) - .flatten() - .collect() -} - -fn result_is_live(commits: &HashSet, d: &DirEntry) -> bool { - let d = d.file_name().into_string().ok(); - - if let Some(d) = d { - /* If it's not actually a commit, don't delete it: */ - if d.len() < 40 { - return true; - } - - commits.contains(&d[..40].to_string()) - } else { - false - } -} - -fn main() { - let args = Args::parse(); - - let rc = ktestrc_read(); - if let Err(e) = rc { - eprintln!("could not read config; {}", e); - process::exit(1); - } - let rc = rc.unwrap(); - - let commits = get_live_commits(&rc); - - for d in rc.output_dir.read_dir().unwrap() - .filter_map(|d| d.ok()) - .filter(|d| !result_is_live(&commits, &d)) - .map(|d| d.path()) { - println!("Removing: {}", d.to_string_lossy()); - - if !args.dry_run { - if d.is_dir() { - std::fs::remove_dir_all(d).ok(); - } else { - std::fs::remove_file(d).ok(); - } - } - } -} diff --git a/ci-web/src/bin/gen-commit-summary.rs b/ci-web/src/bin/gen-commit-summary.rs deleted file mode 100644 index 8ec43d1..0000000 --- a/ci-web/src/bin/gen-commit-summary.rs +++ /dev/null @@ -1,22 +0,0 @@ -use std::process; -use ci_cgi::{ktestrc_read, commit_update_results_from_fs}; -use clap::Parser; - -#[derive(Parser)] -#[command(author, version, about, long_about = None)] -struct Args { - commit: String, -} - -fn main() { - let args = Args::parse(); - - let ktestrc = ktestrc_read(); - if let Err(e) = ktestrc { - eprintln!("could not read config; {}", e); - process::exit(1); - } - let ktestrc = ktestrc.unwrap(); - - commit_update_results_from_fs(&ktestrc, &args.commit); -} diff --git a/ci-web/src/bin/gen-job-list.rs b/ci-web/src/bin/gen-job-list.rs deleted file mode 100644 index 28cad45..0000000 --- a/ci-web/src/bin/gen-job-list.rs +++ /dev/null @@ -1,279 +0,0 @@ -extern crate libc; -use std::collections::BTreeMap; -use std::ffi::OsStr; -use std::fs::File; -use std::io::prelude::*; -use std::path::{Path, PathBuf}; -use std::process; -use std::process::Stdio; -use ci_cgi::{Ktestrc, KtestrcTestGroup, ktestrc_read, git_get_commit, commitdir_get_results, lockfile_exists}; -use ci_cgi::TestResultsMap; -use file_lock::{FileLock, FileOptions}; -use memoize::memoize; -use anyhow; -use chrono::Utc; - -#[memoize] -fn get_subtests(test_path: PathBuf) -> Vec { - let output = std::process::Command::new(&test_path) - .arg("list-tests") - .output() - .expect(&format!("failed to execute process {:?} ", &test_path)) - .stdout; - let output = String::from_utf8_lossy(&output); - - output - .split_whitespace() - .map(|i| i.to_string()) - .collect() -} - -#[derive(Debug)] -pub struct TestJob { - branch: String, - commit: String, - age: u64, - priority: u64, - test: PathBuf, - subtests: Vec, -} - -fn testjob_weight(j: &TestJob) -> u64 { - j.age + j.priority -} - -use std::cmp::Ordering; - -impl Ord for TestJob { - fn cmp(&self, other: &Self) -> Ordering { - testjob_weight(self).cmp(&testjob_weight(other)) - } -} - -impl PartialOrd for TestJob { - fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } -} - -impl PartialEq for TestJob { - fn eq(&self, other: &Self) -> bool { self.cmp(other) == Ordering::Equal } -} - -impl Eq for TestJob {} - -fn subtest_full_name(test_path: &Path, subtest: &String) -> String { - format!("{}.{}", - test_path.file_stem().unwrap().to_string_lossy(), - subtest.replace("/", ".")) -} - -fn have_result(results: &TestResultsMap, subtest: &str) -> bool { - use ci_cgi::TestStatus; - - let r = results.get(subtest); - if let Some(r) = r { - let elapsed = Utc::now() - r.starttime; - let timeout = chrono::Duration::minutes(30); - - r.status != TestStatus::Inprogress || elapsed < timeout - } else { - false - } -} - -fn branch_test_jobs(rc: &Ktestrc, repo: &git2::Repository, - branch: &str, - test_group: &KtestrcTestGroup, - test_path: &Path, - verbose: bool) -> Vec { - let test_path = rc.ktest_dir.join("tests").join(test_path); - let mut ret = Vec::new(); - - let subtests = get_subtests(test_path.clone()); - - if verbose { eprintln!("looking for tests to run for branch {} test {:?} subtests {:?}", - branch, test_path, subtests) } - - let mut walk = repo.revwalk().unwrap(); - let reference = git_get_commit(&repo, branch.to_string()); - if reference.is_err() { - eprintln!("branch {} not found", branch); - return ret; - } - let reference = reference.unwrap(); - - if let Err(e) = walk.push(reference.id()) { - eprintln!("Error walking {}: {}", branch, e); - return ret; - } - - for (age, commit) in walk - .filter_map(|i| i.ok()) - .filter_map(|i| repo.find_commit(i).ok()) - .take(test_group.max_commits as usize) - .enumerate() { - let commit = commit.id().to_string(); - - let results = commitdir_get_results(rc, &commit).unwrap_or(BTreeMap::new()); - - if verbose { eprintln!("at commit {} age {}\nresults {:?}", - &commit, age, results) } - - let missing_subtests: Vec<_> = subtests - .iter() - .filter(|i| { - let full_subtest_name = subtest_full_name(&test_path, &i); - - !have_result(&results, &full_subtest_name) && - !lockfile_exists(rc, &commit, &full_subtest_name, false) - }) - .map(|i| i.clone()) - .collect(); - - if !missing_subtests.is_empty() { - ret.push(TestJob { - branch: branch.to_string(), - commit: commit.clone(), - age: age as u64, - priority: test_group.priority, - test: test_path.to_path_buf(), - subtests: missing_subtests, - }); - } - } - - ret -} - -fn rc_test_jobs(rc: &Ktestrc, repo: &git2::Repository, - verbose: bool) -> Vec { - let mut ret: Vec<_> = rc.branch.iter() - .flat_map(move |(branch, branchconfig)| branchconfig.tests.iter() - .filter_map(|i| rc.test_group.get(i)).map(move |testgroup| (branch, testgroup))) - .flat_map(move |(branch, testgroup)| testgroup.tests.iter() - .flat_map(move |test| branch_test_jobs(rc, repo, &branch, &testgroup, &test, verbose))) - .collect(); - - /* sort by commit, dedup */ - - ret.sort(); - ret.reverse(); - ret -} - -fn write_test_jobs(rc: &Ktestrc, jobs_in: Vec) -> anyhow::Result<()> { - let jobs_fname = rc.output_dir.join("jobs"); - let jobs_fname_new = rc.output_dir.join("jobs.new"); - let mut jobs_out = std::io::BufWriter::new(File::create(&jobs_fname_new)?); - - for job in jobs_in.iter() { - for subtest in job.subtests.iter() { - let _ = jobs_out.write(job.branch.as_bytes()); - let _ = jobs_out.write(b" "); - let _ = jobs_out.write(job.commit.as_bytes()); - let _ = jobs_out.write(b" "); - let _ = jobs_out.write(job.age.to_string().as_bytes()); - let _ = jobs_out.write(b" "); - let _ = jobs_out.write(job.test.as_os_str().as_encoded_bytes()); - let _ = jobs_out.write(b" "); - let _ = jobs_out.write(subtest.as_bytes()); - let _ = jobs_out.write(b"\n"); - } - } - - drop(jobs_out); - std::fs::rename(jobs_fname_new, jobs_fname)?; - Ok(()) -} - -fn fetch_remotes(rc: &Ktestrc, repo: &git2::Repository) -> anyhow::Result { - fn fetch_remotes_locked(rc: &Ktestrc, repo: &git2::Repository) -> Result<(), git2::Error> { - for (branch, branchconfig) in &rc.branch { - let fetch = branchconfig.fetch - .split_whitespace() - .map(|i| OsStr::new(i)); - - let status = std::process::Command::new("git") - .arg("-C") - .arg(&rc.linux_repo) - .arg("fetch") - .args(fetch) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .status() - .expect(&format!("failed to execute fetch")); - if !status.success() { - eprintln!("fetch error: {}", status); - return Ok(()); - } - - let fetch_head = repo.revparse_single("FETCH_HEAD") - .map_err(|e| { eprintln!("error parsing FETCH_HEAD: {}", e); e})? - .peel_to_commit() - .map_err(|e| { eprintln!("error getting FETCH_HEAD: {}", e); e})?; - - repo.branch(branch, &fetch_head, true)?; - } - - Ok(()) - } - - let lockfile = rc.output_dir.join("fetch.lock"); - let metadata = std::fs::metadata(&lockfile); - if let Ok(metadata) = metadata { - let elapsed = metadata.modified().unwrap() - .elapsed() - .unwrap_or_default(); - - if elapsed < std::time::Duration::from_secs(30) { - return Ok(false); - } - } - - let mut filelock = FileLock::lock(lockfile, false, FileOptions::new().create(true).write(true))?; - - eprint!("Fetching remotes..."); - fetch_remotes_locked(rc, repo)?; - eprintln!(" done"); - - filelock.file.write_all(b"ok")?; /* update lockfile mtime */ - - /* - * XXX: return true only if remotes actually changed - */ - Ok(true) -} - -fn update_jobs(rc: &Ktestrc, repo: &git2::Repository) -> anyhow::Result<()> { - if !fetch_remotes(rc, repo)? { - return Ok(()); - } - - let lockfile = rc.output_dir.join("jobs.lock"); - let filelock = FileLock::lock(lockfile, true, FileOptions::new().create(true).write(true))?; - - let jobs_in = rc_test_jobs(rc, repo, false); - write_test_jobs(rc, jobs_in)?; - - drop(filelock); - - Ok(()) -} - -fn main() { - let ktestrc = ktestrc_read(); - if let Err(e) = ktestrc { - eprintln!("could not read config; {}", e); - process::exit(1); - } - let ktestrc = ktestrc.unwrap(); - - let repo = git2::Repository::open(&ktestrc.linux_repo); - if let Err(e) = repo { - eprintln!("Error opening {:?}: {}", ktestrc.linux_repo, e); - eprintln!("Please specify correct linux_repo"); - process::exit(1); - } - let repo = repo.unwrap(); - - update_jobs(&ktestrc, &repo).ok(); -} diff --git a/ci-web/src/bin/get-test-job.rs b/ci-web/src/bin/get-test-job.rs deleted file mode 100644 index c394110..0000000 --- a/ci-web/src/bin/get-test-job.rs +++ /dev/null @@ -1,183 +0,0 @@ -extern crate libc; -use std::path::Path; -use std::process; -use ci_cgi::{Ktestrc, ktestrc_read, lockfile_exists}; -use ci_cgi::{Worker, workers_update}; -use file_lock::{FileLock, FileOptions}; -use clap::Parser; -use chrono::Utc; - -#[derive(Debug)] -struct TestJob { - branch: String, - commit: String, - age: u64, - test: String, - subtests: Vec, -} - -#[derive(Parser)] -#[command(author, version, about, long_about = None)] -struct Args { - #[arg(short, long)] - dry_run: bool, - - #[arg(short, long)] - verbose: bool, - - hostname: String, - workdir: String, -} - -use memmap::MmapOptions; -use std::fs::OpenOptions; -use std::str; - -fn commit_test_matches(job: &Option, commit: &str, test: &str) -> bool { - if let Some(job) = job { - if job.commit == commit && job.test == test { - return true; - } - } - - false -} - -fn get_test_job(rc: &Ktestrc) -> Option { - let file = OpenOptions::new() - .read(true) - .write(true) - .open(rc.output_dir.join("jobs")).unwrap(); - let map = unsafe { MmapOptions::new().map(&file).unwrap() }; - - let mut len = file.metadata().unwrap().len(); - if len == 0 { - return None; - } - - let mut ret = None; - - for job in map.rsplit(|b| *b == b'\n') { - if job.is_empty() { - continue; - } - - let mut fields = job.split(|b| *b == b' '); - let branch = str::from_utf8(fields.next().unwrap()).unwrap(); - let commit = str::from_utf8(fields.next().unwrap()).unwrap(); - let age_str = str::from_utf8(fields.next().unwrap()).unwrap(); - let age = str::parse::(age_str).unwrap(); - let test = str::from_utf8(fields.next().unwrap()).unwrap(); - let subtest = str::from_utf8(fields.next().unwrap()).unwrap(); - - if ret.is_none() { - ret = Some(TestJob { - branch: branch.to_string(), - commit: commit.to_string(), - test: test.to_string(), - age, - subtests: vec![subtest.to_string()], - }); - - len = job.as_ptr() as u64 - map.as_ptr() as u64; - } else if commit_test_matches(&ret, commit, test) { - if let Some(ref mut r) = ret { - r.subtests.push(subtest.to_string()); - len = job.as_ptr() as u64 - map.as_ptr() as u64; - - if r.subtests.len() > 20 { - break; - } - } - } else { - break; - } - } - - let _ = file.set_len(len); - - ret -} - -fn subtest_full_name(test_path: &Path, subtest: &String) -> String { - format!("{}.{}", - test_path.file_stem().unwrap().to_string_lossy(), - subtest.replace("/", ".")) -} - -fn create_job_lockfiles(rc: &Ktestrc, mut job: TestJob) -> Option { - job.subtests = job.subtests.iter() - .filter(|i| lockfile_exists(rc, &job.commit, - &subtest_full_name(&Path::new(&job.test), &i), true)) - .map(|i| i.to_string()) - .collect(); - - if !job.subtests.is_empty() { Some(job) } else { None } -} - -fn get_and_lock_job(rc: &Ktestrc) -> Option { - loop { - let job = get_test_job(rc); - if let Some(job) = job { - let job = create_job_lockfiles(rc, job); - if job.is_some() { - return job; - } - } else { - return job; - } - - } -} - -fn main() { - std::process::Command::new("gen-job-list") - .output() - .expect("failed to execute gen-job-list"); - - let args = Args::parse(); - - let rc = ktestrc_read(); - if let Err(e) = rc { - eprintln!("could not read config; {}", e); - process::exit(1); - } - let rc = rc.unwrap(); - - let lockfile = rc.output_dir.join("jobs.lock"); - let filelock = FileLock::lock(lockfile, true, FileOptions::new().create(true).write(true)).unwrap(); - - let job = if !args.dry_run { - get_and_lock_job(&rc) - } else { - get_test_job(&rc) - }; - - drop(filelock); - - if let Some(job) = job { - let tests = job.test + " " + &job.subtests.join(" "); - - println!("TEST_JOB {} {} {}", job.branch, job.commit, tests); - - workers_update(&rc, Worker { - hostname: args.hostname, - workdir: args.workdir, - starttime: Utc::now(), - branch: job.branch.clone(), - age: job.age, - commit: job.commit.clone(), - tests: tests.clone(), - }); - } else { - workers_update(&rc, Worker { - hostname: args.hostname, - workdir: args.workdir, - starttime: Utc::now(), - branch: "".to_string(), - age: 0, - commit: "".to_string(), - tests: "".to_string(), - }); - } -} diff --git a/ci-web/src/bin/update-lcov.rs b/ci-web/src/bin/update-lcov.rs deleted file mode 100644 index 3a46812..0000000 --- a/ci-web/src/bin/update-lcov.rs +++ /dev/null @@ -1,22 +0,0 @@ -use std::process; -use ci_cgi::{ktestrc_read, update_lcov}; -use clap::Parser; - -#[derive(Parser)] -#[command(author, version, about, long_about = None)] -struct Args { - commit: String, -} - -fn main() { - let args = Args::parse(); - let ktestrc = ktestrc_read(); - if let Err(e) = ktestrc { - eprintln!("could not read config; {}", e); - process::exit(1); - } - let ktestrc = ktestrc.unwrap(); - - update_lcov(&ktestrc, &args.commit); -} - diff --git a/ci-web/src/build.rs b/ci-web/src/build.rs deleted file mode 100644 index f2e2ec2..0000000 --- a/ci-web/src/build.rs +++ /dev/null @@ -1,15 +0,0 @@ -extern crate capnpc; - -fn main() { - capnpc::CompilerCommand::new() - .output_path(".") - .file("src/testresult.capnp") - .run() - .expect("compiling schema"); - - capnpc::CompilerCommand::new() - .output_path(".") - .file("src/worker.capnp") - .run() - .expect("compiling schema"); -} diff --git a/ci-web/src/lib.rs b/ci-web/src/lib.rs deleted file mode 100644 index 0ba1232..0000000 --- a/ci-web/src/lib.rs +++ /dev/null @@ -1,399 +0,0 @@ -use std::collections::BTreeMap; -use std::fs::{File, OpenOptions, create_dir_all, read_to_string}; -use std::io::ErrorKind; -use std::io::prelude::*; -use std::path::PathBuf; -use std::time::SystemTime; -use die::die; -use serde_derive::Deserialize; -use toml; -use anyhow; - -pub mod testresult_capnp; -pub mod worker_capnp; - -pub fn git_get_commit(repo: &git2::Repository, reference: String) -> Result { - let r = repo.revparse_single(&reference); - if let Err(e) = r { - eprintln!("Error from resolve_reference_from_short_name {} in {}: {}", reference, repo.path().display(), e); - return Err(e); - } - - let r = r.unwrap().peel_to_commit(); - if let Err(e) = r { - eprintln!("Error from peel_to_commit {} in {}: {}", reference, repo.path().display(), e); - return Err(e); - } - r -} - -#[derive(Deserialize)] -pub struct KtestrcTestGroup { - pub max_commits: u64, - pub priority: u64, - pub tests: Vec, -} - -#[derive(Deserialize)] -pub struct KtestrcBranch { - pub fetch: String, - pub tests: Vec, -} - -#[derive(Deserialize)] -pub struct Ktestrc { - pub linux_repo: PathBuf, - pub output_dir: PathBuf, - pub ktest_dir: PathBuf, - pub test_group: BTreeMap, - pub branch: BTreeMap, -} - -pub fn ktestrc_read() -> anyhow::Result { - let config = read_to_string("/etc/ktest-ci.toml")?; - let ktestrc: Ktestrc = toml::from_str(&config)?; - - Ok(ktestrc) -} - -pub use testresult_capnp::test_result::Status as TestStatus; - -impl TestStatus { - fn from_str(status: &str) -> TestStatus { - if status.is_empty() { - TestStatus::Inprogress - } else if status.contains("IN PROGRESS") { - TestStatus::Inprogress - } else if status.contains("PASSED") { - TestStatus::Passed - } else if status.contains("FAILED") { - TestStatus::Failed - } else if status.contains("NOTRUN") { - TestStatus::Notrun - } else if status.contains("NOT STARTED") { - TestStatus::Notstarted - } else { - TestStatus::Unknown - } - } - - pub fn to_str(&self) -> &'static str { - match self { - TestStatus::Inprogress => "In progress", - TestStatus::Passed => "Passed", - TestStatus::Failed => "Failed", - TestStatus::Notrun => "Not run", - TestStatus::Notstarted => "Not started", - TestStatus::Unknown => "Unknown", - } - } - - pub fn table_class(&self) -> &'static str { - match self { - TestStatus::Inprogress => "table-secondary", - TestStatus::Passed => "table-success", - TestStatus::Failed => "table-danger", - TestStatus::Notrun => "table-secondary", - TestStatus::Notstarted => "table-secondary", - TestStatus::Unknown => "table-secondary", - } - } -} - -#[derive(Copy, Clone, Debug)] -pub struct TestResult { - pub status: TestStatus, - pub starttime: DateTime, - pub duration: u64, -} - -pub type TestResultsMap = BTreeMap; - -fn commitdir_get_results_fs(ktestrc: &Ktestrc, commit_id: &String) -> TestResultsMap { - fn read_test_result(testdir: &std::fs::DirEntry) -> Option { - let mut f = File::open(&testdir.path().join("status")).ok()?; - let mut status = String::new(); - f.read_to_string(&mut status).ok()?; - - Some(TestResult { - status: TestStatus::from_str(&status), - starttime: f.metadata().ok()?.modified().ok()?.into(), - duration: read_to_string(&testdir.path().join("duration")).unwrap_or("0".to_string()).parse().unwrap_or(0), - }) - } - - let mut results = BTreeMap::new(); - - let results_dir = ktestrc.output_dir.join(commit_id).read_dir(); - - if let Ok(results_dir) = results_dir { - for d in results_dir.filter_map(|i| i.ok()) { - if let Some(r) = read_test_result(&d) { - results.insert(d.file_name().into_string().unwrap(), r); - } - } - } - - results -} - -use testresult_capnp::test_results; -use capnp::serialize; - -fn results_to_capnp(ktestrc: &Ktestrc, commit_id: &String, results_in: &TestResultsMap) -> anyhow::Result<()> { - let mut message = capnp::message::Builder::new_default(); - let results = message.init_root::(); - let mut result_list = results.init_entries(results_in.len().try_into().unwrap()); - - for (idx, (name, result_in)) in results_in.iter().enumerate() { - let mut result = result_list.reborrow().get(idx.try_into().unwrap()); - - result.set_name(name); - result.set_duration(result_in.duration.try_into().unwrap()); - result.set_status(result_in.status); - } - - let fname = ktestrc.output_dir.join(commit_id.clone() + ".capnp"); - let fname_new = ktestrc.output_dir.join(commit_id.clone() + ".capnp.new"); - - let mut out = File::create(&fname_new)?; - - serialize::write_message(&mut out, &message)?; - drop(out); - std::fs::rename(fname_new, fname)?; - - Ok(()) -} - -pub fn commit_update_results_from_fs(ktestrc: &Ktestrc, commit_id: &String) { - let results = commitdir_get_results_fs(&ktestrc, commit_id); - - results_to_capnp(ktestrc, commit_id, &results) - .map_err(|e| eprintln!("error generating capnp: {}", e)).ok(); -} - -pub fn commitdir_get_results(ktestrc: &Ktestrc, commit_id: &String) -> anyhow::Result { - let f = std::fs::read(ktestrc.output_dir.join(commit_id.to_owned() + ".capnp"))?; - - let message_reader = serialize::read_message_from_flat_slice(&mut &f[..], capnp::message::ReaderOptions::new())?; - let entries = message_reader.get_root::()? - .get_entries()?; - - let mut results = BTreeMap::new(); - for e in entries { - let r = TestResult { - status: e.get_status()?, - starttime: Utc.timestamp_opt(e.get_starttime(), 0).unwrap(), - duration: e.get_duration() - }; - - results.insert(e.get_name()?.to_string()?, r); - } - - Ok(results) -} - -use chrono::{DateTime, TimeZone, Utc}; - -#[derive(Debug)] -pub struct Worker { - pub hostname: String, - pub workdir: String, - pub starttime: DateTime, - pub branch: String, - pub age: u64, - pub commit: String, - pub tests: String, -} - -pub type Workers = Vec; - -use worker_capnp::workers; - -fn workers_parse(f: Vec) -> anyhow::Result { - let message_reader = serialize::read_message_from_flat_slice(&mut &f[..], capnp::message::ReaderOptions::new())?; - let entries = message_reader.get_root::()? - .get_entries()?; - - let workers = entries.iter().map(|e| Worker { - hostname: e.get_hostname().unwrap().to_string().unwrap(), - workdir: e.get_workdir().unwrap().to_string().unwrap(), - starttime: Utc.timestamp_opt(e.get_starttime(), 0).unwrap(), - branch: e.get_branch().unwrap().to_string().unwrap(), - commit: e.get_commit().unwrap().to_string().unwrap(), - age: e.get_age(), - tests: e.get_tests().unwrap().to_string().unwrap(), - }).collect(); - - Ok(workers) -} - -pub fn workers_get(ktestrc: &Ktestrc) -> anyhow::Result { - let f = std::fs::read(ktestrc.output_dir.join("workers.capnp"))?; - - workers_parse(f) -} - -use file_lock::{FileLock, FileOptions}; - -pub fn workers_update(ktestrc: &Ktestrc, n: Worker) -> Option<()> { - let fname = ktestrc.output_dir.join("workers.capnp"); - let foptions = FileOptions::new().read(true).write(true).append(false).create(true); - - let mut filelock = FileLock::lock(fname, true, foptions) - .map_err(|e| eprintln!("error locking workers: {}", e)).ok()?; - - let mut f = Vec::new(); - filelock.file.read_to_end(&mut f).ok()?; - - let mut workers: Workers = workers_parse(f) - .map_err(|e| eprintln!("error parsing workers: {}", e)) - .unwrap_or_default() - .into_iter() - .filter(|w| w.hostname != n.hostname || w.workdir != n.workdir) - .collect(); - - workers.push(n); - - let mut message = capnp::message::Builder::new_default(); - let workers_message = message.init_root::(); - let mut workers_list = workers_message.init_entries(workers.len().try_into().unwrap()); - - for (idx, src) in workers.iter().enumerate() { - let mut dst = workers_list.reborrow().get(idx.try_into().unwrap()); - - dst.set_hostname(&src.hostname); - dst.set_workdir(&src.workdir); - dst.set_starttime(src.starttime.timestamp()); - dst.set_branch(&src.branch); - dst.set_commit(&src.commit); - dst.set_age(src.age); - dst.set_tests(&src.tests); - } - - filelock.file.set_len(0).ok()?; - filelock.file.rewind().ok()?; - - serialize::write_message(&mut filelock.file, &message) - .map_err(|e| eprintln!("error writing workers: {}", e)).ok()?; - - Some(()) -} - -pub fn update_lcov(rc: &Ktestrc, commit_id: &String) -> Option<()> { - let commit_dir = rc.output_dir.join(commit_id); - - if !std::fs::remove_file(commit_dir.join("lcov-stale")).is_ok() { return Some(()); } - - let lockfile = "/home/testdashboard/linux-1-lock"; - let filelock = FileLock::lock(lockfile, true, FileOptions::new().create(true).write(true)).ok()?; - - let mut args = Vec::new(); - - let new_lcov: Vec<_> = std::fs::read_dir(&commit_dir).ok()? - .filter_map(|d| d.ok()) - .filter_map(|d| d.file_name().into_string().ok()) - .filter(|d| d.starts_with("lcov.partial.")) - .collect(); - - for d in &new_lcov { - args.push("--add-tracefile".to_string()); - args.push(d.clone()); - } - - if commit_dir.join("lcov.info").exists() { - args.push("--add-tracefile".to_string()); - args.push("lcov.info".to_string()); - } - - let status = std::process::Command::new("lcov") - .current_dir(&commit_dir) - .arg("--quiet") - .arg("--output-file") - .arg("lcov.info.new") - .args(args) - .status() - .expect(&format!("failed to execute lcov")); - if !status.success() { - eprintln!("lcov error: {}", status); - return Some(()); - } - - std::fs::rename(commit_dir.join("lcov.info.new"), commit_dir.join("lcov.info")).ok()?; - - for d in &new_lcov { std::fs::remove_file(commit_dir.join(d)).ok(); } - - let status = std::process::Command::new("git") - .current_dir("/home/testdashboard/linux-1") - .arg("checkout") - .arg("-f") - .arg(commit_id) - .status() - .expect(&format!("failed to execute genhtml")); - if !status.success() { - eprintln!("git checkout error: {}", status); - return Some(()); - } - - let status = std::process::Command::new("genhtml") - .current_dir("/home/testdashboard/linux-1") - .arg("--output-directory") - .arg(commit_dir.join("lcov")) - .arg(commit_dir.join("lcov.info")) - .status() - .expect(&format!("failed to execute genhtml")); - if !status.success() { - eprintln!("genhtml error: {}", status); - return Some(()); - } - - drop(filelock); - Some(()) -} - -pub fn lockfile_exists(rc: &Ktestrc, commit: &str, test_name: &str, create: bool) -> bool { - let lockfile = rc.output_dir.join(commit).join(test_name).join("status"); - - let timeout = std::time::Duration::from_secs(3600); - let metadata = std::fs::metadata(&lockfile); - - if let Ok(metadata) = metadata { - let elapsed = metadata.modified().unwrap() - .elapsed() - .unwrap_or(std::time::Duration::from_secs(0)); - - if metadata.is_file() && - metadata.len() == 0 && - elapsed > timeout && - std::fs::remove_file(&lockfile).is_ok() { - eprintln!("Deleted stale lock file {:?}, mtime {:?} now {:?} elapsed {:?})", - &lockfile, metadata.modified().unwrap(), - SystemTime::now(), - elapsed); - } - } - - if !create { - lockfile.exists() - } else { - let dir = lockfile.parent().unwrap(); - let r = create_dir_all(dir); - if let Err(e) = r { - if e.kind() != ErrorKind::AlreadyExists { - die!("error creating {:?}: {}", dir, e); - } - } - - let r = OpenOptions::new() - .write(true) - .create_new(true) - .open(&lockfile); - if let Err(ref e) = r { - if e.kind() != ErrorKind::AlreadyExists { - die!("error creating {:?}: {}", lockfile, e); - } - } - - r.is_ok() - } -} diff --git a/ci-web/src/testresult.capnp b/ci-web/src/testresult.capnp deleted file mode 100644 index 5699954..0000000 --- a/ci-web/src/testresult.capnp +++ /dev/null @@ -1,22 +0,0 @@ -@0x9527f7d16acca92e; - -struct TestResult { - name @0: Text; - starttime @3: Int64; - duration @1: UInt64; - status @2: Status; - enum Status { - inprogress @0; - passed @1; - failed @2; - notrun @3; - notstarted @4; - unknown @5; - } -} - -struct TestResults { - entries @0: List(TestResult); -} - -# vim: sts=4:sw=4 diff --git a/ci-web/src/worker.capnp b/ci-web/src/worker.capnp deleted file mode 100644 index 66e7158..0000000 --- a/ci-web/src/worker.capnp +++ /dev/null @@ -1,18 +0,0 @@ -@0xe7ff10499731ce1f; - -struct Worker { - hostname @0: Text; - workdir @1: Text; - starttime @2: Int64; - - branch @3: Text; - commit @4: Text; - age @5: UInt64; - tests @6: Text; -} - -struct Workers { - entries @0: List(Worker); -} - -# vim: sts=4:sw=4 diff --git a/commit-filter b/commit-filter new file mode 100644 index 0000000..c2abea9 --- /dev/null +++ b/commit-filter @@ -0,0 +1,70 @@ + + + +
+ Filter by: +
+ + \ No newline at end of file diff --git a/src/.gitignore b/src/.gitignore new file mode 100644 index 0000000..83d49a1 --- /dev/null +++ b/src/.gitignore @@ -0,0 +1 @@ +*_capnp.rs diff --git a/src/bin/cgi.rs b/src/bin/cgi.rs new file mode 100644 index 0000000..31137ca --- /dev/null +++ b/src/bin/cgi.rs @@ -0,0 +1,406 @@ +use std::collections::BTreeMap; +use std::fmt::Write; +use regex::Regex; +use chrono::Duration; +extern crate cgi; +extern crate querystring; + +use ci_cgi::{Ktestrc, ktestrc_read, TestResultsMap, TestStatus, commitdir_get_results, git_get_commit, workers_get, update_lcov}; + +const COMMIT_FILTER: &str = include_str!("../../commit-filter"); +const STYLESHEET: &str = "bootstrap.min.css"; + +fn filter_results(r: TestResultsMap, tests_matching: &Regex) -> TestResultsMap { + r.iter() + .filter(|i| tests_matching.is_match(&i.0) ) + .map(|(k, v)| (k.clone(), *v)) + .collect() +} + +struct Ci { + ktestrc: Ktestrc, + repo: git2::Repository, + stylesheet: String, + script_name: String, + + branch: Option, + commit: Option, + tests_matching: Regex, +} + +fn commitdir_get_results_filtered(ci: &Ci, commit_id: &String) -> TestResultsMap { + let results = commitdir_get_results(&ci.ktestrc, commit_id).unwrap_or(BTreeMap::new()); + + filter_results(results, &ci.tests_matching) +} + +struct CommitResults { + id: String, + message: String, + tests: TestResultsMap, +} + +fn commit_get_results(ci: &Ci, commit: &git2::Commit) -> CommitResults { + let id = commit.id().to_string(); + let tests = commitdir_get_results_filtered(ci, &id); + + CommitResults { + id: id, + message: commit.message().unwrap().to_string(), + tests: tests, + } +} + +fn branch_get_results(ci: &Ci) -> Result, String> { + let mut nr_empty = 0; + let mut nr_commits = 0; + let mut ret: Vec = Vec::new(); + + let branch = ci.branch.as_ref().unwrap(); + let mut walk = ci.repo.revwalk().unwrap(); + + let reference = git_get_commit(&ci.repo, branch.clone()); + if reference.is_err() { + /* XXX: return a 404 */ + return Err(format!("commit not found")); + } + let reference = reference.unwrap(); + + if let Err(e) = walk.push(reference.id()) { + return Err(format!("Error walking {}: {}", branch, e)); + } + + for commit in walk + .filter_map(|i| i.ok()) + .filter_map(|i| ci.repo.find_commit(i).ok()) { + let r = commit_get_results(ci, &commit); + + if !r.tests.is_empty() { + nr_empty = 0; + } else { + nr_empty += 1; + if nr_empty > 100 { + break; + } + } + + ret.push(r); + + nr_commits += 1; + if nr_commits > 50 { + break; + } + } + + + while !ret.is_empty() && ret[ret.len() - 1].tests.is_empty() { + ret.pop(); + } + + Ok(ret) +} + +fn ci_log(ci: &Ci) -> cgi::Response { + let mut out = String::new(); + let branch = ci.branch.as_ref().unwrap(); + + let commits = branch_get_results(ci); + if let Err(e) = commits { + return error_response(e); + } + + let commits = commits.unwrap(); + + let mut multiple_test_view = false; + for r in &commits { + if r.tests.len() > 1 { + multiple_test_view = true; + } + } + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "{}", branch).unwrap(); + writeln!(&mut out, "", ci.stylesheet).unwrap(); + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "
").unwrap(); + writeln!(&mut out, "").unwrap(); + + + if multiple_test_view { + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + + let mut nr_empty = 0; + for r in &commits { + if !r.tests.is_empty() { + if nr_empty != 0 { + writeln!(&mut out, "", nr_empty).unwrap(); + nr_empty = 0; + } + + fn count(r: &TestResultsMap, t: TestStatus) -> usize { + r.iter().filter(|x| x.1.status == t).count() + } + + let subject_len = r.message.find('\n').unwrap_or(r.message.len()); + + let duration: u64 = r.tests.iter().map(|x| x.1.duration).sum(); + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "", + ci.script_name, branch, + r.id, &r.id.as_str()[..14]).unwrap(); + writeln!(&mut out, "", &r.message[..subject_len]).unwrap(); + writeln!(&mut out, "", count(&r.tests, TestStatus::Passed)).unwrap(); + writeln!(&mut out, "", count(&r.tests, TestStatus::Failed)).unwrap(); + writeln!(&mut out, "", count(&r.tests, TestStatus::Notstarted)).unwrap(); + writeln!(&mut out, "", count(&r.tests, TestStatus::Notrun)).unwrap(); + writeln!(&mut out, "", count(&r.tests, TestStatus::Inprogress)).unwrap(); + writeln!(&mut out, "", count(&r.tests, TestStatus::Unknown)).unwrap(); + writeln!(&mut out, "", r.tests.len()).unwrap(); + writeln!(&mut out, "", duration).unwrap(); + writeln!(&mut out, "").unwrap(); + } else { + nr_empty += 1; + } + } + } else { + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + + let mut nr_empty = 0; + for r in &commits { + if let Some(t) = r.tests.first_key_value() { + if nr_empty != 0 { + writeln!(&mut out, "", nr_empty).unwrap(); + nr_empty = 0; + } + + let subject_len = r.message.find('\n').unwrap_or(r.message.len()); + + writeln!(&mut out, "", t.1.status.table_class()).unwrap(); + writeln!(&mut out, "", + ci.script_name, branch, + r.id, &r.id.as_str()[..14]).unwrap(); + writeln!(&mut out, "", &r.message[..subject_len]).unwrap(); + writeln!(&mut out, "", t.1.status.to_str()).unwrap(); + writeln!(&mut out, "", t.1.duration).unwrap(); + writeln!(&mut out, "", &r.id, t.0).unwrap(); + writeln!(&mut out, "", &r.id, t.0).unwrap(); + writeln!(&mut out, "", &r.id, t.0).unwrap(); + writeln!(&mut out, "").unwrap(); + } else { + nr_empty += 1; + } + } + } + + writeln!(&mut out, "
Commit Description Passed Failed Not started Not run In progress Unknown Total Duration
({} untested commits)
{} {} {} {} {} {} {} {} {} {}s
Commit Description Status Duration
({} untested commits)
{} {} {} {}s log full log output directory
").unwrap(); + writeln!(&mut out, "
").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + cgi::html_response(200, out) +} + +fn ci_commit(ci: &Ci) -> cgi::Response { + let commit_id = ci.commit.as_ref().unwrap(); + let mut out = String::new(); + let commit = git_get_commit(&ci.repo, commit_id.clone()); + if commit.is_err() { + /* XXX: return a 404 */ + return error_response(format!("commit not found")); + } + let commit = commit.unwrap(); + let commit_id = commit.id().to_string(); // normalize + + let message = commit.message().unwrap(); + let subject_len = message.find('\n').unwrap_or(message.len()); + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "{}", &message[..subject_len]).unwrap(); + writeln!(&mut out, "", ci.stylesheet).unwrap(); + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "
").unwrap(); + + writeln!(&mut out, "

{}

", &message[..subject_len]).unwrap(); + + update_lcov(&ci.ktestrc, &commit_id); + + if ci.ktestrc.output_dir.join(&commit_id).join("lcov").exists() { + writeln!(&mut out, "

Code coverage

", &commit_id).unwrap(); + } + + out.push_str(COMMIT_FILTER); + + writeln!(&mut out, "").unwrap(); + + for (name, result) in commitdir_get_results_filtered(ci, &commit_id) { + writeln!(&mut out, "", result.status.table_class()).unwrap(); + writeln!(&mut out, "", name).unwrap(); + writeln!(&mut out, "", result.status.to_str()).unwrap(); + writeln!(&mut out, "", result.duration).unwrap(); + writeln!(&mut out, "", &commit_id, name).unwrap(); + writeln!(&mut out, "", &commit_id, name).unwrap(); + writeln!(&mut out, "", &commit_id, name).unwrap(); + + if let Some(branch) = &ci.branch { + writeln!(&mut out, "", + ci.script_name, &branch, name).unwrap(); + } + + writeln!(&mut out, "").unwrap(); + } + + writeln!(&mut out, "
{} {} {}s log full log output directory git log
").unwrap(); + writeln!(&mut out, "
").unwrap(); + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + cgi::html_response(200, out) +} + +fn ci_list_branches(ci: &Ci, out: &mut String) { + writeln!(out, "
").unwrap(); + + for (b, _) in &ci.ktestrc.branch { + writeln!(out, "", ci.script_name, b, b).unwrap(); + } + + writeln!(out, "
{}
").unwrap(); +} + +fn ci_worker_status(ci: &Ci, out: &mut String) -> Option<()>{ + use chrono::prelude::Utc; + + let workers = workers_get(&ci.ktestrc).ok()?; + + writeln!(out, "
").unwrap(); + + writeln!(out, "").unwrap(); + writeln!(out, "").unwrap(); + writeln!(out, "").unwrap(); + writeln!(out, "").unwrap(); + writeln!(out, "").unwrap(); + writeln!(out, "").unwrap(); + + let now = Utc::now(); + let tests_dir = ci.ktestrc.ktest_dir.clone().into_os_string().into_string().unwrap() + "/tests/"; + + for w in workers { + let elapsed = (now - w.starttime).max(Duration::zero()); + let tests = w.tests.strip_prefix(&tests_dir).unwrap_or(&w.tests); + + writeln!(out, "").unwrap(); + writeln!(out, "", w.hostname, w.workdir).unwrap(); + writeln!(out, "", w.branch, w.age).unwrap(); + writeln!(out, "", tests).unwrap(); + writeln!(out, "", + elapsed.num_hours(), + elapsed.num_minutes() % 60, + elapsed.num_seconds() % 60).unwrap(); + writeln!(out, "").unwrap(); + } + + writeln!(out, "
Host.workdir Commit Tests Elapsed time
{}.{} {}~{} {} {}:{:02}:{:02}
").unwrap(); + + Some(()) +} + +fn ci_home(ci: &Ci) -> cgi::Response { + let mut out = String::new(); + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "CI branch list").unwrap(); + writeln!(&mut out, "", ci.stylesheet).unwrap(); + + writeln!(&mut out, "").unwrap(); + + ci_list_branches(ci, &mut out); + + ci_worker_status(ci, &mut out); + + writeln!(&mut out, "").unwrap(); + writeln!(&mut out, "").unwrap(); + + cgi::html_response(200, out) +} + +fn cgi_header_get(request: &cgi::Request, name: &str) -> String { + request.headers().get(name) + .map(|x| x.to_str()) + .transpose().ok().flatten() + .map(|x| x.to_string()) + .unwrap_or(String::new()) +} + +fn error_response(msg: String) -> cgi::Response { + let mut out = String::new(); + writeln!(&mut out, "{}", msg).unwrap(); + let env: Vec<_> = std::env::vars().collect(); + writeln!(&mut out, "env: {:?}", env).unwrap(); + cgi::text_response(200, out) +} + +cgi::cgi_main! {|request: cgi::Request| -> cgi::Response { + let ktestrc = ktestrc_read(); + if let Err(e) = ktestrc { + return error_response(format!("could not read config; {}", e)); + } + let ktestrc = ktestrc.unwrap(); + + if !ktestrc.output_dir.exists() { + return error_response(format!("required file missing: JOBSERVER_OUTPUT_DIR (got {:?})", + ktestrc.output_dir)); + } + + unsafe { + git2::opts::set_verify_owner_validation(false) + .expect("set_verify_owner_validation should never fail"); + } + + let repo = git2::Repository::open(&ktestrc.linux_repo); + if let Err(e) = repo { + return error_response(format!("error opening repository {:?}: {}", ktestrc.linux_repo, e)); + } + let repo = repo.unwrap(); + + let query = cgi_header_get(&request, "x-cgi-query-string"); + let query: std::collections::HashMap<_, _> = + querystring::querify(&query).into_iter().collect(); + + let tests_matching = query.get("test").unwrap_or(&""); + + let ci = Ci { + ktestrc: ktestrc, + repo: repo, + stylesheet: String::from(STYLESHEET), + script_name: cgi_header_get(&request, "x-cgi-script-name"), + + branch: query.get("branch").map(|x| x.to_string()), + commit: query.get("commit").map(|x| x.to_string()), + tests_matching: Regex::new(tests_matching).unwrap_or(Regex::new("").unwrap()), + }; + + if ci.commit.is_some() { + ci_commit(&ci) + } else if ci.branch.is_some() { + ci_log(&ci) + } else { + ci_home(&ci) + } +} } diff --git a/src/bin/gc-results.rs b/src/bin/gc-results.rs new file mode 100644 index 0000000..006004c --- /dev/null +++ b/src/bin/gc-results.rs @@ -0,0 +1,98 @@ +extern crate libc; +use std::process; +use std::collections::HashSet; +use std::fs::DirEntry; +use ci_cgi::{Ktestrc, ktestrc_read, git_get_commit}; +use clap::Parser; + +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +struct Args { + #[arg(short, long)] + dry_run: bool, +} + +fn branch_get_commits(repo: &git2::Repository, + branch: &str, + max_commits: u64) -> Vec { + let max_commits = max_commits.try_into().unwrap(); + let mut walk = repo.revwalk().unwrap(); + let reference = git_get_commit(&repo, branch.to_string()); + if reference.is_err() { + eprintln!("branch {} not found", branch); + return Vec::new(); + } + let reference = reference.unwrap(); + + if let Err(e) = walk.push(reference.id()) { + eprintln!("Error walking {}: {}", branch, e); + return Vec::new(); + } + + walk.filter_map(|i| i.ok()) + .take(max_commits) + .filter_map(|i| repo.find_commit(i).ok()) + .map(|i| i.id().to_string()) + .collect() +} + +fn get_live_commits(rc: &Ktestrc) -> HashSet +{ + let repo = git2::Repository::open(&rc.linux_repo); + if let Err(e) = repo { + eprintln!("Error opening {:?}: {}", rc.linux_repo, e); + eprintln!("Please specify correct linux_repo"); + process::exit(1); + } + let repo = repo.unwrap(); + + rc.branch.iter() + .flat_map(move |(branch, branchconfig)| branchconfig.tests.iter() + .filter_map(|i| rc.test_group.get(i)).map(move |test_group| (branch, test_group))) + .map(|(branch, test_group)| branch_get_commits(&repo, &branch, test_group.max_commits)) + .flatten() + .collect() +} + +fn result_is_live(commits: &HashSet, d: &DirEntry) -> bool { + let d = d.file_name().into_string().ok(); + + if let Some(d) = d { + /* If it's not actually a commit, don't delete it: */ + if d.len() < 40 { + return true; + } + + commits.contains(&d[..40].to_string()) + } else { + false + } +} + +fn main() { + let args = Args::parse(); + + let rc = ktestrc_read(); + if let Err(e) = rc { + eprintln!("could not read config; {}", e); + process::exit(1); + } + let rc = rc.unwrap(); + + let commits = get_live_commits(&rc); + + for d in rc.output_dir.read_dir().unwrap() + .filter_map(|d| d.ok()) + .filter(|d| !result_is_live(&commits, &d)) + .map(|d| d.path()) { + println!("Removing: {}", d.to_string_lossy()); + + if !args.dry_run { + if d.is_dir() { + std::fs::remove_dir_all(d).ok(); + } else { + std::fs::remove_file(d).ok(); + } + } + } +} diff --git a/src/bin/gen-commit-summary.rs b/src/bin/gen-commit-summary.rs new file mode 100644 index 0000000..8ec43d1 --- /dev/null +++ b/src/bin/gen-commit-summary.rs @@ -0,0 +1,22 @@ +use std::process; +use ci_cgi::{ktestrc_read, commit_update_results_from_fs}; +use clap::Parser; + +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +struct Args { + commit: String, +} + +fn main() { + let args = Args::parse(); + + let ktestrc = ktestrc_read(); + if let Err(e) = ktestrc { + eprintln!("could not read config; {}", e); + process::exit(1); + } + let ktestrc = ktestrc.unwrap(); + + commit_update_results_from_fs(&ktestrc, &args.commit); +} diff --git a/src/bin/gen-job-list.rs b/src/bin/gen-job-list.rs new file mode 100644 index 0000000..28cad45 --- /dev/null +++ b/src/bin/gen-job-list.rs @@ -0,0 +1,279 @@ +extern crate libc; +use std::collections::BTreeMap; +use std::ffi::OsStr; +use std::fs::File; +use std::io::prelude::*; +use std::path::{Path, PathBuf}; +use std::process; +use std::process::Stdio; +use ci_cgi::{Ktestrc, KtestrcTestGroup, ktestrc_read, git_get_commit, commitdir_get_results, lockfile_exists}; +use ci_cgi::TestResultsMap; +use file_lock::{FileLock, FileOptions}; +use memoize::memoize; +use anyhow; +use chrono::Utc; + +#[memoize] +fn get_subtests(test_path: PathBuf) -> Vec { + let output = std::process::Command::new(&test_path) + .arg("list-tests") + .output() + .expect(&format!("failed to execute process {:?} ", &test_path)) + .stdout; + let output = String::from_utf8_lossy(&output); + + output + .split_whitespace() + .map(|i| i.to_string()) + .collect() +} + +#[derive(Debug)] +pub struct TestJob { + branch: String, + commit: String, + age: u64, + priority: u64, + test: PathBuf, + subtests: Vec, +} + +fn testjob_weight(j: &TestJob) -> u64 { + j.age + j.priority +} + +use std::cmp::Ordering; + +impl Ord for TestJob { + fn cmp(&self, other: &Self) -> Ordering { + testjob_weight(self).cmp(&testjob_weight(other)) + } +} + +impl PartialOrd for TestJob { + fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) } +} + +impl PartialEq for TestJob { + fn eq(&self, other: &Self) -> bool { self.cmp(other) == Ordering::Equal } +} + +impl Eq for TestJob {} + +fn subtest_full_name(test_path: &Path, subtest: &String) -> String { + format!("{}.{}", + test_path.file_stem().unwrap().to_string_lossy(), + subtest.replace("/", ".")) +} + +fn have_result(results: &TestResultsMap, subtest: &str) -> bool { + use ci_cgi::TestStatus; + + let r = results.get(subtest); + if let Some(r) = r { + let elapsed = Utc::now() - r.starttime; + let timeout = chrono::Duration::minutes(30); + + r.status != TestStatus::Inprogress || elapsed < timeout + } else { + false + } +} + +fn branch_test_jobs(rc: &Ktestrc, repo: &git2::Repository, + branch: &str, + test_group: &KtestrcTestGroup, + test_path: &Path, + verbose: bool) -> Vec { + let test_path = rc.ktest_dir.join("tests").join(test_path); + let mut ret = Vec::new(); + + let subtests = get_subtests(test_path.clone()); + + if verbose { eprintln!("looking for tests to run for branch {} test {:?} subtests {:?}", + branch, test_path, subtests) } + + let mut walk = repo.revwalk().unwrap(); + let reference = git_get_commit(&repo, branch.to_string()); + if reference.is_err() { + eprintln!("branch {} not found", branch); + return ret; + } + let reference = reference.unwrap(); + + if let Err(e) = walk.push(reference.id()) { + eprintln!("Error walking {}: {}", branch, e); + return ret; + } + + for (age, commit) in walk + .filter_map(|i| i.ok()) + .filter_map(|i| repo.find_commit(i).ok()) + .take(test_group.max_commits as usize) + .enumerate() { + let commit = commit.id().to_string(); + + let results = commitdir_get_results(rc, &commit).unwrap_or(BTreeMap::new()); + + if verbose { eprintln!("at commit {} age {}\nresults {:?}", + &commit, age, results) } + + let missing_subtests: Vec<_> = subtests + .iter() + .filter(|i| { + let full_subtest_name = subtest_full_name(&test_path, &i); + + !have_result(&results, &full_subtest_name) && + !lockfile_exists(rc, &commit, &full_subtest_name, false) + }) + .map(|i| i.clone()) + .collect(); + + if !missing_subtests.is_empty() { + ret.push(TestJob { + branch: branch.to_string(), + commit: commit.clone(), + age: age as u64, + priority: test_group.priority, + test: test_path.to_path_buf(), + subtests: missing_subtests, + }); + } + } + + ret +} + +fn rc_test_jobs(rc: &Ktestrc, repo: &git2::Repository, + verbose: bool) -> Vec { + let mut ret: Vec<_> = rc.branch.iter() + .flat_map(move |(branch, branchconfig)| branchconfig.tests.iter() + .filter_map(|i| rc.test_group.get(i)).map(move |testgroup| (branch, testgroup))) + .flat_map(move |(branch, testgroup)| testgroup.tests.iter() + .flat_map(move |test| branch_test_jobs(rc, repo, &branch, &testgroup, &test, verbose))) + .collect(); + + /* sort by commit, dedup */ + + ret.sort(); + ret.reverse(); + ret +} + +fn write_test_jobs(rc: &Ktestrc, jobs_in: Vec) -> anyhow::Result<()> { + let jobs_fname = rc.output_dir.join("jobs"); + let jobs_fname_new = rc.output_dir.join("jobs.new"); + let mut jobs_out = std::io::BufWriter::new(File::create(&jobs_fname_new)?); + + for job in jobs_in.iter() { + for subtest in job.subtests.iter() { + let _ = jobs_out.write(job.branch.as_bytes()); + let _ = jobs_out.write(b" "); + let _ = jobs_out.write(job.commit.as_bytes()); + let _ = jobs_out.write(b" "); + let _ = jobs_out.write(job.age.to_string().as_bytes()); + let _ = jobs_out.write(b" "); + let _ = jobs_out.write(job.test.as_os_str().as_encoded_bytes()); + let _ = jobs_out.write(b" "); + let _ = jobs_out.write(subtest.as_bytes()); + let _ = jobs_out.write(b"\n"); + } + } + + drop(jobs_out); + std::fs::rename(jobs_fname_new, jobs_fname)?; + Ok(()) +} + +fn fetch_remotes(rc: &Ktestrc, repo: &git2::Repository) -> anyhow::Result { + fn fetch_remotes_locked(rc: &Ktestrc, repo: &git2::Repository) -> Result<(), git2::Error> { + for (branch, branchconfig) in &rc.branch { + let fetch = branchconfig.fetch + .split_whitespace() + .map(|i| OsStr::new(i)); + + let status = std::process::Command::new("git") + .arg("-C") + .arg(&rc.linux_repo) + .arg("fetch") + .args(fetch) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .status() + .expect(&format!("failed to execute fetch")); + if !status.success() { + eprintln!("fetch error: {}", status); + return Ok(()); + } + + let fetch_head = repo.revparse_single("FETCH_HEAD") + .map_err(|e| { eprintln!("error parsing FETCH_HEAD: {}", e); e})? + .peel_to_commit() + .map_err(|e| { eprintln!("error getting FETCH_HEAD: {}", e); e})?; + + repo.branch(branch, &fetch_head, true)?; + } + + Ok(()) + } + + let lockfile = rc.output_dir.join("fetch.lock"); + let metadata = std::fs::metadata(&lockfile); + if let Ok(metadata) = metadata { + let elapsed = metadata.modified().unwrap() + .elapsed() + .unwrap_or_default(); + + if elapsed < std::time::Duration::from_secs(30) { + return Ok(false); + } + } + + let mut filelock = FileLock::lock(lockfile, false, FileOptions::new().create(true).write(true))?; + + eprint!("Fetching remotes..."); + fetch_remotes_locked(rc, repo)?; + eprintln!(" done"); + + filelock.file.write_all(b"ok")?; /* update lockfile mtime */ + + /* + * XXX: return true only if remotes actually changed + */ + Ok(true) +} + +fn update_jobs(rc: &Ktestrc, repo: &git2::Repository) -> anyhow::Result<()> { + if !fetch_remotes(rc, repo)? { + return Ok(()); + } + + let lockfile = rc.output_dir.join("jobs.lock"); + let filelock = FileLock::lock(lockfile, true, FileOptions::new().create(true).write(true))?; + + let jobs_in = rc_test_jobs(rc, repo, false); + write_test_jobs(rc, jobs_in)?; + + drop(filelock); + + Ok(()) +} + +fn main() { + let ktestrc = ktestrc_read(); + if let Err(e) = ktestrc { + eprintln!("could not read config; {}", e); + process::exit(1); + } + let ktestrc = ktestrc.unwrap(); + + let repo = git2::Repository::open(&ktestrc.linux_repo); + if let Err(e) = repo { + eprintln!("Error opening {:?}: {}", ktestrc.linux_repo, e); + eprintln!("Please specify correct linux_repo"); + process::exit(1); + } + let repo = repo.unwrap(); + + update_jobs(&ktestrc, &repo).ok(); +} diff --git a/src/bin/get-test-job.rs b/src/bin/get-test-job.rs new file mode 100644 index 0000000..c394110 --- /dev/null +++ b/src/bin/get-test-job.rs @@ -0,0 +1,183 @@ +extern crate libc; +use std::path::Path; +use std::process; +use ci_cgi::{Ktestrc, ktestrc_read, lockfile_exists}; +use ci_cgi::{Worker, workers_update}; +use file_lock::{FileLock, FileOptions}; +use clap::Parser; +use chrono::Utc; + +#[derive(Debug)] +struct TestJob { + branch: String, + commit: String, + age: u64, + test: String, + subtests: Vec, +} + +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +struct Args { + #[arg(short, long)] + dry_run: bool, + + #[arg(short, long)] + verbose: bool, + + hostname: String, + workdir: String, +} + +use memmap::MmapOptions; +use std::fs::OpenOptions; +use std::str; + +fn commit_test_matches(job: &Option, commit: &str, test: &str) -> bool { + if let Some(job) = job { + if job.commit == commit && job.test == test { + return true; + } + } + + false +} + +fn get_test_job(rc: &Ktestrc) -> Option { + let file = OpenOptions::new() + .read(true) + .write(true) + .open(rc.output_dir.join("jobs")).unwrap(); + let map = unsafe { MmapOptions::new().map(&file).unwrap() }; + + let mut len = file.metadata().unwrap().len(); + if len == 0 { + return None; + } + + let mut ret = None; + + for job in map.rsplit(|b| *b == b'\n') { + if job.is_empty() { + continue; + } + + let mut fields = job.split(|b| *b == b' '); + let branch = str::from_utf8(fields.next().unwrap()).unwrap(); + let commit = str::from_utf8(fields.next().unwrap()).unwrap(); + let age_str = str::from_utf8(fields.next().unwrap()).unwrap(); + let age = str::parse::(age_str).unwrap(); + let test = str::from_utf8(fields.next().unwrap()).unwrap(); + let subtest = str::from_utf8(fields.next().unwrap()).unwrap(); + + if ret.is_none() { + ret = Some(TestJob { + branch: branch.to_string(), + commit: commit.to_string(), + test: test.to_string(), + age, + subtests: vec![subtest.to_string()], + }); + + len = job.as_ptr() as u64 - map.as_ptr() as u64; + } else if commit_test_matches(&ret, commit, test) { + if let Some(ref mut r) = ret { + r.subtests.push(subtest.to_string()); + len = job.as_ptr() as u64 - map.as_ptr() as u64; + + if r.subtests.len() > 20 { + break; + } + } + } else { + break; + } + } + + let _ = file.set_len(len); + + ret +} + +fn subtest_full_name(test_path: &Path, subtest: &String) -> String { + format!("{}.{}", + test_path.file_stem().unwrap().to_string_lossy(), + subtest.replace("/", ".")) +} + +fn create_job_lockfiles(rc: &Ktestrc, mut job: TestJob) -> Option { + job.subtests = job.subtests.iter() + .filter(|i| lockfile_exists(rc, &job.commit, + &subtest_full_name(&Path::new(&job.test), &i), true)) + .map(|i| i.to_string()) + .collect(); + + if !job.subtests.is_empty() { Some(job) } else { None } +} + +fn get_and_lock_job(rc: &Ktestrc) -> Option { + loop { + let job = get_test_job(rc); + if let Some(job) = job { + let job = create_job_lockfiles(rc, job); + if job.is_some() { + return job; + } + } else { + return job; + } + + } +} + +fn main() { + std::process::Command::new("gen-job-list") + .output() + .expect("failed to execute gen-job-list"); + + let args = Args::parse(); + + let rc = ktestrc_read(); + if let Err(e) = rc { + eprintln!("could not read config; {}", e); + process::exit(1); + } + let rc = rc.unwrap(); + + let lockfile = rc.output_dir.join("jobs.lock"); + let filelock = FileLock::lock(lockfile, true, FileOptions::new().create(true).write(true)).unwrap(); + + let job = if !args.dry_run { + get_and_lock_job(&rc) + } else { + get_test_job(&rc) + }; + + drop(filelock); + + if let Some(job) = job { + let tests = job.test + " " + &job.subtests.join(" "); + + println!("TEST_JOB {} {} {}", job.branch, job.commit, tests); + + workers_update(&rc, Worker { + hostname: args.hostname, + workdir: args.workdir, + starttime: Utc::now(), + branch: job.branch.clone(), + age: job.age, + commit: job.commit.clone(), + tests: tests.clone(), + }); + } else { + workers_update(&rc, Worker { + hostname: args.hostname, + workdir: args.workdir, + starttime: Utc::now(), + branch: "".to_string(), + age: 0, + commit: "".to_string(), + tests: "".to_string(), + }); + } +} diff --git a/src/bin/update-lcov.rs b/src/bin/update-lcov.rs new file mode 100644 index 0000000..3a46812 --- /dev/null +++ b/src/bin/update-lcov.rs @@ -0,0 +1,22 @@ +use std::process; +use ci_cgi::{ktestrc_read, update_lcov}; +use clap::Parser; + +#[derive(Parser)] +#[command(author, version, about, long_about = None)] +struct Args { + commit: String, +} + +fn main() { + let args = Args::parse(); + let ktestrc = ktestrc_read(); + if let Err(e) = ktestrc { + eprintln!("could not read config; {}", e); + process::exit(1); + } + let ktestrc = ktestrc.unwrap(); + + update_lcov(&ktestrc, &args.commit); +} + diff --git a/src/build.rs b/src/build.rs new file mode 100644 index 0000000..f2e2ec2 --- /dev/null +++ b/src/build.rs @@ -0,0 +1,15 @@ +extern crate capnpc; + +fn main() { + capnpc::CompilerCommand::new() + .output_path(".") + .file("src/testresult.capnp") + .run() + .expect("compiling schema"); + + capnpc::CompilerCommand::new() + .output_path(".") + .file("src/worker.capnp") + .run() + .expect("compiling schema"); +} diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..0ba1232 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,399 @@ +use std::collections::BTreeMap; +use std::fs::{File, OpenOptions, create_dir_all, read_to_string}; +use std::io::ErrorKind; +use std::io::prelude::*; +use std::path::PathBuf; +use std::time::SystemTime; +use die::die; +use serde_derive::Deserialize; +use toml; +use anyhow; + +pub mod testresult_capnp; +pub mod worker_capnp; + +pub fn git_get_commit(repo: &git2::Repository, reference: String) -> Result { + let r = repo.revparse_single(&reference); + if let Err(e) = r { + eprintln!("Error from resolve_reference_from_short_name {} in {}: {}", reference, repo.path().display(), e); + return Err(e); + } + + let r = r.unwrap().peel_to_commit(); + if let Err(e) = r { + eprintln!("Error from peel_to_commit {} in {}: {}", reference, repo.path().display(), e); + return Err(e); + } + r +} + +#[derive(Deserialize)] +pub struct KtestrcTestGroup { + pub max_commits: u64, + pub priority: u64, + pub tests: Vec, +} + +#[derive(Deserialize)] +pub struct KtestrcBranch { + pub fetch: String, + pub tests: Vec, +} + +#[derive(Deserialize)] +pub struct Ktestrc { + pub linux_repo: PathBuf, + pub output_dir: PathBuf, + pub ktest_dir: PathBuf, + pub test_group: BTreeMap, + pub branch: BTreeMap, +} + +pub fn ktestrc_read() -> anyhow::Result { + let config = read_to_string("/etc/ktest-ci.toml")?; + let ktestrc: Ktestrc = toml::from_str(&config)?; + + Ok(ktestrc) +} + +pub use testresult_capnp::test_result::Status as TestStatus; + +impl TestStatus { + fn from_str(status: &str) -> TestStatus { + if status.is_empty() { + TestStatus::Inprogress + } else if status.contains("IN PROGRESS") { + TestStatus::Inprogress + } else if status.contains("PASSED") { + TestStatus::Passed + } else if status.contains("FAILED") { + TestStatus::Failed + } else if status.contains("NOTRUN") { + TestStatus::Notrun + } else if status.contains("NOT STARTED") { + TestStatus::Notstarted + } else { + TestStatus::Unknown + } + } + + pub fn to_str(&self) -> &'static str { + match self { + TestStatus::Inprogress => "In progress", + TestStatus::Passed => "Passed", + TestStatus::Failed => "Failed", + TestStatus::Notrun => "Not run", + TestStatus::Notstarted => "Not started", + TestStatus::Unknown => "Unknown", + } + } + + pub fn table_class(&self) -> &'static str { + match self { + TestStatus::Inprogress => "table-secondary", + TestStatus::Passed => "table-success", + TestStatus::Failed => "table-danger", + TestStatus::Notrun => "table-secondary", + TestStatus::Notstarted => "table-secondary", + TestStatus::Unknown => "table-secondary", + } + } +} + +#[derive(Copy, Clone, Debug)] +pub struct TestResult { + pub status: TestStatus, + pub starttime: DateTime, + pub duration: u64, +} + +pub type TestResultsMap = BTreeMap; + +fn commitdir_get_results_fs(ktestrc: &Ktestrc, commit_id: &String) -> TestResultsMap { + fn read_test_result(testdir: &std::fs::DirEntry) -> Option { + let mut f = File::open(&testdir.path().join("status")).ok()?; + let mut status = String::new(); + f.read_to_string(&mut status).ok()?; + + Some(TestResult { + status: TestStatus::from_str(&status), + starttime: f.metadata().ok()?.modified().ok()?.into(), + duration: read_to_string(&testdir.path().join("duration")).unwrap_or("0".to_string()).parse().unwrap_or(0), + }) + } + + let mut results = BTreeMap::new(); + + let results_dir = ktestrc.output_dir.join(commit_id).read_dir(); + + if let Ok(results_dir) = results_dir { + for d in results_dir.filter_map(|i| i.ok()) { + if let Some(r) = read_test_result(&d) { + results.insert(d.file_name().into_string().unwrap(), r); + } + } + } + + results +} + +use testresult_capnp::test_results; +use capnp::serialize; + +fn results_to_capnp(ktestrc: &Ktestrc, commit_id: &String, results_in: &TestResultsMap) -> anyhow::Result<()> { + let mut message = capnp::message::Builder::new_default(); + let results = message.init_root::(); + let mut result_list = results.init_entries(results_in.len().try_into().unwrap()); + + for (idx, (name, result_in)) in results_in.iter().enumerate() { + let mut result = result_list.reborrow().get(idx.try_into().unwrap()); + + result.set_name(name); + result.set_duration(result_in.duration.try_into().unwrap()); + result.set_status(result_in.status); + } + + let fname = ktestrc.output_dir.join(commit_id.clone() + ".capnp"); + let fname_new = ktestrc.output_dir.join(commit_id.clone() + ".capnp.new"); + + let mut out = File::create(&fname_new)?; + + serialize::write_message(&mut out, &message)?; + drop(out); + std::fs::rename(fname_new, fname)?; + + Ok(()) +} + +pub fn commit_update_results_from_fs(ktestrc: &Ktestrc, commit_id: &String) { + let results = commitdir_get_results_fs(&ktestrc, commit_id); + + results_to_capnp(ktestrc, commit_id, &results) + .map_err(|e| eprintln!("error generating capnp: {}", e)).ok(); +} + +pub fn commitdir_get_results(ktestrc: &Ktestrc, commit_id: &String) -> anyhow::Result { + let f = std::fs::read(ktestrc.output_dir.join(commit_id.to_owned() + ".capnp"))?; + + let message_reader = serialize::read_message_from_flat_slice(&mut &f[..], capnp::message::ReaderOptions::new())?; + let entries = message_reader.get_root::()? + .get_entries()?; + + let mut results = BTreeMap::new(); + for e in entries { + let r = TestResult { + status: e.get_status()?, + starttime: Utc.timestamp_opt(e.get_starttime(), 0).unwrap(), + duration: e.get_duration() + }; + + results.insert(e.get_name()?.to_string()?, r); + } + + Ok(results) +} + +use chrono::{DateTime, TimeZone, Utc}; + +#[derive(Debug)] +pub struct Worker { + pub hostname: String, + pub workdir: String, + pub starttime: DateTime, + pub branch: String, + pub age: u64, + pub commit: String, + pub tests: String, +} + +pub type Workers = Vec; + +use worker_capnp::workers; + +fn workers_parse(f: Vec) -> anyhow::Result { + let message_reader = serialize::read_message_from_flat_slice(&mut &f[..], capnp::message::ReaderOptions::new())?; + let entries = message_reader.get_root::()? + .get_entries()?; + + let workers = entries.iter().map(|e| Worker { + hostname: e.get_hostname().unwrap().to_string().unwrap(), + workdir: e.get_workdir().unwrap().to_string().unwrap(), + starttime: Utc.timestamp_opt(e.get_starttime(), 0).unwrap(), + branch: e.get_branch().unwrap().to_string().unwrap(), + commit: e.get_commit().unwrap().to_string().unwrap(), + age: e.get_age(), + tests: e.get_tests().unwrap().to_string().unwrap(), + }).collect(); + + Ok(workers) +} + +pub fn workers_get(ktestrc: &Ktestrc) -> anyhow::Result { + let f = std::fs::read(ktestrc.output_dir.join("workers.capnp"))?; + + workers_parse(f) +} + +use file_lock::{FileLock, FileOptions}; + +pub fn workers_update(ktestrc: &Ktestrc, n: Worker) -> Option<()> { + let fname = ktestrc.output_dir.join("workers.capnp"); + let foptions = FileOptions::new().read(true).write(true).append(false).create(true); + + let mut filelock = FileLock::lock(fname, true, foptions) + .map_err(|e| eprintln!("error locking workers: {}", e)).ok()?; + + let mut f = Vec::new(); + filelock.file.read_to_end(&mut f).ok()?; + + let mut workers: Workers = workers_parse(f) + .map_err(|e| eprintln!("error parsing workers: {}", e)) + .unwrap_or_default() + .into_iter() + .filter(|w| w.hostname != n.hostname || w.workdir != n.workdir) + .collect(); + + workers.push(n); + + let mut message = capnp::message::Builder::new_default(); + let workers_message = message.init_root::(); + let mut workers_list = workers_message.init_entries(workers.len().try_into().unwrap()); + + for (idx, src) in workers.iter().enumerate() { + let mut dst = workers_list.reborrow().get(idx.try_into().unwrap()); + + dst.set_hostname(&src.hostname); + dst.set_workdir(&src.workdir); + dst.set_starttime(src.starttime.timestamp()); + dst.set_branch(&src.branch); + dst.set_commit(&src.commit); + dst.set_age(src.age); + dst.set_tests(&src.tests); + } + + filelock.file.set_len(0).ok()?; + filelock.file.rewind().ok()?; + + serialize::write_message(&mut filelock.file, &message) + .map_err(|e| eprintln!("error writing workers: {}", e)).ok()?; + + Some(()) +} + +pub fn update_lcov(rc: &Ktestrc, commit_id: &String) -> Option<()> { + let commit_dir = rc.output_dir.join(commit_id); + + if !std::fs::remove_file(commit_dir.join("lcov-stale")).is_ok() { return Some(()); } + + let lockfile = "/home/testdashboard/linux-1-lock"; + let filelock = FileLock::lock(lockfile, true, FileOptions::new().create(true).write(true)).ok()?; + + let mut args = Vec::new(); + + let new_lcov: Vec<_> = std::fs::read_dir(&commit_dir).ok()? + .filter_map(|d| d.ok()) + .filter_map(|d| d.file_name().into_string().ok()) + .filter(|d| d.starts_with("lcov.partial.")) + .collect(); + + for d in &new_lcov { + args.push("--add-tracefile".to_string()); + args.push(d.clone()); + } + + if commit_dir.join("lcov.info").exists() { + args.push("--add-tracefile".to_string()); + args.push("lcov.info".to_string()); + } + + let status = std::process::Command::new("lcov") + .current_dir(&commit_dir) + .arg("--quiet") + .arg("--output-file") + .arg("lcov.info.new") + .args(args) + .status() + .expect(&format!("failed to execute lcov")); + if !status.success() { + eprintln!("lcov error: {}", status); + return Some(()); + } + + std::fs::rename(commit_dir.join("lcov.info.new"), commit_dir.join("lcov.info")).ok()?; + + for d in &new_lcov { std::fs::remove_file(commit_dir.join(d)).ok(); } + + let status = std::process::Command::new("git") + .current_dir("/home/testdashboard/linux-1") + .arg("checkout") + .arg("-f") + .arg(commit_id) + .status() + .expect(&format!("failed to execute genhtml")); + if !status.success() { + eprintln!("git checkout error: {}", status); + return Some(()); + } + + let status = std::process::Command::new("genhtml") + .current_dir("/home/testdashboard/linux-1") + .arg("--output-directory") + .arg(commit_dir.join("lcov")) + .arg(commit_dir.join("lcov.info")) + .status() + .expect(&format!("failed to execute genhtml")); + if !status.success() { + eprintln!("genhtml error: {}", status); + return Some(()); + } + + drop(filelock); + Some(()) +} + +pub fn lockfile_exists(rc: &Ktestrc, commit: &str, test_name: &str, create: bool) -> bool { + let lockfile = rc.output_dir.join(commit).join(test_name).join("status"); + + let timeout = std::time::Duration::from_secs(3600); + let metadata = std::fs::metadata(&lockfile); + + if let Ok(metadata) = metadata { + let elapsed = metadata.modified().unwrap() + .elapsed() + .unwrap_or(std::time::Duration::from_secs(0)); + + if metadata.is_file() && + metadata.len() == 0 && + elapsed > timeout && + std::fs::remove_file(&lockfile).is_ok() { + eprintln!("Deleted stale lock file {:?}, mtime {:?} now {:?} elapsed {:?})", + &lockfile, metadata.modified().unwrap(), + SystemTime::now(), + elapsed); + } + } + + if !create { + lockfile.exists() + } else { + let dir = lockfile.parent().unwrap(); + let r = create_dir_all(dir); + if let Err(e) = r { + if e.kind() != ErrorKind::AlreadyExists { + die!("error creating {:?}: {}", dir, e); + } + } + + let r = OpenOptions::new() + .write(true) + .create_new(true) + .open(&lockfile); + if let Err(ref e) = r { + if e.kind() != ErrorKind::AlreadyExists { + die!("error creating {:?}: {}", lockfile, e); + } + } + + r.is_ok() + } +} diff --git a/src/testresult.capnp b/src/testresult.capnp new file mode 100644 index 0000000..5699954 --- /dev/null +++ b/src/testresult.capnp @@ -0,0 +1,22 @@ +@0x9527f7d16acca92e; + +struct TestResult { + name @0: Text; + starttime @3: Int64; + duration @1: UInt64; + status @2: Status; + enum Status { + inprogress @0; + passed @1; + failed @2; + notrun @3; + notstarted @4; + unknown @5; + } +} + +struct TestResults { + entries @0: List(TestResult); +} + +# vim: sts=4:sw=4 diff --git a/src/worker.capnp b/src/worker.capnp new file mode 100644 index 0000000..66e7158 --- /dev/null +++ b/src/worker.capnp @@ -0,0 +1,18 @@ +@0xe7ff10499731ce1f; + +struct Worker { + hostname @0: Text; + workdir @1: Text; + starttime @2: Int64; + + branch @3: Text; + commit @4: Text; + age @5: UInt64; + tests @6: Text; +} + +struct Workers { + entries @0: List(Worker); +} + +# vim: sts=4:sw=4 -- cgit v1.2.3