From 2d8973bcf7dcee23ccdcb7c3197a3b8c3a801b5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Oliver=20B=C3=B8ving?= Date: Sat, 17 Feb 2024 20:09:58 +0100 Subject: [PATCH] Add basic CI (#156) * Add basic CI * Add liburing installation step to CI workflow * Run `npm install` as part of ci/check * Add `@types/node` package * Add `submodules: 'recursive'` to CI * Skip test if test data is not available * Install `cargo-about` in CI --- .github/workflows/ci.yaml | 48 +++++++++++++++++++ .../core/src/ranking/models/cross_encoder.rs | 8 ++-- crates/core/src/summarizer.rs | 8 ++-- crates/core/src/warc.rs | 9 ++-- crates/core/src/widgets/thesaurus.rs | 7 ++- crates/zimba/src/lib.rs | 6 ++- crates/zimba/src/wiki.rs | 16 ++++++- frontend/package.json | 3 +- scripts/ci/check | 2 +- 9 files changed, 87 insertions(+), 20 deletions(-) create mode 100644 .github/workflows/ci.yaml diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..06b9cba9 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,48 @@ +# Based on: https://github.com/clap-rs/clap/blob/master/.github/workflows/ci.yml + +name: CI + +permissions: + contents: read + +on: + pull_request: + push: + +env: + RUST_BACKTRACE: 1 + CARGO_TERM_COLOR: always + CLICOLOR: 1 + +jobs: + test: + name: Test + strategy: + matrix: + build: [linux] + include: + - build: linux + os: ubuntu-latest + rust: "stable" + runs-on: ${{ matrix.os }} + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + with: + toolchain: ${{ matrix.rust }} + - uses: Swatinem/rust-cache@v2 + - name: Install liburing + run: | + sudo apt-get update + sudo apt-get install -y liburing-dev + - uses: taiki-e/install-action@v2 + with: + tool: cargo-about + - name: Run CI + run: ./scripts/ci/all + - name: Test + run: cargo test diff --git a/crates/core/src/ranking/models/cross_encoder.rs b/crates/core/src/ranking/models/cross_encoder.rs index 2d6c8e51..d13096e8 100644 --- a/crates/core/src/ranking/models/cross_encoder.rs +++ b/crates/core/src/ranking/models/cross_encoder.rs @@ -165,12 +165,12 @@ mod tests { #[test] fn sanity_check() { - if !Path::new("../../data/cross_encoder").exists() { + let data_path = Path::new("../../data/cross_encoder"); + if !data_path.exists() { + // Skip the test if the test data is not available return; } - - let model = CrossEncoderModel::open("../../data/cross_encoder") - .expect("Failed to find cross-encoder model"); + let model = CrossEncoderModel::open(data_path).expect("Failed to find cross-encoder model"); let s = model.run( "how many people live in paris", diff --git a/crates/core/src/summarizer.rs b/crates/core/src/summarizer.rs index 901a8c0c..cb4b2591 100644 --- a/crates/core/src/summarizer.rs +++ b/crates/core/src/summarizer.rs @@ -545,12 +545,12 @@ mod tests { #[test] fn test_dual_encoder() { - if !Path::new("../../data/summarizer/dual_encoder").exists() { + let data_path = Path::new("../../data/summarizer/dual_encoder"); + if !data_path.exists() { + // Skip the test if the test data is not available return; } - - let model = - DualEncoder::open("../../data/summarizer/dual_encoder").expect("Failed to load model"); + let model = DualEncoder::open(data_path).expect("Failed to load model"); let query = "What is the capital of France?"; let pos = "The capital of France is Paris."; let neg = "The best baguette in Paris can be found at Boulangerie Pichard."; diff --git a/crates/core/src/warc.rs b/crates/core/src/warc.rs index 112ed285..c8664dc4 100644 --- a/crates/core/src/warc.rs +++ b/crates/core/src/warc.rs @@ -725,16 +725,15 @@ mod tests { #[test] fn internet_archive_parse() { - if !Path::new("../../data/internet_archive.warc.gz").exists() { + let data_path = Path::new("../../data/internet_archive.warc.gz"); + if !data_path.exists() { + // Skip the test if the test data is not available return; } let mut records = 0; - for record in WarcFile::open("../../data/internet_archive.warc.gz") - .unwrap() - .records() - { + for record in WarcFile::open(data_path).unwrap().records() { records += 1; if let Err(err) = record { panic!("Error: {:?}", err); diff --git a/crates/core/src/widgets/thesaurus.rs b/crates/core/src/widgets/thesaurus.rs index 736eeb63..adeed6a2 100644 --- a/crates/core/src/widgets/thesaurus.rs +++ b/crates/core/src/widgets/thesaurus.rs @@ -504,7 +504,12 @@ mod tests { #[test] fn build_dict() { - let dict = Dictionary::build("../../data/english-wordnet-2022-subset.ttl").unwrap(); + let data_path = Path::new("../../data/english-wordnet-2022-subset.ttl"); + if !data_path.exists() { + // Skip the test if the test data is not available + return; + } + let dict = Dictionary::build(data_path).unwrap(); let infos = dict.get(Lemma("barely".to_string())); diff --git a/crates/zimba/src/lib.rs b/crates/zimba/src/lib.rs index 8febd32d..c718b83f 100644 --- a/crates/zimba/src/lib.rs +++ b/crates/zimba/src/lib.rs @@ -636,11 +636,13 @@ mod tests { #[test] fn it_works() { - if !Path::new("../../data/test.zim").exists() { + let data_path = Path::new("../../data/test.zim"); + if !data_path.exists() { + // Skip test if data file is not present return; } - let zim = ZimFile::open("../../data/test.zim").unwrap(); + let zim = ZimFile::open(data_path).unwrap(); assert_eq!(zim.header.magic, 72173914); assert_eq!(zim.header.major_version, 5); diff --git a/crates/zimba/src/wiki.rs b/crates/zimba/src/wiki.rs index 3dd07e8d..f317e105 100644 --- a/crates/zimba/src/wiki.rs +++ b/crates/zimba/src/wiki.rs @@ -256,11 +256,18 @@ impl<'a> Iterator for ImageIterator<'a> { #[cfg(test)] mod tests { + use std::path::Path; + use super::*; #[test] fn test_article_iterator() { - let zim = ZimFile::open("../../data/test.zim").unwrap(); + let data_path = Path::new("../../data/test.zim"); + if !data_path.exists() { + // Skip the test if the test data is not available + return; + } + let zim = ZimFile::open(data_path).unwrap(); let mut iter = ArticleIterator::new(&zim).unwrap(); let article = iter.next().unwrap(); @@ -271,7 +278,12 @@ mod tests { #[test] fn test_image_iterator() { - let zim = ZimFile::open("../../data/test.zim").unwrap(); + let data_path = Path::new("../../data/test.zim"); + if !data_path.exists() { + // Skip the test if the test data is not available + return; + } + let zim = ZimFile::open(data_path).unwrap(); let mut iter = ImageIterator::new(&zim).unwrap(); let image = iter.next().unwrap(); diff --git a/frontend/package.json b/frontend/package.json index 40eff060..df0fff5a 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -28,6 +28,7 @@ "@tailwindcss/line-clamp": "^0.4.4", "@tailwindcss/typography": "^0.5.10", "@types/file-saver": "^2.0.7", + "@types/node": "^20.11.19", "@typescript-eslint/eslint-plugin": "^6.14.0", "@typescript-eslint/parser": "^6.14.0", "autoprefixer": "^10.4.16", @@ -60,4 +61,4 @@ "tailwind-merge": "^2.1.0", "ts-pattern": "^5.0.6" } -} \ No newline at end of file +} diff --git a/scripts/ci/check b/scripts/ci/check index 8aa9f335..e7e240d6 100755 --- a/scripts/ci/check +++ b/scripts/ci/check @@ -3,4 +3,4 @@ set -e cargo check cargo check --no-default-features -cd frontend && npm run check +cd frontend && npm install && npm run check