Add basic CI (#156)

* Add basic CI

* Add liburing installation step to CI workflow

* Run `npm install` as part of ci/check

* Add `@types/node` package

* Add `submodules: 'recursive'` to CI

* Skip test if test data is not available

* Install `cargo-about` in CI

.github/workflows/ci.yaml

@@ -0,0 +1,48 @@
+# Based on: https://github.com/clap-rs/clap/blob/master/.github/workflows/ci.yml
+
+name: CI
+
+permissions:
+  contents: read
+
+on:
+  pull_request:
+  push:
+
+env:
+  RUST_BACKTRACE: 1
+  CARGO_TERM_COLOR: always
+  CLICOLOR: 1
+
+jobs:
+  test:
+    name: Test
+    strategy:
+      matrix:
+        build: [linux]
+        include:
+          - build: linux
+            os: ubuntu-latest
+            rust: "stable"
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: ${{ matrix.rust }}
+      - uses: Swatinem/rust-cache@v2
+      - name: Install liburing
+        run: |
+            sudo apt-get update
+            sudo apt-get install -y liburing-dev
+      - uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-about
+      - name: Run CI
+        run: ./scripts/ci/all
+      - name: Test
+        run: cargo test

crates/core/src/ranking/models/cross_encoder.rs

@@ -165,12 +165,12 @@ mod tests {
 
     #[test]
     fn sanity_check() {
-        if !Path::new("../../data/cross_encoder").exists() {
+        let data_path = Path::new("../../data/cross_encoder");
+        if !data_path.exists() {
+            // Skip the test if the test data is not available
             return;
         }
-
-        let model = CrossEncoderModel::open("../../data/cross_encoder")
-            .expect("Failed to find cross-encoder model");
+        let model = CrossEncoderModel::open(data_path).expect("Failed to find cross-encoder model");
 
         let s = model.run(
             "how many people live in paris",

crates/core/src/summarizer.rs

@@ -545,12 +545,12 @@ mod tests {
 
     #[test]
     fn test_dual_encoder() {
-        if !Path::new("../../data/summarizer/dual_encoder").exists() {
+        let data_path = Path::new("../../data/summarizer/dual_encoder");
+        if !data_path.exists() {
+            // Skip the test if the test data is not available
             return;
         }
-
-        let model =
-            DualEncoder::open("../../data/summarizer/dual_encoder").expect("Failed to load model");
+        let model = DualEncoder::open(data_path).expect("Failed to load model");
         let query = "What is the capital of France?";
         let pos = "The capital of France is Paris.";
         let neg = "The best baguette in Paris can be found at Boulangerie Pichard.";

crates/core/src/warc.rs

@@ -725,16 +725,15 @@ mod tests {
 
     #[test]
     fn internet_archive_parse() {
-        if !Path::new("../../data/internet_archive.warc.gz").exists() {
+        let data_path = Path::new("../../data/internet_archive.warc.gz");
+        if !data_path.exists() {
+            // Skip the test if the test data is not available
             return;
         }
 
         let mut records = 0;
 
-        for record in WarcFile::open("../../data/internet_archive.warc.gz")
-            .unwrap()
-            .records()
-        {
+        for record in WarcFile::open(data_path).unwrap().records() {
             records += 1;
             if let Err(err) = record {
                 panic!("Error: {:?}", err);

crates/core/src/widgets/thesaurus.rs

@@ -504,7 +504,12 @@ mod tests {
 
     #[test]
     fn build_dict() {
-        let dict = Dictionary::build("../../data/english-wordnet-2022-subset.ttl").unwrap();
+        let data_path = Path::new("../../data/english-wordnet-2022-subset.ttl");
+        if !data_path.exists() {
+            // Skip the test if the test data is not available
+            return;
+        }
+        let dict = Dictionary::build(data_path).unwrap();
 
         let infos = dict.get(Lemma("barely".to_string()));
 

crates/zimba/src/lib.rs

@@ -636,11 +636,13 @@ mod tests {
 
     #[test]
     fn it_works() {
-        if !Path::new("../../data/test.zim").exists() {
+        let data_path = Path::new("../../data/test.zim");
+        if !data_path.exists() {
+            // Skip test if data file is not present
             return;
         }
 
-        let zim = ZimFile::open("../../data/test.zim").unwrap();
+        let zim = ZimFile::open(data_path).unwrap();
 
         assert_eq!(zim.header.magic, 72173914);
         assert_eq!(zim.header.major_version, 5);

crates/zimba/src/wiki.rs

@@ -256,11 +256,18 @@ impl<'a> Iterator for ImageIterator<'a> {
 
 #[cfg(test)]
 mod tests {
+    use std::path::Path;
+
     use super::*;
 
     #[test]
     fn test_article_iterator() {
-        let zim = ZimFile::open("../../data/test.zim").unwrap();
+        let data_path = Path::new("../../data/test.zim");
+        if !data_path.exists() {
+            // Skip the test if the test data is not available
+            return;
+        }
+        let zim = ZimFile::open(data_path).unwrap();
         let mut iter = ArticleIterator::new(&zim).unwrap();
 
         let article = iter.next().unwrap();
@@ -271,7 +278,12 @@ mod tests {
 
     #[test]
     fn test_image_iterator() {
-        let zim = ZimFile::open("../../data/test.zim").unwrap();
+        let data_path = Path::new("../../data/test.zim");
+        if !data_path.exists() {
+            // Skip the test if the test data is not available
+            return;
+        }
+        let zim = ZimFile::open(data_path).unwrap();
         let mut iter = ImageIterator::new(&zim).unwrap();
 
         let image = iter.next().unwrap();

frontend/package.json

@@ -28,6 +28,7 @@
     "@tailwindcss/line-clamp": "^0.4.4",
     "@tailwindcss/typography": "^0.5.10",
     "@types/file-saver": "^2.0.7",
+    "@types/node": "^20.11.19",
     "@typescript-eslint/eslint-plugin": "^6.14.0",
     "@typescript-eslint/parser": "^6.14.0",
     "autoprefixer": "^10.4.16",
@@ -60,4 +61,4 @@
     "tailwind-merge": "^2.1.0",
     "ts-pattern": "^5.0.6"
   }
-}
+}

scripts/ci/check

@@ -3,4 +3,4 @@ set -e
 
 cargo check
 cargo check --no-default-features
-cd frontend && npm run check
+cd frontend && npm install && npm run check