feat: add cargo test to github actions and run only unit tests in ci [TAB-185] (#390)

* feat: add cargo test to github actions

* chore: fix lint

* chore: add openblas dependency

* chore: update build dependency

* chore: resolve comments

* chore: fix lint

* chore: fix lint

* chore: test installing dependencies

* chore: refactor integ test

* update ci

* cleanup

---------

Co-authored-by: Meng Zhang <meng@tabbyml.com>

.github/workflows/ci.yml

@@ -21,19 +21,46 @@ concurrency:
 jobs:
   tests:
     runs-on: ubuntu-latest
+    env:
+      SCCACHE_GHA_ENABLED: true
+      RUSTC_WRAPPER: sccache
+      CARGO_INCREMENTAL: 0
     steps:
       - uses: actions/checkout@v3
         with:
           submodules: recursive
+
       - name: Install Rust
         uses: actions-rs/toolchain@v1
         with:
           toolchain: nightly
           components: rustfmt
 
+      - name: Sccache cache
+        uses: mozilla-actions/sccache-action@v0.0.3
+        with:
+          version: "v0.4.0"
+
+      - name: Cargo registry cache
+        uses: actions/cache@v3
+        with:
+          key: cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}-${{ github.sha }}
+          restore-keys: |
+            cargo-${{ runner.os }}-${{ hashFiles('**/Cargo.toml') }}-
+            cargo-${{ runner.os }}-
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+
       - name: Cargo fmt
         run: cargo +nightly fmt --check
 
+      - run: bash ./ci/prepare_build_environment.sh
+
+      - name: Run unit tests
+        run: cargo test --bin tabby --lib
+
+
   release-binary:
     needs: tests
     runs-on: ${{ matrix.os }}
@@ -81,9 +108,11 @@ jobs:
           path: |
             ~/.cargo/registry
             ~/.cargo/git
+
       - run: bash ./ci/prepare_build_environment.sh
+
       - name: Bulid release binary
-        run: cargo build --bin tabby --no-default-features ${{ matrix.flags }} --release --target ${{ matrix.target }}
+        run: cargo build --no-default-features ${{ matrix.flags }} --release --target ${{ matrix.target }}
 
       - name: Rename release binary
         run: mv target/${{ matrix.target }}/release/tabby tabby_${{ matrix.target }}

Cargo.lock

@@ -3030,6 +3030,7 @@ dependencies = [
  "tabby-common",
  "tantivy",
  "temp_testdir",
+ "tokio",
  "tracing",
  "tracing-test",
  "tree-sitter-go",

crates/llama-cpp-bindings/build.rs

@@ -1,7 +1,11 @@
 use cmake::Config;
 
 fn main() {
-    let dst = Config::new("llama.cpp").define("LLAMA_METAL", "ON").build();
+    let mut config = Config::new("llama.cpp");
+    if cfg!(target_os = "macos") {
+        config.define("LLAMA_METAL", "ON");
+    }
+    let dst = config.build();
 
     println!("cargo:rerun-if-changed=cc/*.h");
     println!("cargo:rerun-if-changed=cc/*.cc");
@@ -9,10 +13,13 @@ fn main() {
     println!("cargo:rustc-link-search=native={}/build", dst.display());
     println!("cargo:rustc-link-lib=llama");
     println!("cargo:rustc-link-lib=ggml_static");
-    println!("cargo:rustc-link-lib=framework=Foundation");
-    println!("cargo:rustc-link-lib=framework=Accelerate");
-    println!("cargo:rustc-link-lib=framework=Metal");
-    println!("cargo:rustc-link-lib=framework=MetalKit");
+
+    if cfg!(target_os = "macos") {
+        println!("cargo:rustc-link-lib=framework=Foundation");
+        println!("cargo:rustc-link-lib=framework=Accelerate");
+        println!("cargo:rustc-link-lib=framework=Metal");
+        println!("cargo:rustc-link-lib=framework=MetalKit");
+    }
 
     cxx_build::bridge("src/lib.rs")
         .file("src/engine.cc")

crates/tabby-common/src/config.rs

@@ -4,11 +4,11 @@ use std::{
 };
 
 use filenamify::filenamify;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 
 use crate::path::{config_file, repositories_dir};
 
-#[derive(Deserialize, Default)]
+#[derive(Serialize, Deserialize, Default)]
 pub struct Config {
     #[serde(default)]
     pub repositories: Vec<Repository>,
@@ -17,7 +17,7 @@ pub struct Config {
     pub experimental: Experimental,
 }
 
-#[derive(Deserialize, Default)]
+#[derive(Serialize, Deserialize, Default)]
 pub struct Experimental {
     #[serde(default = "default_as_false")]
     pub enable_prompt_rewrite: bool,
@@ -37,9 +37,15 @@ impl Config {
             )
         })
     }
+
+    #[cfg(feature = "testutils")]
+    pub fn save(&self) {
+        serdeconv::to_toml_file(self, crate::path::config_file().as_path())
+            .expect("Failed to write config file");
+    }
 }
 
-#[derive(Deserialize)]
+#[derive(Serialize, Deserialize)]
 pub struct Repository {
     pub git_url: String,
 }

crates/tabby-scheduler/Cargo.toml

@@ -30,3 +30,4 @@ tree-sitter-lua = "0.0.19"
 temp_testdir = "0.2"
 tabby-common = { path = "../tabby-common", features = [ "testutils" ] }
 tracing-test = "0.1"
+tokio = { workspace = true, features = ["rt"] }

crates/tabby-scheduler/src/lib.rs

@@ -55,32 +55,3 @@ pub async fn scheduler(now: bool) -> Result<()> {
 
     Ok(())
 }
-
-#[cfg(test)]
-mod tests {
-    use tabby_common::{
-        config::{Config, Experimental, Repository},
-        path::set_tabby_root,
-    };
-    use temp_testdir::*;
-    use tracing_test::traced_test;
-
-    use super::*;
-
-    #[traced_test]
-    #[test]
-    fn end_to_end() {
-        set_tabby_root(TempDir::default().to_path_buf());
-
-        let config = Config {
-            repositories: vec![Repository {
-                git_url: "https://github.com/TabbyML/interview-questions".to_owned(),
-            }],
-            experimental: Experimental::default(),
-        };
-
-        repository::sync_repositories(&config).unwrap();
-        dataset::create_dataset(&config).unwrap();
-        index::index_repositories(&config).unwrap();
-    }
-}

crates/tabby-scheduler/tests/integration_test.rs

@@ -0,0 +1,31 @@
+#[cfg(test)]
+mod tests {
+    use std::fs::create_dir_all;
+
+    use tabby_common::{
+        config::{Config, Experimental, Repository},
+        path::set_tabby_root,
+    };
+    use temp_testdir::*;
+    use tracing_test::traced_test;
+
+    #[traced_test]
+    #[tokio::test]
+    async fn end_to_end() {
+        let root = TempDir::default();
+        create_dir_all(&root).expect("Failed to create tabby root");
+        set_tabby_root(root.to_path_buf());
+
+        let config = Config {
+            repositories: vec![Repository {
+                git_url: "https://github.com/TabbyML/interview-questions".to_owned(),
+            }],
+            experimental: Experimental::default(),
+        };
+
+        config.save();
+
+        let res = tabby_scheduler::scheduler(true).await;
+        res.expect("Failed to run scheduler");
+    }
+}