From da02d471a9927a7d232ccb3e616b713bcf916b47 Mon Sep 17 00:00:00 2001 From: Meng Zhang Date: Sun, 4 Jun 2023 20:08:43 -0700 Subject: [PATCH] chore: add tabby-scheduler [TAB-17] (#192) * add scheduler * update fmt * add integration tests for scheduler --- Cargo.lock | 24 +++++++ Cargo.toml | 1 + crates/tabby-common/Cargo.toml | 5 +- crates/tabby-common/src/config.rs | 26 +++++++ crates/tabby-common/src/events.rs | 2 +- crates/tabby-common/src/lib.rs | 1 + crates/tabby-common/src/path.rs | 39 +++++++++-- crates/tabby-scheduler/Cargo.toml | 14 ++++ crates/tabby-scheduler/src/git.rs | 0 crates/tabby-scheduler/src/lib.rs | 70 +++++++++++++++++++ .../tests/sync_repositories.rs | 19 +++++ 11 files changed, 193 insertions(+), 8 deletions(-) create mode 100644 crates/tabby-common/src/config.rs create mode 100644 crates/tabby-scheduler/Cargo.toml create mode 100644 crates/tabby-scheduler/src/git.rs create mode 100644 crates/tabby-scheduler/src/lib.rs create mode 100644 crates/tabby-scheduler/tests/sync_repositories.rs diff --git a/Cargo.lock b/Cargo.lock index c4d7632..5ebd731 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -708,6 +708,15 @@ dependencies = [ "instant", ] +[[package]] +name = "filenamify" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b781e8974b2cc71ac3c587c881c11ee5fe9a379f43503674e1e1052647593b4c" +dependencies = [ + "regex", +] + [[package]] name = "filetime" version = "0.2.21" @@ -2163,6 +2172,15 @@ dependencies = [ "serdeconv", ] +[[package]] +name = "tabby-scheduler" +version = "0.1.0" +dependencies = [ + "filenamify", + "tabby-common", + "temp_testdir", +] + [[package]] name = "tar" version = "0.4.38" @@ -2174,6 +2192,12 @@ dependencies = [ "xattr", ] +[[package]] +name = "temp_testdir" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921f1e9c427802414907a48b21a6504ff6b3a15a1a3cf37e699590949ad9befc" + [[package]] name = "tempfile" version = "3.5.0" diff --git a/Cargo.toml b/Cargo.toml index b8b77ff..06ec188 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ members = [ "crates/tabby", "crates/tabby-common", + "crates/tabby-scheduler", "crates/ctranslate2-bindings", "crates/rust-cxx-cmake-bridge", ] diff --git a/crates/tabby-common/Cargo.toml b/crates/tabby-common/Cargo.toml index 0cf0b63..e6890d8 100644 --- a/crates/tabby-common/Cargo.toml +++ b/crates/tabby-common/Cargo.toml @@ -3,10 +3,11 @@ name = "tabby-common" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] chrono = "0.4.26" lazy_static = { workspace = true } serde = { workspace = true } serdeconv = { workspace = true } + +[features] +testutils = [] diff --git a/crates/tabby-common/src/config.rs b/crates/tabby-common/src/config.rs new file mode 100644 index 0000000..a89370e --- /dev/null +++ b/crates/tabby-common/src/config.rs @@ -0,0 +1,26 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Deserialize)] +#[cfg_attr(feature = "testutils", derive(Serialize))] +pub struct Config { + pub repositories: Vec, +} + +impl Config { + pub fn load() -> Self { + serdeconv::from_toml_file(crate::path::config_file().as_path()) + .expect("Failed to read config file") + } + + #[cfg(feature = "testutils")] + pub fn save(&self) { + let config_file = crate::path::config_file(); + std::fs::create_dir_all(config_file.parent().unwrap()).unwrap(); + serdeconv::to_toml_file(self, config_file).expect("Failed to write config file") + } +} + +#[derive(Serialize, Deserialize)] +pub struct Repository { + pub git_url: String, +} diff --git a/crates/tabby-common/src/events.rs b/crates/tabby-common/src/events.rs index 6ee30bc..76c3259 100644 --- a/crates/tabby-common/src/events.rs +++ b/crates/tabby-common/src/events.rs @@ -7,7 +7,7 @@ use std::sync::Mutex; lazy_static! { static ref WRITER: Mutex> = { - let events_dir = &crate::path::EVENTS_DIR; + let events_dir = crate::path::events_dir(); std::fs::create_dir_all(events_dir.as_path()).ok(); let now = Utc::now(); diff --git a/crates/tabby-common/src/lib.rs b/crates/tabby-common/src/lib.rs index 0db41fb..9f22303 100644 --- a/crates/tabby-common/src/lib.rs +++ b/crates/tabby-common/src/lib.rs @@ -1,2 +1,3 @@ +pub mod config; pub mod events; pub mod path; diff --git a/crates/tabby-common/src/path.rs b/crates/tabby-common/src/path.rs index 4306ff2..67b2e41 100644 --- a/crates/tabby-common/src/path.rs +++ b/crates/tabby-common/src/path.rs @@ -1,23 +1,52 @@ +use std::cell::Cell; use std::env; use std::path::PathBuf; +use std::sync::Mutex; use lazy_static::lazy_static; lazy_static! { - pub static ref TABBY_ROOT: PathBuf = { - match env::var("TABBY_ROOT") { + static ref TABBY_ROOT: Mutex> = { + Mutex::new(Cell::new(match env::var("TABBY_ROOT") { Ok(x) => PathBuf::from(x), Err(_) => PathBuf::from(env::var("HOME").unwrap()).join(".tabby"), - } + })) }; - pub static ref EVENTS_DIR: PathBuf = TABBY_ROOT.join("events"); +} + +#[cfg(feature = "testutils")] +pub fn set_tabby_root(path: PathBuf) { + println!("SET TABBY ROOT: '{}'", path.display()); + let cell = TABBY_ROOT.lock().unwrap(); + cell.replace(path); +} + +fn tabby_root() -> PathBuf { + let mut cell = TABBY_ROOT.lock().unwrap(); + cell.get_mut().clone() +} + +pub fn config_file() -> PathBuf { + tabby_root().join("config.toml") +} + +pub fn repositories_dir() -> PathBuf { + tabby_root().join("repositories") +} + +pub fn models_dir() -> PathBuf { + tabby_root().join("models") +} + +pub fn events_dir() -> PathBuf { + tabby_root().join("events") } pub struct ModelDir(PathBuf); impl ModelDir { pub fn new(model: &str) -> Self { - Self(TABBY_ROOT.join("models").join(model)) + Self(models_dir().join(model)) } pub fn from(path: &str) -> Self { diff --git a/crates/tabby-scheduler/Cargo.toml b/crates/tabby-scheduler/Cargo.toml new file mode 100644 index 0000000..e37e257 --- /dev/null +++ b/crates/tabby-scheduler/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "tabby-scheduler" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +filenamify = "0.1.0" +tabby-common = { path = "../tabby-common" } + +[dev-dependencies] +temp_testdir = "0.2" +tabby-common = { path = "../tabby-common", features = [ "testutils" ] } diff --git a/crates/tabby-scheduler/src/git.rs b/crates/tabby-scheduler/src/git.rs new file mode 100644 index 0000000..e69de29 diff --git a/crates/tabby-scheduler/src/lib.rs b/crates/tabby-scheduler/src/lib.rs new file mode 100644 index 0000000..ec97044 --- /dev/null +++ b/crates/tabby-scheduler/src/lib.rs @@ -0,0 +1,70 @@ +use std::path::PathBuf; +use std::process::Command; + +use tabby_common::{ + config::{Config, Repository}, + path::repositories_dir, +}; + +use filenamify::filenamify; + +trait ConfigExt { + fn sync_repositories(&self); +} + +impl ConfigExt for Config { + fn sync_repositories(&self) { + for repository in self.repositories.iter() { + repository.sync() + } + } +} + +trait RepositoryExt { + fn dir(&self) -> PathBuf; + fn sync(&self); +} + +impl RepositoryExt for Repository { + fn dir(&self) -> PathBuf { + repositories_dir().join(filenamify(&self.git_url)) + } + + fn sync(&self) { + let dir = self.dir(); + let dir_string = dir.display().to_string(); + let status = if dir.exists() { + Command::new("git") + .current_dir(&dir) + .arg("pull") + .status() + .expect("git could not be executed") + } else { + std::fs::create_dir_all(&dir) + .unwrap_or_else(|_| panic!("Failed to create dir {}", dir_string)); + Command::new("git") + .current_dir(dir.parent().unwrap()) + .arg("clone") + .arg("--depth") + .arg("1") + .arg(&self.git_url) + .arg(dir) + .status() + .expect("git could not be executed") + }; + + if let Some(code) = status.code() { + if code != 0 { + panic!( + "Failed to pull remote '{}'\nConsider remove dir '{}' and retry", + &self.git_url, &dir_string + ); + } + } + } +} + +pub fn job_sync_repositories() { + let config = Config::load(); + config.sync_repositories(); +} diff --git a/crates/tabby-scheduler/tests/sync_repositories.rs b/crates/tabby-scheduler/tests/sync_repositories.rs new file mode 100644 index 0000000..96f7cc2 --- /dev/null +++ b/crates/tabby-scheduler/tests/sync_repositories.rs @@ -0,0 +1,19 @@ +use tabby_scheduler::*; +use temp_testdir::*; + +use tabby_common::config::{Config, Repository}; +use tabby_common::path::set_tabby_root; + +#[test] +fn it_works() { + set_tabby_root(TempDir::default().to_path_buf()); + + let config = Config { + repositories: vec![Repository { + git_url: "https://github.com/TabbyML/interview-questions".to_owned(), + }], + }; + + config.save(); + job_sync_repositories() +}