diff --git a/Cargo.lock b/Cargo.lock index 83dbc93..b23d509 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -700,6 +700,19 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b02b629252fe8ef6460461409564e2c21d0c8e77e0944f3d189ff06c4e932ad" +[[package]] +name = "cargo-lock" +version = "9.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e11c675378efb449ed3ce8de78d75d0d80542fc98487c26aba28eb3b82feac72" +dependencies = [ + "petgraph", + "semver", + "serde", + "toml", + "url", +] + [[package]] name = "cc" version = "1.0.79" @@ -4091,6 +4104,9 @@ name = "semver" version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" +dependencies = [ + "serde", +] [[package]] name = "serde" @@ -4636,6 +4652,7 @@ name = "tabby-scheduler" version = "0.6.0-dev" dependencies = [ "anyhow", + "cargo-lock", "file-rotate", "ignore", "job_scheduler", diff --git a/crates/tabby-common/src/lib.rs b/crates/tabby-common/src/lib.rs index b157cbc..ed90ff4 100644 --- a/crates/tabby-common/src/lib.rs +++ b/crates/tabby-common/src/lib.rs @@ -56,8 +56,8 @@ pub struct Tag { pub syntax_type_name: String, } -#[derive(Default, Serialize, Deserialize, Clone)] -pub struct Dependency { +#[derive(Default, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash, Clone)] +pub struct Package { pub language: String, pub name: String, #[serde(skip_serializing_if = "Option::is_none")] @@ -66,5 +66,5 @@ pub struct Dependency { #[derive(Default, Serialize, Deserialize)] pub struct DependencyFile { - pub deps: Vec, + pub direct: Vec, } diff --git a/crates/tabby-scheduler/Cargo.toml b/crates/tabby-scheduler/Cargo.toml index 0d2c99e..c8cbab7 100644 --- a/crates/tabby-scheduler/Cargo.toml +++ b/crates/tabby-scheduler/Cargo.toml @@ -26,6 +26,7 @@ ignore = "0.4.20" kdam = { version = "0.5.0" } requirements = "0.3.0" serdeconv.workspace = true +cargo-lock = { version = "9.0.0", features = ["dependency-tree"] } [dev-dependencies] temp_testdir = "0.2" diff --git a/crates/tabby-scheduler/src/dataset/deps.rs b/crates/tabby-scheduler/src/dataset/deps.rs deleted file mode 100644 index 521b21f..0000000 --- a/crates/tabby-scheduler/src/dataset/deps.rs +++ /dev/null @@ -1,52 +0,0 @@ -use std::{collections::HashSet, path::Path}; - -use anyhow::Result; -use tabby_common::{Dependency, DependencyFile}; -use tracing::warn; - -pub fn collect(path: &Path, file: &mut DependencyFile) { - if let Ok(mut deps) = process_requirements_txt(path) { - file.deps.append(&mut deps); - } - - remove_duplicates(file); -} - -fn process_requirements_txt(path: &Path) -> Result> { - let requirements_txt = path.join("requirements.txt"); - let content = std::fs::read_to_string(requirements_txt)?; - - let mut deps = vec![]; - match requirements::parse_str(&content) { - Ok(requirements) => { - for requirement in requirements { - if let Some(name) = requirement.name { - deps.push(Dependency { - language: "python".to_owned(), - name, - version: None, // requirements.txt doesn't come with accurate version information. - }); - } - } - } - Err(err) => { - warn!("Failed to parse requirements.txt: {}", err); - } - } - - Ok(deps) -} - -fn remove_duplicates(file: &mut DependencyFile) { - let mut keys: HashSet<(String, String)> = HashSet::default(); - let mut deps = vec![]; - for x in &file.deps { - let key = (x.language.clone(), x.name.clone()); - if !keys.contains(&key) { - keys.insert(key); - deps.push(x.clone()); - } - } - - file.deps = deps; -} diff --git a/crates/tabby-scheduler/src/dataset/deps/mod.rs b/crates/tabby-scheduler/src/dataset/deps/mod.rs new file mode 100644 index 0000000..c4e29af --- /dev/null +++ b/crates/tabby-scheduler/src/dataset/deps/mod.rs @@ -0,0 +1,20 @@ +mod python; +mod rust; + +use std::{collections::HashSet, path::Path}; + +use tabby_common::DependencyFile; + +pub fn collect(path: &Path, file: &mut DependencyFile) { + if let Ok(mut deps) = python::process_requirements_txt(path) { + file.direct.append(&mut deps); + } + + if let Ok(mut deps) = rust::process_cargo(path) { + file.direct.append(&mut deps); + } + + // Remove duplicates across sources. + let deps = file.direct.clone().into_iter().collect::>(); + file.direct = deps.into_iter().collect(); +} diff --git a/crates/tabby-scheduler/src/dataset/deps/python.rs b/crates/tabby-scheduler/src/dataset/deps/python.rs new file mode 100644 index 0000000..a8cdb3d --- /dev/null +++ b/crates/tabby-scheduler/src/dataset/deps/python.rs @@ -0,0 +1,30 @@ +use std::{collections::HashSet, path::Path}; + +use anyhow::Result; +use tabby_common::Package; +use tracing::warn; + +pub fn process_requirements_txt(path: &Path) -> Result> { + let requirements_txt = path.join("requirements.txt"); + let content = std::fs::read_to_string(requirements_txt)?; + + let mut deps = HashSet::new(); + match requirements::parse_str(&content) { + Ok(requirements) => { + for requirement in requirements { + if let Some(name) = requirement.name { + deps.insert(Package { + language: "python".to_owned(), + name, + version: None, // requirements.txt doesn't come with accurate version information. + }); + } + } + } + Err(err) => { + warn!("Failed to parse requirements.txt: {}", err); + } + } + + Ok(deps.into_iter().collect()) +} diff --git a/crates/tabby-scheduler/src/dataset/deps/rust.rs b/crates/tabby-scheduler/src/dataset/deps/rust.rs new file mode 100644 index 0000000..19da9d1 --- /dev/null +++ b/crates/tabby-scheduler/src/dataset/deps/rust.rs @@ -0,0 +1,42 @@ +use std::path::Path; + +use anyhow::Result; +use cargo_lock::dependency::graph::EdgeDirection; +use tabby_common::Package; + +fn extract_deps<'a, I>(packages: I) -> Vec +where + I: IntoIterator, +{ + let mut res = packages + .into_iter() + .map(|package| Package { + language: String::from("rust"), + name: package.name.to_string(), + version: Some(package.version.to_string()), + }) + .collect::>() + .into_iter() + .collect::>(); + res.sort_unstable(); + res +} + +pub fn process_cargo(path: &Path) -> Result> { + let cargo_lock_file = path.join("Cargo.lock"); + + let lockfile = cargo_lock::Lockfile::load(cargo_lock_file)?; + + let tree = lockfile.dependency_tree()?; + let graph = tree.graph(); + + let root_pkg_idx = graph + .externals(EdgeDirection::Incoming) + .collect::>(); + let direct_deps_idx = root_pkg_idx + .iter() + .flat_map(|idx| graph.neighbors_directed(*idx, EdgeDirection::Outgoing)) + .collect::>(); + let deps = extract_deps(direct_deps_idx.iter().map(|dep_idx| &graph[*dep_idx])); + Ok(deps) +}