feat: implement rust cargo.lock dependency extraction (#874)

* feat: implement rust cargo.lock dependency extraction

* [autofix.ci] apply automated fixes

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>

Cargo.lock

@@ -700,6 +700,19 @@ version = "1.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7b02b629252fe8ef6460461409564e2c21d0c8e77e0944f3d189ff06c4e932ad"
 
+[[package]]
+name = "cargo-lock"
+version = "9.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e11c675378efb449ed3ce8de78d75d0d80542fc98487c26aba28eb3b82feac72"
+dependencies = [
+ "petgraph",
+ "semver",
+ "serde",
+ "toml",
+ "url",
+]
+
 [[package]]
 name = "cc"
 version = "1.0.79"
@@ -4091,6 +4104,9 @@ name = "semver"
 version = "1.0.20"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090"
+dependencies = [
+ "serde",
+]
 
 [[package]]
 name = "serde"
@@ -4636,6 +4652,7 @@ name = "tabby-scheduler"
 version = "0.6.0-dev"
 dependencies = [
  "anyhow",
+ "cargo-lock",
  "file-rotate",
  "ignore",
  "job_scheduler",

crates/tabby-common/src/lib.rs

@@ -56,8 +56,8 @@ pub struct Tag {
     pub syntax_type_name: String,
 }
 
-#[derive(Default, Serialize, Deserialize, Clone)]
-pub struct Dependency {
+#[derive(Default, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash, Clone)]
+pub struct Package {
     pub language: String,
     pub name: String,
     #[serde(skip_serializing_if = "Option::is_none")]
@@ -66,5 +66,5 @@ pub struct Dependency {
 
 #[derive(Default, Serialize, Deserialize)]
 pub struct DependencyFile {
-    pub deps: Vec<Dependency>,
+    pub direct: Vec<Package>,
 }

crates/tabby-scheduler/Cargo.toml

@@ -26,6 +26,7 @@ ignore = "0.4.20"
 kdam = { version = "0.5.0" }
 requirements = "0.3.0"
 serdeconv.workspace = true
+cargo-lock = { version = "9.0.0", features = ["dependency-tree"] }
 
 [dev-dependencies]
 temp_testdir = "0.2"

crates/tabby-scheduler/src/dataset/deps.rs

@@ -1,52 +0,0 @@
-use std::{collections::HashSet, path::Path};
-
-use anyhow::Result;
-use tabby_common::{Dependency, DependencyFile};
-use tracing::warn;
-
-pub fn collect(path: &Path, file: &mut DependencyFile) {
-    if let Ok(mut deps) = process_requirements_txt(path) {
-        file.deps.append(&mut deps);
-    }
-
-    remove_duplicates(file);
-}
-
-fn process_requirements_txt(path: &Path) -> Result<Vec<Dependency>> {
-    let requirements_txt = path.join("requirements.txt");
-    let content = std::fs::read_to_string(requirements_txt)?;
-
-    let mut deps = vec![];
-    match requirements::parse_str(&content) {
-        Ok(requirements) => {
-            for requirement in requirements {
-                if let Some(name) = requirement.name {
-                    deps.push(Dependency {
-                        language: "python".to_owned(),
-                        name,
-                        version: None, // requirements.txt doesn't come with accurate version information.
-                    });
-                }
-            }
-        }
-        Err(err) => {
-            warn!("Failed to parse requirements.txt: {}", err);
-        }
-    }
-
-    Ok(deps)
-}
-
-fn remove_duplicates(file: &mut DependencyFile) {
-    let mut keys: HashSet<(String, String)> = HashSet::default();
-    let mut deps = vec![];
-    for x in &file.deps {
-        let key = (x.language.clone(), x.name.clone());
-        if !keys.contains(&key) {
-            keys.insert(key);
-            deps.push(x.clone());
-        }
-    }
-
-    file.deps = deps;
-}

crates/tabby-scheduler/src/dataset/deps/mod.rs

@@ -0,0 +1,20 @@
+mod python;
+mod rust;
+
+use std::{collections::HashSet, path::Path};
+
+use tabby_common::DependencyFile;
+
+pub fn collect(path: &Path, file: &mut DependencyFile) {
+    if let Ok(mut deps) = python::process_requirements_txt(path) {
+        file.direct.append(&mut deps);
+    }
+
+    if let Ok(mut deps) = rust::process_cargo(path) {
+        file.direct.append(&mut deps);
+    }
+
+    // Remove duplicates across sources.
+    let deps = file.direct.clone().into_iter().collect::<HashSet<_>>();
+    file.direct = deps.into_iter().collect();
+}

crates/tabby-scheduler/src/dataset/deps/python.rs

@@ -0,0 +1,30 @@
+use std::{collections::HashSet, path::Path};
+
+use anyhow::Result;
+use tabby_common::Package;
+use tracing::warn;
+
+pub fn process_requirements_txt(path: &Path) -> Result<Vec<Package>> {
+    let requirements_txt = path.join("requirements.txt");
+    let content = std::fs::read_to_string(requirements_txt)?;
+
+    let mut deps = HashSet::new();
+    match requirements::parse_str(&content) {
+        Ok(requirements) => {
+            for requirement in requirements {
+                if let Some(name) = requirement.name {
+                    deps.insert(Package {
+                        language: "python".to_owned(),
+                        name,
+                        version: None, // requirements.txt doesn't come with accurate version information.
+                    });
+                }
+            }
+        }
+        Err(err) => {
+            warn!("Failed to parse requirements.txt: {}", err);
+        }
+    }
+
+    Ok(deps.into_iter().collect())
+}

crates/tabby-scheduler/src/dataset/deps/rust.rs

@@ -0,0 +1,42 @@
+use std::path::Path;
+
+use anyhow::Result;
+use cargo_lock::dependency::graph::EdgeDirection;
+use tabby_common::Package;
+
+fn extract_deps<'a, I>(packages: I) -> Vec<Package>
+where
+    I: IntoIterator<Item = &'a cargo_lock::Package>,
+{
+    let mut res = packages
+        .into_iter()
+        .map(|package| Package {
+            language: String::from("rust"),
+            name: package.name.to_string(),
+            version: Some(package.version.to_string()),
+        })
+        .collect::<std::collections::HashSet<_>>()
+        .into_iter()
+        .collect::<Vec<_>>();
+    res.sort_unstable();
+    res
+}
+
+pub fn process_cargo(path: &Path) -> Result<Vec<Package>> {
+    let cargo_lock_file = path.join("Cargo.lock");
+
+    let lockfile = cargo_lock::Lockfile::load(cargo_lock_file)?;
+
+    let tree = lockfile.dependency_tree()?;
+    let graph = tree.graph();
+
+    let root_pkg_idx = graph
+        .externals(EdgeDirection::Incoming)
+        .collect::<std::collections::HashSet<_>>();
+    let direct_deps_idx = root_pkg_idx
+        .iter()
+        .flat_map(|idx| graph.neighbors_directed(*idx, EdgeDirection::Outgoing))
+        .collect::<std::collections::HashSet<_>>();
+    let deps = extract_deps(direct_deps_idx.iter().map(|dep_idx| &graph[*dep_idx]));
+    Ok(deps)
+}