diff --git a/Cargo.lock b/Cargo.lock index b23d509..8e4dc9b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4704,6 +4704,8 @@ dependencies = [ "tokio", "tokio-rusqlite", "tokio-tungstenite", + "tower", + "tower-http 0.4.0", "tracing", "unicase", "uuid 1.4.1", @@ -5315,10 +5317,16 @@ dependencies = [ "http", "http-body", "http-range-header", + "httpdate", + "mime", + "mime_guess", + "percent-encoding", "pin-project-lite", "tokio", + "tokio-util", "tower-layer", "tower-service", + "tracing", ] [[package]] diff --git a/crates/tabby-common/src/config.rs b/crates/tabby-common/src/config.rs index e344f53..006fcbf 100644 --- a/crates/tabby-common/src/config.rs +++ b/crates/tabby-common/src/config.rs @@ -50,13 +50,17 @@ impl RepositoryConfig { let path = self.git_url.strip_prefix("file://").unwrap(); path.into() } else { - repositories_dir().join(filenamify(&self.git_url)) + repositories_dir().join(self.name()) } } pub fn is_local_dir(&self) -> bool { self.git_url.starts_with("file://") } + + pub fn name(&self) -> String { + filenamify(&self.git_url) + } } #[derive(Serialize, Deserialize)] @@ -96,4 +100,12 @@ mod tests { }; assert!(!repo.is_local_dir()); } + + #[test] + fn test_repository_config_name() { + let repo = RepositoryConfig { + git_url: "https://github.com/TabbyML/tabby.git".to_owned(), + }; + assert_eq!(repo.name(), "https_github.com_TabbyML_tabby.git"); + } } diff --git a/crates/tabby-common/src/lib.rs b/crates/tabby-common/src/lib.rs index ed90ff4..8966adf 100644 --- a/crates/tabby-common/src/lib.rs +++ b/crates/tabby-common/src/lib.rs @@ -45,7 +45,7 @@ impl SourceFile { } } -#[derive(Serialize, Deserialize)] +#[derive(Serialize, Deserialize, Clone, Debug)] pub struct Tag { pub range: Range, pub name_range: Range, diff --git a/ee/tabby-webserver/Cargo.toml b/ee/tabby-webserver/Cargo.toml index fdb4ecf..71a05e9 100644 --- a/ee/tabby-webserver/Cargo.toml +++ b/ee/tabby-webserver/Cargo.toml @@ -29,6 +29,8 @@ thiserror.workspace = true tokio = { workspace = true, features = ["fs"] } tokio-rusqlite = "0.4.0" tokio-tungstenite = "0.20.1" +tower = { version = "0.4", features = ["util"] } +tower-http = { version = "0.4.0", features = ["fs", "trace"] } tracing.workspace = true unicase = "2.7.0" diff --git a/ee/tabby-webserver/docs/api_spec.md b/ee/tabby-webserver/docs/api_spec.md new file mode 100644 index 0000000..531a327 --- /dev/null +++ b/ee/tabby-webserver/docs/api_spec.md @@ -0,0 +1,143 @@ +# API Specs + +## Repository api: `/repositories` + +### Resolve + +Get file or directory content from local repositories + +**URL:** `/repositories/{name}/resolve/{path}` + +**Method:** `GET` + +**Request examples:** + +- Get directory content + +```shell +curl --request GET \ + --url http://localhost:8080/repositories/https_github.com_TabbyML_tabby.git/resolve/ + +curl --request GET \ + --url http://localhost:9090/repositories/https_github.com_TabbyML_tabby.git/resolve/ee/tabby-webserver/ +``` + +- Get file content + +```shell +curl --request GET \ + --url http://localhost:8080/repositories/https_github.com_TabbyML_tabby.git/resolve/package.json + +curl --request GET \ + --url http://localhost:9090/repositories/https_github.com_TabbyML_tabby.git/resolve/ee/tabby-webserver/src/api.rs +``` + +**Response examples:** + +- All directory query will return a list of string, with each string represents an entry under that directory. The `Content-Type` for directory query is `application/vnd.directory+json`. + +For `/repositories/https_github.com_TabbyML_tabby.git/resolve/ee/tabby-webserver/`, the response is: + +```json +{ + "entries": [ + "ee/tabby-webserver/src", + "ee/tabby-webserver/ui", + "ee/tabby-webserver/examples", + "ee/tabby-webserver/Cargo.toml", + "ee/tabby-webserver/graphql" + ] +} +``` + +- The file query will return file content, the `Content-Type` will be guessed from the file extension. + +For request `/repositories/https_github.com_TabbyML_tabby.git/resolve/package.json`, the content type is `application/json`, and the response is: + +```json +{ + "private": true, + "workspaces": [ + "clients/tabby-agent", + "clients/vscode", + "clients/vim", + "clients/intellij" + ], + "engines": { + "node": ">=18" + } +} +``` + +For request `/repositories/https_github.com_TabbyML_tabby.git/resolve/ee/tabby-webserver/src/api.rs`, the content type is `text/x-rust`, and the response is: + +```text +use async_trait::async_trait; +use juniper::{GraphQLEnum, GraphQLObject}; +use serde::{Deserialize, Serialize}; +use tabby_common::api::{ + code::{CodeSearch, CodeSearchError, SearchResponse}, + event::RawEventLogger, +}; +use thiserror::Error; +use tokio_tungstenite::connect_async; + +use crate::websocket::WebSocketTransport; + +#[derive(GraphQLEnum, Serialize, Deserialize, Clone, Debug)] +pub enum WorkerKind { + Completion, + Chat, +} + +......omit...... +``` + +### Meta + +Get dataset entry for each indexed file in the repository + +**URL:** `/repositories/{name}/meta/{path}` + +**Method:** `GET` + +**Request example:** + +```shell +curl --request GET \ + --url http://localhost:9090/repositories/https_github.com_TabbyML_tabby.git/meta/ee/tabby-webserver/src/lib.rs +``` + +**Response example:** + +The `Content-Type` for successful response is always `application/json`. + +```json +{ + "git_url": "https://github.com/TabbyML/tabby.git", + "filepath": "ee/tabby-webserver/src/lib.rs", + "language": "rust", + "max_line_length": 88, + "avg_line_length": 26.340782, + "alphanum_fraction": 0.56416017, + "tags": [ + { + "range": { + "start": 0, + "end": 12 + }, + "name_range": { + "start": 8, + "end": 11 + }, + "line_range": { + "start": 0, + "end": 12 + }, + "is_definition": true, + "syntax_type_name": "module" + }, + ......omit...... + ] +} +``` diff --git a/ee/tabby-webserver/src/lib.rs b/ee/tabby-webserver/src/lib.rs index e93f338..8a5ad9e 100644 --- a/ee/tabby-webserver/src/lib.rs +++ b/ee/tabby-webserver/src/lib.rs @@ -11,6 +11,7 @@ use tracing::{error, warn}; use websocket::WebSocketTransport; mod db; +mod repositories; mod server; mod ui; mod websocket; @@ -49,7 +50,8 @@ pub async fn attach_webserver( ) .route("/graphql", routing::get(playground("/graphql", None))) .layer(Extension(schema)) - .route("/hub", routing::get(ws_handler).with_state(ctx)); + .route("/hub", routing::get(ws_handler).with_state(ctx)) + .nest("/repositories", repositories::routes()); let ui = ui .route("/graphiql", routing::get(graphiql("/graphql", None))) diff --git a/ee/tabby-webserver/src/repositories.rs b/ee/tabby-webserver/src/repositories.rs new file mode 100644 index 0000000..b9d3ca2 --- /dev/null +++ b/ee/tabby-webserver/src/repositories.rs @@ -0,0 +1,56 @@ +mod resolve; + +use anyhow::Result; +use axum::{extract::Path, http::StatusCode, response::Response, routing, Json, Router}; +use tabby_common::path::repositories_dir; +use tracing::{instrument, warn}; + +use crate::{ + repositories, + repositories::resolve::{resolve_dir, resolve_file, resolve_meta, Meta, ResolveParams}, +}; + +pub fn routes() -> Router { + Router::new() + .route("/:name/resolve/", routing::get(repositories::resolve)) + .route("/:name/resolve/*path", routing::get(repositories::resolve)) + .route("/:name/meta/", routing::get(repositories::meta)) + .route("/:name/meta/*path", routing::get(repositories::meta)) +} + +#[instrument(skip(repo))] +async fn resolve(Path(repo): Path) -> Result { + let root = repositories_dir().join(repo.name_str()); + let full_path = root.join(repo.path_str()); + let is_dir = tokio::fs::metadata(full_path.clone()) + .await + .map(|m| m.is_dir()) + .unwrap_or(false); + + if is_dir { + return match resolve_dir(root, full_path.clone()).await { + Ok(resp) => Ok(resp), + Err(err) => { + warn!("failed to resolve_dir <{:?}>: {}", full_path, err); + Err(StatusCode::INTERNAL_SERVER_ERROR) + } + }; + } + + match resolve_file(root, &repo).await { + Ok(resp) => Ok(resp), + Err(err) => { + warn!("failed to resolve_file <{:?}>: {}", full_path, err); + Err(StatusCode::INTERNAL_SERVER_ERROR) + } + } +} + +#[instrument(skip(repo))] +async fn meta(Path(repo): Path) -> Result, StatusCode> { + let key = repo.dataset_key(); + if let Some(resp) = resolve_meta(&key) { + return Ok(Json(resp)); + } + Err(StatusCode::NOT_FOUND) +} diff --git a/ee/tabby-webserver/src/repositories/resolve.rs b/ee/tabby-webserver/src/repositories/resolve.rs new file mode 100644 index 0000000..b550f47 --- /dev/null +++ b/ee/tabby-webserver/src/repositories/resolve.rs @@ -0,0 +1,156 @@ +use std::{collections::HashMap, path::PathBuf, str::FromStr}; + +use anyhow::Result; +use axum::{ + body::boxed, + http::{header, Request, Uri}, + response::{IntoResponse, Response}, + Json, +}; +use hyper::Body; +use lazy_static::lazy_static; +use serde::{Deserialize, Serialize}; +use tabby_common::{config::Config, SourceFile, Tag}; +use tower::ServiceExt; +use tower_http::services::ServeDir; + +lazy_static! { + static ref META: HashMap = load_meta(); +} + +const DIRECTORY_MIME_TYPE: &str = "application/vnd.directory+json"; + +#[derive(Hash, PartialEq, Eq, Debug)] +pub struct DatasetKey { + local_name: String, + rel_path: String, +} + +#[derive(Deserialize, Debug)] +pub struct ResolveParams { + name: String, + path: Option, +} + +impl ResolveParams { + pub fn dataset_key(&self) -> DatasetKey { + DatasetKey { + local_name: self.name.clone(), + rel_path: self.path_str().to_string(), + } + } + + pub fn name_str(&self) -> &str { + self.name.as_str() + } + + pub fn path_str(&self) -> &str { + self.path.as_deref().unwrap_or("") + } +} + +#[derive(Serialize)] +struct ListDir { + entries: Vec, +} + +#[derive(Serialize, Deserialize, Clone, Debug)] +pub struct Meta { + git_url: String, + filepath: String, + language: String, + max_line_length: usize, + avg_line_length: f32, + alphanum_fraction: f32, + tags: Vec, +} + +impl From for Meta { + fn from(file: SourceFile) -> Self { + Self { + git_url: file.git_url, + filepath: file.filepath, + language: file.language, + max_line_length: file.max_line_length, + avg_line_length: file.avg_line_length, + alphanum_fraction: file.alphanum_fraction, + tags: file.tags, + } + } +} + +/// TODO: implement auto reloading logic in future (so changes produced by tabby-scheduler command will be loaded) +fn load_meta() -> HashMap { + let mut dataset = HashMap::new(); + let repo_conf = match Config::load() { + Ok(config) => config + .repositories + .into_iter() + .map(|repo| (repo.git_url.clone(), repo)) + .collect::>(), + Err(_) => { + return dataset; + } + }; + let iter = match SourceFile::all() { + Ok(all) => all, + Err(_) => { + return dataset; + } + }; + for file in iter { + if let Some(name) = repo_conf.get(&file.git_url).map(|repo| repo.name()) { + let key = DatasetKey { + local_name: name, + rel_path: file.filepath.clone(), + }; + dataset.insert(key, file.into()); + } + } + dataset +} + +/// Resolve a directory +pub async fn resolve_dir(root: PathBuf, full_path: PathBuf) -> Result { + let mut read_dir = tokio::fs::read_dir(full_path).await?; + let mut entries = vec![]; + + while let Some(entry) = read_dir.next_entry().await? { + let path = entry + .path() + .strip_prefix(&root)? + .to_str() + .unwrap() + .to_string(); + entries.push(path); + } + + let body = Json(ListDir { entries }).into_response(); + let resp = Response::builder() + .header(header::CONTENT_TYPE, DIRECTORY_MIME_TYPE) + .body(body.into_body())?; + + Ok(resp) +} + +/// Resolve a file +pub async fn resolve_file(root: PathBuf, repo: &ResolveParams) -> Result { + let uri = if !repo.path_str().starts_with('/') { + let path = format!("/{}", repo.path_str()); + Uri::from_str(path.as_str())? + } else { + Uri::from_str(repo.path_str())? + }; + + let req = Request::builder().uri(uri).body(Body::empty()).unwrap(); + let resp = ServeDir::new(root).oneshot(req).await?; + + Ok(resp.map(boxed)) +} + +pub fn resolve_meta(key: &DatasetKey) -> Option { + if let Some(meta) = META.get(key) { + return Some(meta.clone()); + } + None +}