feat: fork aim-downloader into crates/aim-downloader to make it compatible with windows (#996)
* feat: support build on windows * revert change * feat: migrate aim source code * revert change * [autofix.ci] apply automated fixes * fix test * remove test dir, skip test * fix test --------- Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>r0.7
Eric
GitHub
parent
95d15f1224
commit
1fa0106ca3
File diff suppressed because it is too large
Load Diff
|
|
@ -8,6 +8,7 @@ members = [
|
|||
"crates/tabby-inference",
|
||||
"crates/llama-cpp-bindings",
|
||||
"crates/http-api-bindings",
|
||||
"crates/aim-downloader",
|
||||
"crates/juniper-axum",
|
||||
"ee/tabby-webserver",
|
||||
]
|
||||
|
|
|
|||
|
|
@ -0,0 +1,26 @@
|
|||
[package]
|
||||
name = "aim-downloader"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
authors.workspace = true
|
||||
homepage.workspace = true
|
||||
|
||||
[dependencies]
|
||||
async-stream = "0.3.5"
|
||||
clap = { version = "4.4.11", features = ["cargo", "string"] }
|
||||
custom_error = "1.9.2"
|
||||
dotenvy = "0.15.7"
|
||||
futures-util = "0.3.29"
|
||||
home = "0.5.5"
|
||||
indicatif = "0.17.7"
|
||||
netrc = "0.4.1"
|
||||
regex.workspace = true
|
||||
reqwest = { workspace = true, features = ["stream"] }
|
||||
sha2 = "0.10.8"
|
||||
strfmt = "0.2.4"
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
tokio-util = { version="0.7.10", features = ["full"] }
|
||||
url-parse = "1.0.7"
|
||||
|
||||
[dev-dependencies]
|
||||
serial_test = "2.0.0"
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
MIT License
|
||||
|
||||
Copyright (c) 2021-3021 Mihai Galos
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
# http_downloader
|
||||
|
||||
Adopted from https://github.com/mihaigalos/aim
|
||||
|
|
@ -0,0 +1,248 @@
|
|||
use url_parse::core::Parser;
|
||||
|
||||
use crate::netrc::netrc;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct ParsedAddress {
|
||||
pub server: String,
|
||||
pub username: String,
|
||||
pub password: String,
|
||||
pub path_segments: Vec<String>,
|
||||
pub file: String,
|
||||
}
|
||||
|
||||
impl PartialEq for ParsedAddress {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
let result = self.server == other.server
|
||||
&& self.username == other.username
|
||||
&& self.password == other.password
|
||||
&& self.file == other.file;
|
||||
|
||||
let mut paths_equal = true;
|
||||
for it in self.path_segments.iter().zip(self.path_segments.iter()) {
|
||||
let (left, right) = it;
|
||||
paths_equal = paths_equal && (left == right);
|
||||
}
|
||||
|
||||
result && paths_equal
|
||||
}
|
||||
}
|
||||
|
||||
impl ParsedAddress {
|
||||
pub fn parse_address(address: &str, silent: bool) -> ParsedAddress {
|
||||
let netrc = netrc(silent);
|
||||
let url = Parser::new(None).parse(address).unwrap();
|
||||
let server = format!(
|
||||
"{}:{}",
|
||||
url.host_str()
|
||||
.ok_or_else(|| panic!("failed to parse hostname from url: {url}"))
|
||||
.unwrap(),
|
||||
url.port_or_known_default()
|
||||
.ok_or_else(|| panic!("failed to parse port from url: {url}"))
|
||||
.unwrap(),
|
||||
);
|
||||
|
||||
let url_username = url.username();
|
||||
let username = if url_username.is_none() {
|
||||
"anonymous".to_string()
|
||||
} else {
|
||||
url.username().unwrap()
|
||||
};
|
||||
|
||||
let password = url.password().unwrap_or_else(|| "anonymous".to_string());
|
||||
if !silent && username != "anonymous" && password != "anonymous" {
|
||||
println!("🔑 Parsed credentials from URL.");
|
||||
}
|
||||
|
||||
let (username, password) = ParsedAddress::mixin_netrc(&netrc, &server, username, password);
|
||||
|
||||
let mut path_segments: Vec<String> = url
|
||||
.path_segments()
|
||||
.ok_or_else(|| panic!("failed to get url path segments: {url}"))
|
||||
.unwrap();
|
||||
|
||||
let file = path_segments
|
||||
.pop()
|
||||
.ok_or_else(|| panic!("got empty path segments from url: {url}"))
|
||||
.unwrap();
|
||||
|
||||
ParsedAddress {
|
||||
server,
|
||||
username,
|
||||
password,
|
||||
path_segments,
|
||||
file,
|
||||
}
|
||||
}
|
||||
|
||||
fn mixin_netrc(
|
||||
netrc: &Option<netrc::Netrc>,
|
||||
server: &str,
|
||||
username: String,
|
||||
password: String,
|
||||
) -> (String, String) {
|
||||
let mut user = username.clone();
|
||||
let mut pass = password.clone();
|
||||
if !netrc.is_none() && username == "anonymous" && password == "anonymous" {
|
||||
for host in netrc.as_ref().unwrap().hosts.iter().enumerate() {
|
||||
let (_i, (netrc_name, machine)) = host;
|
||||
|
||||
let mut name = netrc_name.to_string();
|
||||
if let Some(port) = machine.port {
|
||||
name = name + ":" + &port.to_string()[..];
|
||||
}
|
||||
if server == name {
|
||||
user = machine.login.clone();
|
||||
pass = machine.password.clone().unwrap();
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
(user, pass)
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parseaddress_operator_equals_works_when_typical() {
|
||||
let left = ParsedAddress {
|
||||
server: "do.main".to_string(),
|
||||
username: "user".to_string(),
|
||||
password: "pass".to_string(),
|
||||
path_segments: vec!["my".to_string(), "path".to_string()],
|
||||
file: "pass".to_string(),
|
||||
};
|
||||
let right = ParsedAddress {
|
||||
server: "do.main".to_string(),
|
||||
username: "user".to_string(),
|
||||
password: "pass".to_string(),
|
||||
path_segments: vec!["my".to_string(), "path".to_string()],
|
||||
file: "pass".to_string(),
|
||||
};
|
||||
|
||||
assert!(left == right);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parseaddress_operator_equals_fails_when_not_equal() {
|
||||
let left = ParsedAddress {
|
||||
server: "do.main".to_string(),
|
||||
username: "user".to_string(),
|
||||
password: "pass".to_string(),
|
||||
path_segments: vec!["my".to_string(), "path".to_string()],
|
||||
file: "pass".to_string(),
|
||||
};
|
||||
let right = ParsedAddress {
|
||||
server: "do".to_string(),
|
||||
username: "user".to_string(),
|
||||
password: "pass".to_string(),
|
||||
path_segments: vec!["my".to_string(), "path".to_string()],
|
||||
file: "pass".to_string(),
|
||||
};
|
||||
|
||||
assert!(left != right);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parse_works() {
|
||||
let expected = ParsedAddress {
|
||||
server: "do.main:21".to_string(),
|
||||
username: "user".to_string(),
|
||||
password: "pass".to_string(),
|
||||
path_segments: vec!["index".to_string()],
|
||||
file: "file".to_string(),
|
||||
};
|
||||
|
||||
let actual = ParsedAddress::parse_address("ftp://user:pass@do.main:21/index/file", true);
|
||||
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mixin_works() {
|
||||
let expected_username = "test";
|
||||
let expected_password = "p@ssw0rd";
|
||||
let input = "machine example.com login test password p@ssw0rd";
|
||||
let input = std::io::BufReader::new(input.as_bytes());
|
||||
let netrc = netrc::Netrc::parse(input).unwrap();
|
||||
let username_decoded_from_url = "anonymous".to_string();
|
||||
let password_decoded_from_url = "anonymous".to_string();
|
||||
|
||||
let (actual_username, actual_password) = ParsedAddress::mixin_netrc(
|
||||
&Some(netrc),
|
||||
"example.com",
|
||||
username_decoded_from_url,
|
||||
password_decoded_from_url,
|
||||
);
|
||||
|
||||
assert_eq!(actual_username, expected_username);
|
||||
assert_eq!(actual_password, expected_password);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mixin_works_with_port() {
|
||||
let expected_username = "test";
|
||||
let expected_password = "p@ssw0rd";
|
||||
let input = "machine example.com login test password p@ssw0rd port 443";
|
||||
let input = std::io::BufReader::new(input.as_bytes());
|
||||
let netrc = netrc::Netrc::parse(input).unwrap();
|
||||
let username_decoded_from_url = "anonymous".to_string();
|
||||
let password_decoded_from_url = "anonymous".to_string();
|
||||
|
||||
let (actual_username, actual_password) = ParsedAddress::mixin_netrc(
|
||||
&Some(netrc),
|
||||
"example.com:443",
|
||||
username_decoded_from_url,
|
||||
password_decoded_from_url,
|
||||
);
|
||||
|
||||
assert_eq!(actual_username, expected_username);
|
||||
assert_eq!(actual_password, expected_password);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parse_works_with_netrc_mixin() {
|
||||
let expected = ParsedAddress {
|
||||
server: "do.main:21".to_string(),
|
||||
username: "test".to_string(),
|
||||
password: "p@ssw0rd".to_string(),
|
||||
path_segments: vec!["index".to_string()],
|
||||
file: "file".to_string(),
|
||||
};
|
||||
let data = "machine do.main login test password p@ssw0rd port 21";
|
||||
|
||||
std::fs::write(".netrc.test", data).expect("Unable to write file");
|
||||
let actual = ParsedAddress::parse_address("ftp://do.main/index/file", true);
|
||||
|
||||
assert_eq!(actual, expected);
|
||||
std::fs::remove_file(".netrc.test").unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parse_works_when_ssh_user() {
|
||||
let expected = ParsedAddress {
|
||||
server: "localhost:2223".to_string(),
|
||||
username: "user".to_string(),
|
||||
password: "anonymous".to_string(),
|
||||
path_segments: vec!["".to_string()],
|
||||
file: "file".to_string(),
|
||||
};
|
||||
|
||||
let actual = ParsedAddress::parse_address("ssh://user@localhost:2223/file", true);
|
||||
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn parse_works_when_not_silent() {
|
||||
let expected = ParsedAddress {
|
||||
server: "localhost:2223".to_string(),
|
||||
username: "user".to_string(),
|
||||
password: "pass".to_string(),
|
||||
path_segments: vec!["".to_string()],
|
||||
file: "file".to_string(),
|
||||
};
|
||||
|
||||
let actual = ParsedAddress::parse_address("ssh://user:pass@localhost:2223/file", false);
|
||||
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
|
@ -0,0 +1,165 @@
|
|||
use std::{collections::HashMap, env};
|
||||
|
||||
use dotenvy::dotenv;
|
||||
use indicatif::{ProgressBar, ProgressDrawTarget, ProgressStyle};
|
||||
use strfmt::strfmt;
|
||||
|
||||
const DEFAULT_AIM_PROGRESSBAR_DOWNLOADED_MESSAGE: &str = "🎯 Downloaded {input} to {output}";
|
||||
const DEFAULT_AIM_PROGRESSBAR_MESSAGE_FORMAT: &str = "🎯 Transferring {url}";
|
||||
const DEFAULT_AIM_PROGRESSBAR_PROGRESS_CHARS: &str = "█▉▊▋▌▍▎▏ ";
|
||||
const DEFAULT_AIM_PROGRESSBAR_TEMPLATE: &str = "{msg}\n{spinner:.cyan} {elapsed_precise} ▕{bar:.white}▏ {bytes}/{total_bytes} {bytes_per_sec} ETA {eta}.";
|
||||
const DEFAULT_AIM_PROGRESSBAR_UPLOADED_MESSAGE: &str = "🎯 Uploaded {input} to {output}";
|
||||
|
||||
const THRESHOLD_IF_TOTALBYTES_BELOW_THEN_AUTO_SILENT_MODE: u64 = 1024 * 1024;
|
||||
|
||||
fn construct_progress_bar(
|
||||
total_size: u64,
|
||||
url: &str,
|
||||
message_format: &str,
|
||||
progress_chars: &str,
|
||||
template: &str,
|
||||
) -> indicatif::ProgressBar {
|
||||
let pb = ProgressBar::new(total_size);
|
||||
pb.set_draw_target(ProgressDrawTarget::hidden());
|
||||
let mut vars: HashMap<String, String> = HashMap::new();
|
||||
|
||||
if message_format.contains("{url}") {
|
||||
vars.insert("url".to_string(), url.to_string());
|
||||
}
|
||||
|
||||
pb.set_message(strfmt(message_format, &vars).unwrap());
|
||||
pb.set_style(
|
||||
ProgressStyle::default_bar()
|
||||
.template(template)
|
||||
.unwrap()
|
||||
.progress_chars(progress_chars),
|
||||
);
|
||||
pb
|
||||
}
|
||||
|
||||
pub struct WrappedBar {
|
||||
pub silent: bool,
|
||||
pub output: Option<indicatif::ProgressBar>,
|
||||
downloaded_message: String,
|
||||
uploaded_message: String,
|
||||
}
|
||||
|
||||
impl WrappedBar {
|
||||
pub fn new_empty() -> Self {
|
||||
WrappedBar {
|
||||
silent: true,
|
||||
output: None,
|
||||
downloaded_message: "".to_string(),
|
||||
uploaded_message: "".to_string(),
|
||||
}
|
||||
}
|
||||
pub fn new_empty_verbose() -> Self {
|
||||
WrappedBar {
|
||||
silent: false,
|
||||
output: None,
|
||||
downloaded_message: "".to_string(),
|
||||
uploaded_message: "".to_string(),
|
||||
}
|
||||
}
|
||||
pub fn new(total_size: u64, url: &str, silent: bool) -> Self {
|
||||
dotenv().ok();
|
||||
let message_format = &env::var("AIM_PROGRESSBAR_MESSAGE_FORMAT")
|
||||
.unwrap_or_else(|_| DEFAULT_AIM_PROGRESSBAR_MESSAGE_FORMAT.to_string());
|
||||
let progress_chars = &env::var("AIM_PROGRESSBAR_PROGRESS_CHARS")
|
||||
.unwrap_or_else(|_| DEFAULT_AIM_PROGRESSBAR_PROGRESS_CHARS.to_string());
|
||||
let template = &env::var("AIM_PROGRESSBAR_TEMPLATE")
|
||||
.unwrap_or_else(|_| DEFAULT_AIM_PROGRESSBAR_TEMPLATE.to_string());
|
||||
let downloaded_message = &env::var("AIM_PROGRESSBAR_DOWNLOADED_MESSAGE")
|
||||
.unwrap_or_else(|_| DEFAULT_AIM_PROGRESSBAR_DOWNLOADED_MESSAGE.to_string());
|
||||
let uploaded_message = &env::var("AIM_PROGRESSBAR_UPLOADED_MESSAGE")
|
||||
.unwrap_or_else(|_| DEFAULT_AIM_PROGRESSBAR_UPLOADED_MESSAGE.to_string());
|
||||
let output = match silent {
|
||||
false => Some(construct_progress_bar(
|
||||
total_size,
|
||||
url,
|
||||
message_format,
|
||||
progress_chars,
|
||||
template,
|
||||
)),
|
||||
true => None,
|
||||
};
|
||||
WrappedBar {
|
||||
silent,
|
||||
output,
|
||||
downloaded_message: downloaded_message.to_string(),
|
||||
uploaded_message: uploaded_message.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_length(&mut self, len: u64) {
|
||||
if len < THRESHOLD_IF_TOTALBYTES_BELOW_THEN_AUTO_SILENT_MODE {
|
||||
self.silent = true;
|
||||
}
|
||||
if !self.silent {
|
||||
self.output
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.set_draw_target(ProgressDrawTarget::stderr());
|
||||
self.output.as_ref().unwrap().set_length(len);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set_position(&self, pos: u64) {
|
||||
if !self.silent {
|
||||
self.output.as_ref().unwrap().set_position(pos);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn finish_download(&self, input: &str, output: &str) {
|
||||
if !self.silent {
|
||||
let mut vars: HashMap<String, String> = HashMap::new();
|
||||
|
||||
if self.downloaded_message.contains("{input}") {
|
||||
vars.insert("input".to_string(), input.to_string());
|
||||
}
|
||||
|
||||
if self.downloaded_message.contains("{output}") {
|
||||
vars.insert("output".to_string(), output.to_string());
|
||||
}
|
||||
self.output
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.finish_with_message(strfmt(&self.downloaded_message, &vars).unwrap());
|
||||
}
|
||||
}
|
||||
pub fn finish_upload(&self, input: &str, output: &str) {
|
||||
if !self.silent {
|
||||
let mut vars: HashMap<String, String> = HashMap::new();
|
||||
|
||||
if self.uploaded_message.contains("{input}") {
|
||||
vars.insert("input".to_string(), input.to_string());
|
||||
}
|
||||
|
||||
if self.uploaded_message.contains("{output}") {
|
||||
vars.insert("output".to_string(), output.to_string());
|
||||
}
|
||||
self.output
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.finish_with_message(strfmt(&self.uploaded_message, &vars).unwrap());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bar_finish_download_works_when_typical() {
|
||||
let bar = WrappedBar::new(42, "url", false);
|
||||
bar.finish_download("", "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bar_finish_upload_works_when_typical() {
|
||||
let bar = WrappedBar::new(42, "url", false);
|
||||
bar.finish_upload("", "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bar_set_length_works_when_typical() {
|
||||
let mut bar = WrappedBar::new(42, "url", false);
|
||||
bar.set_length(42);
|
||||
}
|
||||
|
|
@ -0,0 +1,7 @@
|
|||
use clap::crate_version;
|
||||
|
||||
pub const CLIENT_ID: &str = concat!(
|
||||
env!("CARGO_PKG_REPOSITORY"),
|
||||
"/releases/tag/",
|
||||
crate_version!()
|
||||
);
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
extern crate custom_error;
|
||||
use custom_error::custom_error;
|
||||
|
||||
custom_error! {
|
||||
pub ValidateError
|
||||
Sha256Mismatch = "Invalid sha256.",
|
||||
}
|
||||
|
||||
custom_error! {
|
||||
pub HTTPHeaderError
|
||||
NotPresent = "Cannot find requested header.",
|
||||
}
|
||||
|
||||
impl From<ValidateError> for std::io::Error {
|
||||
fn from(cause: ValidateError) -> std::io::Error {
|
||||
std::io::Error::new(std::io::ErrorKind::Other, cause.to_string())
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_from_validate_error_to_std_io_error_works_when_typical() {
|
||||
let _ = match Err(ValidateError::Sha256Mismatch) {
|
||||
Ok(v) => v,
|
||||
Err(e) => {
|
||||
let error: std::io::Error = e.into();
|
||||
error
|
||||
}
|
||||
};
|
||||
}
|
||||
|
|
@ -0,0 +1,77 @@
|
|||
use std::{fs, io, str};
|
||||
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
use crate::error::ValidateError;
|
||||
|
||||
pub struct HashChecker;
|
||||
impl HashChecker {
|
||||
pub fn check(filename: &str, expected_hash: &str) -> Result<(), ValidateError> {
|
||||
let mut result = Ok(());
|
||||
if filename != "stdout" && (!expected_hash.is_empty()) {
|
||||
let actual_hash = HashChecker::sha256sum(filename);
|
||||
if actual_hash != expected_hash {
|
||||
result = Err(ValidateError::Sha256Mismatch);
|
||||
}
|
||||
match result {
|
||||
Ok(()) => println!("✅ Checksum OK."),
|
||||
Err(ValidateError::Sha256Mismatch) => println!(
|
||||
"❌ Checksum verification failed for {filename}:\n expected: {expected_hash}\n got: {actual_hash}"),
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn sha256sum(filename: &str) -> String {
|
||||
let mut hasher = Sha256::new();
|
||||
let mut file = fs::File::open(filename).unwrap();
|
||||
|
||||
io::copy(&mut file, &mut hasher).unwrap();
|
||||
let computed_hash = hasher.finalize();
|
||||
drop(file);
|
||||
|
||||
format!("{computed_hash:x}")
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
macro_rules! assert_err {
|
||||
($expression:expr, $($pattern:tt)+) => {
|
||||
match $expression {
|
||||
$($pattern)+ => (),
|
||||
ref e => panic!("expected `{}` but got `{:?}`", stringify!($($pattern)+), e),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_api_fails_when_checksum_mismatch() {
|
||||
let expected = "AAAA847124bfb9d9a9d44af6f00d8003006c44b9ef9ba458b5d4d3fc5f81bde5";
|
||||
|
||||
assert_err!(
|
||||
HashChecker::check("LICENCE.md", expected),
|
||||
Err(ValidateError::Sha256Mismatch)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sha256sum_api() {
|
||||
let expected = "21d7847124bfb9d9a9d44af6f00d8003006c44b9ef9ba458b5d4d3fc5f81bde5";
|
||||
|
||||
let actual = HashChecker::sha256sum("LICENCE.md");
|
||||
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_api_works_when_typical() {
|
||||
let expected = "21d7847124bfb9d9a9d44af6f00d8003006c44b9ef9ba458b5d4d3fc5f81bde5";
|
||||
|
||||
let is_match = HashChecker::check("LICENCE.md", expected).is_ok();
|
||||
|
||||
assert!(is_match);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,268 @@
|
|||
use std::{cmp::min, io::Error};
|
||||
|
||||
use futures_util::StreamExt;
|
||||
use regex::Regex;
|
||||
use reqwest::Client;
|
||||
use tokio_util::io::ReaderStream;
|
||||
|
||||
use crate::{
|
||||
address::ParsedAddress, bar::WrappedBar, consts::*, error::ValidateError, hash::HashChecker, io,
|
||||
};
|
||||
|
||||
pub struct HTTPSHandler;
|
||||
impl HTTPSHandler {
|
||||
pub async fn get(
|
||||
input: &str,
|
||||
output: &str,
|
||||
bar: &mut WrappedBar,
|
||||
expected_sha256: &str,
|
||||
) -> Result<(), ValidateError> {
|
||||
HTTPSHandler::_get(input, output, bar).await?;
|
||||
HashChecker::check(output, expected_sha256)
|
||||
}
|
||||
|
||||
pub async fn put(input: &str, output: &str, mut bar: WrappedBar) -> Result<(), ValidateError> {
|
||||
let parsed_address = ParsedAddress::parse_address(output, bar.silent);
|
||||
let file = tokio::fs::File::open(&input)
|
||||
.await
|
||||
.expect("Cannot open input file for HTTPS read");
|
||||
let total_size = file
|
||||
.metadata()
|
||||
.await
|
||||
.expect("Cannot determine input file size for HTTPS read")
|
||||
.len();
|
||||
let input_ = input.to_string();
|
||||
let output_ = output.to_string();
|
||||
let mut reader_stream = ReaderStream::new(file);
|
||||
|
||||
let mut uploaded = HTTPSHandler::get_already_uploaded(output, bar.silent).await;
|
||||
bar.set_length(total_size);
|
||||
|
||||
let async_stream = async_stream::stream! {
|
||||
while let Some(chunk) = reader_stream.next().await {
|
||||
if let Ok(chunk) = &chunk {
|
||||
let new = min(uploaded + (chunk.len() as u64), total_size);
|
||||
uploaded = new;
|
||||
bar.set_position(new);
|
||||
if uploaded >= total_size {
|
||||
bar.finish_upload(&input_, &output_);
|
||||
}
|
||||
}
|
||||
yield chunk;
|
||||
}
|
||||
};
|
||||
|
||||
let response = reqwest::Client::new()
|
||||
.put(output)
|
||||
.header("content-type", "application/octet-stream")
|
||||
.header(
|
||||
"Range",
|
||||
"bytes=".to_owned() + &uploaded.to_string()[..] + "-",
|
||||
)
|
||||
.header(
|
||||
reqwest::header::USER_AGENT,
|
||||
reqwest::header::HeaderValue::from_static(CLIENT_ID),
|
||||
)
|
||||
.basic_auth(parsed_address.username, Some(parsed_address.password))
|
||||
.body(reqwest::Body::wrap_stream(async_stream))
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
println!("{:?}", response.text().await.unwrap());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get_links(input: String) -> Result<Vec<String>, Error> {
|
||||
let mut result = Vec::new();
|
||||
let res = HTTPSHandler::list(&input).await.unwrap();
|
||||
let lines: Vec<&str> = res.split('\n').collect();
|
||||
|
||||
for line in lines {
|
||||
let re = Regex::new(r#".*href="/(.+?)".*"#).unwrap();
|
||||
let caps = re.captures(line);
|
||||
if let Some(e) = caps {
|
||||
result.push(e.get(1).unwrap().as_str().to_string())
|
||||
}
|
||||
}
|
||||
result.push("..".to_string());
|
||||
|
||||
result.sort();
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn list(input: &str) -> Result<String, ValidateError> {
|
||||
let is_silent = true;
|
||||
let parsed_address = ParsedAddress::parse_address(input, is_silent);
|
||||
|
||||
let res = Client::new()
|
||||
.get(input)
|
||||
.header(
|
||||
reqwest::header::USER_AGENT,
|
||||
reqwest::header::HeaderValue::from_static(CLIENT_ID),
|
||||
)
|
||||
.basic_auth(parsed_address.username, Some(parsed_address.password))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|_| format!("Failed to GET from {}", &input))
|
||||
.unwrap()
|
||||
.text()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
Ok(res)
|
||||
}
|
||||
|
||||
async fn _get(input: &str, output: &str, bar: &mut WrappedBar) -> Result<(), ValidateError> {
|
||||
let parsed_address = ParsedAddress::parse_address(input, bar.silent);
|
||||
let (mut out, mut downloaded) = io::get_output(output, bar.silent);
|
||||
|
||||
let res = Client::new()
|
||||
.get(input)
|
||||
.header(
|
||||
"Range",
|
||||
"bytes=".to_owned() + &downloaded.to_string()[..] + "-",
|
||||
)
|
||||
.header(
|
||||
reqwest::header::USER_AGENT,
|
||||
reqwest::header::HeaderValue::from_static(CLIENT_ID),
|
||||
)
|
||||
.basic_auth(parsed_address.username, Some(parsed_address.password))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|_| format!("Failed to GET from {} to {}", &input, &output))
|
||||
.unwrap();
|
||||
let total_size = downloaded + res.content_length().unwrap_or(0);
|
||||
|
||||
bar.set_length(total_size);
|
||||
|
||||
let mut stream = res.bytes_stream();
|
||||
while let Some(item) = stream.next().await {
|
||||
let chunk = item.map_err(|_| "Error while downloading.").unwrap();
|
||||
out.write_all(&chunk)
|
||||
.map_err(|_| "Error while writing to output.")
|
||||
.unwrap();
|
||||
let new = min(downloaded + (chunk.len() as u64), total_size);
|
||||
downloaded = new;
|
||||
bar.set_position(new);
|
||||
}
|
||||
|
||||
bar.finish_download(input, output);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn get_already_uploaded(output: &str, silent: bool) -> u64 {
|
||||
let parsed_address = ParsedAddress::parse_address(output, silent);
|
||||
let res = Client::new()
|
||||
.get(output)
|
||||
.header(
|
||||
reqwest::header::USER_AGENT,
|
||||
reqwest::header::HeaderValue::from_static(CLIENT_ID),
|
||||
)
|
||||
.basic_auth(parsed_address.username, Some(parsed_address.password))
|
||||
.send()
|
||||
.await
|
||||
.map_err(|_| format!("Failed to GET already uploaded size from {}", &output))
|
||||
.unwrap();
|
||||
res.content_length().unwrap_or(0)
|
||||
}
|
||||
}
|
||||
|
||||
#[ignore]
|
||||
#[tokio::test]
|
||||
async fn get_https_works() {
|
||||
let expected_hash = "0e0f0d7139c8c7e3ff20cb243e94bc5993517d88e8be8d59129730607d5c631b";
|
||||
let out_file = "tokei-x86_64-unknown-linux-gnu.tar.gz";
|
||||
|
||||
let result = HTTPSHandler::get("https://github.com/XAMPPRocky/tokei/releases/download/v12.0.4/tokei-x86_64-unknown-linux-gnu.tar.gz", out_file, &mut WrappedBar::new_empty(), expected_hash).await;
|
||||
|
||||
assert!(result.is_ok());
|
||||
std::fs::remove_file(out_file).unwrap();
|
||||
}
|
||||
|
||||
#[ignore]
|
||||
#[tokio::test]
|
||||
async fn get_resume_works() {
|
||||
let expected_size = 561553;
|
||||
let out_file = "test/dua-v2.10.2-x86_64-unknown-linux-musl.tar.gz";
|
||||
std::fs::copy(
|
||||
"test/incomplete_dua-v2.10.2-x86_64-unknown-linux-musl.tar.gz",
|
||||
out_file,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
let _ = HTTPSHandler::get("https://github.com/Byron/dua-cli/releases/download/v2.10.2/dua-v2.10.2-x86_64-unknown-linux-musl.tar.gz", out_file, &mut WrappedBar::new_empty_verbose(), "").await;
|
||||
|
||||
let actual_size = std::fs::metadata(out_file).unwrap().len();
|
||||
assert_eq!(actual_size, expected_size);
|
||||
std::fs::remove_file(out_file).unwrap();
|
||||
}
|
||||
|
||||
#[ignore]
|
||||
#[tokio::test]
|
||||
async fn list_works_when_typical() {
|
||||
let expected = r#"<!doctype html>
|
||||
<html>
|
||||
<head>
|
||||
<title>Example Domain</title>
|
||||
|
||||
<meta charset="utf-8" />
|
||||
<meta http-equiv="Content-type" content="text/html; charset=utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<style type="text/css">
|
||||
body {
|
||||
background-color: #f0f0f2;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
font-family: -apple-system, system-ui, BlinkMacSystemFont, "Segoe UI", "Open Sans", "Helvetica Neue", Helvetica, Arial, sans-serif;
|
||||
}
|
||||
div {
|
||||
width: 600px;
|
||||
margin: 5em auto;
|
||||
padding: 2em;
|
||||
background-color: #fdfdff;
|
||||
border-radius: 0.5em;
|
||||
box-shadow: 2px 3px 7px 2px rgba(0,0,0,0.02);
|
||||
}
|
||||
a:link, a:visited {
|
||||
color: #38488f;
|
||||
text-decoration: none;
|
||||
}
|
||||
@media (max-width: 700px) {
|
||||
div {
|
||||
margin: 0 auto;
|
||||
width: auto;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<div>
|
||||
<h1>Example Domain</h1>
|
||||
<p>This domain is for use in illustrative examples in documents. You may use this
|
||||
domain in literature without prior coordination or asking for permission.</p>
|
||||
<p><a href="https://www.iana.org/domains/example">More information...</a></p>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
"#;
|
||||
|
||||
let result = HTTPSHandler::list("https://example.com").await.unwrap();
|
||||
let result = str::replace(&result, " \n ", "");
|
||||
let result = str::replace(&result, "</style> \n</head>", "</style>\n</head>");
|
||||
|
||||
assert_eq!(result, expected);
|
||||
}
|
||||
|
||||
#[ignore]
|
||||
#[tokio::test]
|
||||
async fn get_links_works_when_typical() {
|
||||
let expected = "..";
|
||||
|
||||
let result = HTTPSHandler::get_links("https://github.com/mihaigalos/aim/releases".to_string())
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(result[0], expected);
|
||||
}
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
use std::{fs::File, io::Write};
|
||||
|
||||
fn get_output_file(path: &str, silent: bool) -> (Option<std::fs::File>, u64) {
|
||||
let mut transferred: u64 = 0;
|
||||
let mut file = None;
|
||||
if path != "stdout" {
|
||||
if std::path::Path::new(path).exists() {
|
||||
if !silent {
|
||||
println!("File exists. Resuming.");
|
||||
}
|
||||
file = Some(
|
||||
std::fs::OpenOptions::new()
|
||||
.write(true)
|
||||
.append(true)
|
||||
.open(path)
|
||||
.unwrap(),
|
||||
);
|
||||
|
||||
let file_size = std::fs::metadata(path).unwrap().len();
|
||||
transferred = file_size;
|
||||
} else {
|
||||
if !silent {
|
||||
println!("Writing to new file.");
|
||||
}
|
||||
file = Some(
|
||||
File::create(path)
|
||||
.map_err(|_| format!("Failed to create file '{path}'"))
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
}
|
||||
(file, transferred)
|
||||
}
|
||||
|
||||
pub fn get_output(path: &str, silent: bool) -> (Box<dyn Write + Send>, u64) {
|
||||
let (file, transferred) = get_output_file(path, silent);
|
||||
let output: Box<dyn Write + Send> = Box::new(std::io::BufWriter::new(match path {
|
||||
"stdout" => Box::new(std::io::stdout()) as Box<dyn Write + Send>,
|
||||
_ => Box::new(file.unwrap()) as Box<dyn Write + Send>,
|
||||
}));
|
||||
|
||||
(output, transferred)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_output_file_file_is_none_when_stdout() {
|
||||
let is_silet = true;
|
||||
let (file, _) = get_output_file("stdout", is_silet);
|
||||
assert!(file.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_output_file_pos_is_zero_when_stdout() {
|
||||
let is_silet = true;
|
||||
let (_, position) = get_output_file("stdout", is_silet);
|
||||
assert_eq!(position, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_output_file_file_is_none_when_newfile() {
|
||||
let is_silet = true;
|
||||
let filename = "test_get_output_file_file_is_none_when_newfile";
|
||||
|
||||
let (file, _) = get_output_file(filename, is_silet);
|
||||
|
||||
assert!(file.is_some());
|
||||
std::fs::remove_file(filename).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_output_file_file_is_none_when_newfile_and_not_silent() {
|
||||
let is_silet = false;
|
||||
let filename = "test_get_output_file_file_is_none_when_newfile_and_not_silent";
|
||||
|
||||
let (file, _) = get_output_file(filename, is_silet);
|
||||
|
||||
assert!(file.is_some());
|
||||
std::fs::remove_file(filename).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_get_output_file_file_is_none_when_existingfile_and_not_silent() {
|
||||
use std::io::Write;
|
||||
let is_silet = false;
|
||||
let filename = "test_get_output_file_file_is_none_when_existingfile_and_not_silent";
|
||||
let expected_position_byte = 4;
|
||||
let mut file = File::create(filename).unwrap();
|
||||
file.write_all(b"1234").unwrap();
|
||||
|
||||
let (_, position) = get_output_file(filename, is_silet);
|
||||
|
||||
assert_eq!(position, expected_position_byte);
|
||||
std::fs::remove_file(filename).unwrap();
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
pub mod bar;
|
||||
pub mod https;
|
||||
|
||||
mod address;
|
||||
mod consts;
|
||||
mod error;
|
||||
mod hash;
|
||||
mod io;
|
||||
mod netrc;
|
||||
mod untildify;
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
use std::{io::BufReader, path::Path};
|
||||
|
||||
use netrc::Netrc;
|
||||
|
||||
use crate::untildify::untildify;
|
||||
|
||||
fn get_possible_netrc_path(silent: bool) -> String {
|
||||
let candidates = vec![
|
||||
".netrc",
|
||||
".netrc.test",
|
||||
".netrc.test_https",
|
||||
".netrc.test_ftp",
|
||||
".netrc.test_unit",
|
||||
"~/.netrc",
|
||||
];
|
||||
for candidate in candidates {
|
||||
let candidate = untildify(candidate);
|
||||
if Path::new(&candidate).exists() {
|
||||
if !silent {
|
||||
println!("🔑 Parsed .netrc from: {candidate}");
|
||||
}
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
"".to_string()
|
||||
}
|
||||
|
||||
pub fn netrc(silent: bool) -> Option<netrc::Netrc> {
|
||||
let mut result = None;
|
||||
let path = get_possible_netrc_path(silent);
|
||||
if !path.is_empty() {
|
||||
let file = std::fs::File::open(path).unwrap();
|
||||
let parsed = Netrc::parse(BufReader::new(file));
|
||||
result = Some(parsed.unwrap());
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_netrc_with_file_works_when_typical() {
|
||||
use std::io::Write;
|
||||
let mut file = std::fs::File::create(".netrc.test_unit").unwrap();
|
||||
file.write_all(b"machine mydomain.com login myuser password mypass port 1234")
|
||||
.unwrap();
|
||||
|
||||
assert!(netrc(true).is_some());
|
||||
assert!(netrc(false).is_some());
|
||||
|
||||
std::fs::remove_file(".netrc.test_unit").unwrap();
|
||||
}
|
||||
|
||||
#[ignore]
|
||||
#[test]
|
||||
fn test_netrc_with_file_works_when_typical_and_not_silent() {
|
||||
use std::io::Write;
|
||||
let mut file = std::fs::File::create(".netrc.test_unit").unwrap();
|
||||
file.write_all(b"machine mydomain.com login myuser password mypass port 1234")
|
||||
.unwrap();
|
||||
|
||||
let netrc = netrc(false);
|
||||
|
||||
assert!(netrc.is_some());
|
||||
std::fs::remove_file(".netrc.test_unit").unwrap();
|
||||
}
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
//! Replace `~` with user home directory across all platforms
|
||||
//!
|
||||
//! Unit tests come from https://github.com/sathishsoundharajan/untildify (MIT LICENSE)
|
||||
|
||||
pub fn untildify(input_path: &str) -> String {
|
||||
if input_path.is_empty() {
|
||||
return String::from(input_path);
|
||||
}
|
||||
if let Some(home) = home::home_dir() {
|
||||
if input_path == r"~" {
|
||||
return home.into_os_string().into_string().unwrap();
|
||||
}
|
||||
if input_path.starts_with(r"~/") || input_path.starts_with(r"~\") {
|
||||
if let Ok(path) = home.join(&input_path[2..]).into_os_string().into_string() {
|
||||
return path;
|
||||
}
|
||||
}
|
||||
}
|
||||
String::from(input_path)
|
||||
}
|
||||
|
||||
#[cfg(any(unix, target_os = "redox"))]
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::{env, path::Path};
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_returns_untildfyed_string() {
|
||||
env::remove_var("HOME");
|
||||
|
||||
let home = Path::new("/User/Untildify");
|
||||
env::set_var("HOME", home.as_os_str());
|
||||
|
||||
assert_eq!(untildify("~/Desktop"), "/User/Untildify/Desktop");
|
||||
assert_eq!(untildify("~/a/b/c/d/e"), "/User/Untildify/a/b/c/d/e");
|
||||
assert_eq!(untildify("~/"), "/User/Untildify/");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_returns_empty_string() {
|
||||
env::remove_var("HOME");
|
||||
|
||||
let home = Path::new("/User/Untildify");
|
||||
env::set_var("HOME", home.as_os_str());
|
||||
|
||||
assert_eq!(untildify("Desktop"), "Desktop");
|
||||
assert_eq!(untildify(""), "");
|
||||
assert_eq!(untildify("/"), "/");
|
||||
// assert_eq!(untildify("~/Desktop/~/Code"), "/User/Untildify/Desktop/");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_with_dot_folders() {
|
||||
env::remove_var("HOME");
|
||||
|
||||
let home = Path::new("/User/Untildify");
|
||||
env::set_var("HOME", home.as_os_str());
|
||||
|
||||
assert_eq!(untildify("~/.ssh/id_rsa"), "/User/Untildify/.ssh/id_rsa");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::env;
|
||||
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_returns_untildfyed_string() {
|
||||
env::set_var("USERPROFILE", r"C:\Users\Admin");
|
||||
|
||||
assert_eq!(untildify(r"~\Desktop"), r"C:\Users\Admin\Desktop");
|
||||
assert_eq!(untildify(r"~\a\b\c\d\e"), r"C:\Users\Admin\a\b\c\d\e");
|
||||
assert_eq!(untildify(r"~\"), r"C:\Users\Admin\");
|
||||
}
|
||||
}
|
||||
|
|
@ -4,9 +4,9 @@ version = "0.7.0-dev"
|
|||
edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
aim-downloader = { path = "../aim-downloader" }
|
||||
tabby-common = { path = "../tabby-common" }
|
||||
anyhow = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
tokio-retry = "0.3.0"
|
||||
sha256 = "1.4.0"
|
||||
aim = "1.8.5"
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
use std::{fs, path::Path};
|
||||
|
||||
use aim::bar::WrappedBar;
|
||||
use aim_downloader::{bar::WrappedBar, https};
|
||||
use anyhow::{anyhow, Result};
|
||||
use tabby_common::registry::{parse_model_id, ModelRegistry};
|
||||
use tokio_retry::{
|
||||
|
|
@ -60,7 +60,7 @@ async fn download_file(url: &str, path: &Path) -> Result<()> {
|
|||
|
||||
let mut bar = WrappedBar::new(0, url, false);
|
||||
|
||||
aim::https::HTTPSHandler::get(url, &intermediate_filename, &mut bar, "").await?;
|
||||
https::HTTPSHandler::get(url, &intermediate_filename, &mut bar, "").await?;
|
||||
|
||||
fs::rename(intermediate_filename, filename)?;
|
||||
Ok(())
|
||||
|
|
|
|||
Loading…
Reference in New Issue