update: sync for alm
This commit is contained in:
parent
c07aee708f
commit
66a78912e3
1 changed files with 72 additions and 51 deletions
|
|
@ -180,63 +180,84 @@ pub fn to_array(value: Dynamic) -> Array {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(feature = "progress-bars")]
|
|
||||||
pub async fn download_file(url: &str, output_path: &str) -> Result<(), anyhow::Error> {
|
pub async fn download_file(url: &str, output_path: &str) -> Result<(), anyhow::Error> {
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
use tokio::time::{sleep, timeout};
|
||||||
let url = url.to_string();
|
let url = url.to_string();
|
||||||
let output_path = output_path.to_string();
|
let output_path = output_path.to_string();
|
||||||
let download_handle = tokio::spawn(async move {
|
let download_handle = tokio::spawn(async move {
|
||||||
let client = Client::builder()
|
let max_retries = 3;
|
||||||
.user_agent("Mozilla/5.0 (compatible; BotServer/1.0)")
|
for attempt in 1..=max_retries {
|
||||||
.connect_timeout(Duration::from_secs(30))
|
let client = Client::builder()
|
||||||
.read_timeout(Duration::from_secs(300))
|
.user_agent("Mozilla/5.0 (compatible; BotServer/1.0)")
|
||||||
.pool_idle_timeout(Duration::from_secs(90))
|
.connect_timeout(Duration::from_secs(30))
|
||||||
.tcp_keepalive(Duration::from_secs(60))
|
.read_timeout(Duration::from_secs(60))
|
||||||
.build()?;
|
.pool_idle_timeout(Duration::from_secs(90))
|
||||||
let response = client.get(&url).send().await?;
|
.tcp_keepalive(Duration::from_secs(60))
|
||||||
if response.status().is_success() {
|
.build()?;
|
||||||
let total_size = response.content_length().unwrap_or(0);
|
match client.get(&url).send().await {
|
||||||
let pb = ProgressBar::new(total_size);
|
Ok(mut response) if response.status().is_success() => {
|
||||||
pb.set_style(ProgressStyle::default_bar()
|
let total_size = response.content_length();
|
||||||
.template("{msg}\n{spinner:.green} [{elapsed_precise}] [{bar:40.cyan/blue}] {bytes}/{total_bytes} ({eta})")
|
if let Ok(mut file) = TokioFile::create(&output_path).await {
|
||||||
.unwrap_or(ProgressStyle::default_bar())
|
let mut downloaded: u64 = 0;
|
||||||
.progress_chars("#>-"));
|
let mut last_percent = 0;
|
||||||
pb.set_message(format!("Downloading {}", url));
|
let mut success = true;
|
||||||
let mut file = TokioFile::create(&output_path).await?;
|
loop {
|
||||||
let bytes = response.bytes().await?;
|
match timeout(Duration::from_secs(30), response.chunk()).await {
|
||||||
file.write_all(&bytes).await?;
|
Ok(Ok(Some(chunk))) => {
|
||||||
pb.set_position(bytes.len() as u64);
|
if file.write_all(&chunk).await.is_err() {
|
||||||
pb.finish_with_message(format!("Downloaded {}", output_path));
|
success = false;
|
||||||
Ok(())
|
break;
|
||||||
} else {
|
}
|
||||||
Err(anyhow::anyhow!("HTTP {}: {}", response.status(), url))
|
downloaded += chunk.len() as u64;
|
||||||
}
|
if let Some(total) = total_size {
|
||||||
});
|
let percent = (downloaded as f64 / total as f64 * 100.0) as u64;
|
||||||
download_handle.await?
|
if percent > last_percent && percent % 10 == 0 {
|
||||||
}
|
println!("Downloading {}: {}%", url, percent);
|
||||||
|
last_percent = percent;
|
||||||
#[cfg(not(feature = "progress-bars"))]
|
}
|
||||||
pub async fn download_file(url: &str, output_path: &str) -> Result<(), anyhow::Error> {
|
}
|
||||||
use std::time::Duration;
|
}
|
||||||
let url = url.to_string();
|
Ok(Ok(None)) => break,
|
||||||
let output_path = output_path.to_string();
|
Ok(Err(e)) => {
|
||||||
let download_handle = tokio::spawn(async move {
|
log::warn!("Chunk error: {}", e);
|
||||||
let client = Client::builder()
|
success = false;
|
||||||
.user_agent("Mozilla/5.0 (compatible; BotServer/1.0)")
|
break;
|
||||||
.connect_timeout(Duration::from_secs(30))
|
}
|
||||||
.read_timeout(Duration::from_secs(300))
|
Err(_) => {
|
||||||
.pool_idle_timeout(Duration::from_secs(90))
|
log::warn!("Timeout reading chunk");
|
||||||
.tcp_keepalive(Duration::from_secs(60))
|
success = false;
|
||||||
.build()?;
|
break;
|
||||||
let response = client.get(&url).send().await?;
|
}
|
||||||
if response.status().is_success() {
|
}
|
||||||
let mut file = TokioFile::create(&output_path).await?;
|
}
|
||||||
let bytes = response.bytes().await?;
|
if success {
|
||||||
file.write_all(&bytes).await?;
|
let check_total = total_size.unwrap_or(downloaded);
|
||||||
Ok(())
|
if downloaded >= check_total {
|
||||||
} else {
|
println!("Downloaded {}", output_path);
|
||||||
Err(anyhow::anyhow!("HTTP {}: {}", response.status(), url))
|
return Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(response) => {
|
||||||
|
if attempt == max_retries {
|
||||||
|
return Err(anyhow::anyhow!("HTTP {}: {}", response.status(), url));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
log::warn!("Request failed: {}", e);
|
||||||
|
if attempt == max_retries {
|
||||||
|
return Err(anyhow::anyhow!("Request failed: {} - {}", url, e));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if attempt < max_retries {
|
||||||
|
println!("Timeout/Error downloading {}, retrying attempt {}/{}...", url, attempt + 1, max_retries);
|
||||||
|
sleep(Duration::from_secs(5)).await;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
Err(anyhow::anyhow!("Failed to download {} after {} attempts", url, max_retries))
|
||||||
});
|
});
|
||||||
download_handle.await?
|
download_handle.await?
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue