Add limiter for forge downloading

This commit is contained in:
Jai A
2023-04-04 21:17:19 -07:00
parent f66fc06b4f
commit 9754f2d1c5
4 changed files with 59 additions and 17 deletions

View File

@@ -147,7 +147,22 @@ pub async fn retrieve_data(
})
}
futures::future::try_join_all(version_futures).await?;
{
let mut versions = version_futures.into_iter().peekable();
let mut chunk_index = 0;
while versions.peek().is_some() {
let now = Instant::now();
let chunk: Vec<_> = versions.by_ref().take(100).collect();
futures::future::try_join_all(chunk).await?;
chunk_index += 1;
let elapsed = now.elapsed();
info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
}
}
//futures::future::try_join_all(version_futures).await?;
upload_file_to_bucket(
format!(