Merge pull request #3 from modrinth/many-fixes

Many fixes

.env

@@ -1,7 +1,6 @@
 RUST_LOG=info,error
 
 BASE_URL=https://modrinth-cdn-staging.nyc3.digitaloceanspaces.com
-BASE_FOLDER=gamedata
 
 S3_ACCESS_TOKEN=none
 S3_SECRET=none

Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.65
+FROM rust:1.68.2
 
 COPY ./ ./
 

daedalus/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "daedalus"
-version = "0.1.17"
+version = "0.1.18"
 authors = ["Jai A <jaiagr+gpg@pm.me>"]
 edition = "2018"
 license = "MIT"

daedalus/src/lib.rs

@@ -46,22 +46,34 @@ pub fn get_path_from_artifact(artifact: &str) -> Result<String, Error> {
     let name_items = artifact.split(':').collect::<Vec<&str>>();
 
     let package = name_items.first().ok_or_else(|| {
-        Error::ParseError(format!("Unable to find package for library {}", &artifact))
+        Error::ParseError(format!(
+            "Unable to find package for library {}",
+            &artifact
+        ))
     })?;
     let name = name_items.get(1).ok_or_else(|| {
-        Error::ParseError(format!("Unable to find name for library {}", &artifact))
+        Error::ParseError(format!(
+            "Unable to find name for library {}",
+            &artifact
+        ))
     })?;
 
     if name_items.len() == 3 {
         let version_ext = name_items
             .get(2)
             .ok_or_else(|| {
-                Error::ParseError(format!("Unable to find version for library {}", &artifact))
+                Error::ParseError(format!(
+                    "Unable to find version for library {}",
+                    &artifact
+                ))
             })?
             .split('@')
             .collect::<Vec<&str>>();
         let version = version_ext.first().ok_or_else(|| {
-            Error::ParseError(format!("Unable to find version for library {}", &artifact))
+            Error::ParseError(format!(
+                "Unable to find version for library {}",
+                &artifact
+            ))
         })?;
         let ext = version_ext.get(1);
 
@@ -76,18 +88,27 @@ pub fn get_path_from_artifact(artifact: &str) -> Result<String, Error> {
         ))
     } else {
         let version = name_items.get(2).ok_or_else(|| {
-            Error::ParseError(format!("Unable to find version for library {}", &artifact))
+            Error::ParseError(format!(
+                "Unable to find version for library {}",
+                &artifact
+            ))
         })?;
 
         let data_ext = name_items
             .get(3)
             .ok_or_else(|| {
-                Error::ParseError(format!("Unable to find data for library {}", &artifact))
+                Error::ParseError(format!(
+                    "Unable to find data for library {}",
+                    &artifact
+                ))
             })?
             .split('@')
             .collect::<Vec<&str>>();
         let data = data_ext.first().ok_or_else(|| {
-            Error::ParseError(format!("Unable to find data for library {}", &artifact))
+            Error::ParseError(format!(
+                "Unable to find data for library {}",
+                &artifact
+            ))
         })?;
         let ext = data_ext.get(1);
 
@@ -126,10 +147,21 @@ pub async fn download_file_mirrors(
 }
 
 /// Downloads a file with retry and checksum functionality
-pub async fn download_file(url: &str, sha1: Option<&str>) -> Result<bytes::Bytes, Error> {
+pub async fn download_file(
+    url: &str,
+    sha1: Option<&str>,
+) -> Result<bytes::Bytes, Error> {
+    let mut headers = reqwest::header::HeaderMap::new();
+    if let Ok(header) = reqwest::header::HeaderValue::from_str(&format!(
+        "modrinth/daedalus/{} (support@modrinth.com)",
+        env!("CARGO_PKG_VERSION")
+    )) {
+        headers.insert(reqwest::header::USER_AGENT, header);
+    }
     let client = reqwest::Client::builder()
         .tcp_keepalive(Some(std::time::Duration::from_secs(10)))
         .timeout(std::time::Duration::from_secs(15))
+        .default_headers(headers)
         .build()
         .map_err(|err| Error::FetchError {
             inner: err,
@@ -183,7 +215,9 @@ pub async fn download_file(url: &str, sha1: Option<&str>) -> Result<bytes::Bytes
 
 /// Computes a checksum of the input bytes
 pub async fn get_hash(bytes: bytes::Bytes) -> Result<String, Error> {
-    let hash = tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest()).await?;
+    let hash =
+        tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest())
+            .await?;
 
     Ok(hash)
 }

daedalus/src/modded.rs

@@ -15,6 +15,9 @@ pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0;
 /// The latest version of the format the fabric model structs deserialize to
 pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0;
 
+/// The dummy replace string library names, inheritsFrom, and version names should be replaced with
+pub const DUMMY_REPLACE_STRING: &str = "${modrinth.gameVersion}";
+
 /// A data variable entry that depends on the side of the installation
 #[cfg_attr(feature = "bincode", derive(Encode, Decode))]
 #[derive(Serialize, Deserialize, Debug)]
@@ -93,6 +96,8 @@ pub fn merge_partial_version(
     partial: PartialVersionInfo,
     merge: VersionInfo,
 ) -> VersionInfo {
+    let merge_id = merge.id.clone();
+
     VersionInfo {
         arguments: if let Some(partial_args) = partial.arguments {
             if let Some(merge_args) = merge.arguments {
@@ -126,12 +131,22 @@ pub fn merge_partial_version(
         asset_index: merge.asset_index,
         assets: merge.assets,
         downloads: merge.downloads,
-        id: partial.id,
+        id: merge.id,
         java_version: merge.java_version,
         libraries: partial
             .libraries
             .into_iter()
             .chain(merge.libraries)
+            .map(|x| Library {
+                downloads: x.downloads,
+                extract: x.extract,
+                name: x.name.replace(DUMMY_REPLACE_STRING, &merge_id),
+                url: x.url,
+                natives: x.natives,
+                rules: x.rules,
+                checksums: x.checksums,
+                include_in_classpath: x.include_in_classpath,
+            })
             .collect::<Vec<_>>(),
         main_class: if let Some(main_class) = partial.main_class {
             main_class
@@ -163,6 +178,8 @@ pub struct Manifest {
 pub struct Version {
     /// The minecraft version ID
     pub id: String,
+    /// Whether the release is stable or not
+    pub stable: bool,
     /// A map that contains loader versions for the game version
     pub loaders: Vec<LoaderVersion>,
 }

daedalus_client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "daedalus_client"
-version = "0.1.17"
+version = "0.1.18"
 authors = ["Jai A <jaiagr+gpg@pm.me>"]
 edition = "2018"
 

daedalus_client/src/fabric.rs

@@ -1,266 +1,283 @@
-use crate::{format_url, upload_file_to_bucket, Error};
+use crate::{download_file, format_url, upload_file_to_bucket, Error};
-use daedalus::download_file;
 use daedalus::minecraft::{Library, VersionManifest};
-use daedalus::modded::{LoaderVersion, Manifest, PartialVersionInfo, Version};
+use daedalus::modded::{
-use log::info;
+    LoaderVersion, Manifest, PartialVersionInfo, Version, DUMMY_REPLACE_STRING,
+};
 use serde::{Deserialize, Serialize};
 use std::sync::Arc;
-use std::time::{Duration, Instant};
+use tokio::sync::{Mutex, RwLock, Semaphore};
-use tokio::sync::{Mutex, RwLock};
 
 pub async fn retrieve_data(
     minecraft_versions: &VersionManifest,
     uploaded_files: &mut Vec<String>,
+    semaphore: Arc<Semaphore>,
 ) -> Result<(), Error> {
-    let mut list = fetch_fabric_versions(None).await?;
+    let mut list = fetch_fabric_versions(None, semaphore.clone()).await?;
-    let old_manifest = daedalus::modded::fetch_manifest(&format!(
+    let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!(
         "fabric/v{}/manifest.json",
         daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION,
-    ))
+    )))
     .await
     .ok();
 
-    let versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest {
+    let mut versions = if let Some(old_manifest) = old_manifest {
         old_manifest.game_versions
     } else {
         Vec::new()
-    }));
+    };
 
+    let loaders_mutex = RwLock::new(Vec::new());
+
+    {
+        let mut loaders = loaders_mutex.write().await;
+
+        for loader in &list.loader {
+            loaders.push((Box::new(loader.stable), loader.version.clone()))
+        }
+
+        list.loader
+            .retain(|x| loaders.iter().any(|val| val.1 == x.version))
+    }
+
+    const DUMMY_GAME_VERSION: &str = "1.19.4-rc2";
+
+    let loader_version_mutex = Mutex::new(Vec::new());
     let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new()));
 
-    if let Some(latest) = list.loader.get(0) {
+    let loader_versions = futures::future::try_join_all(
-        let loaders_mutex = Arc::new(RwLock::new(Vec::new()));
+        loaders_mutex.read().await.clone().into_iter().map(
-        let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new()));
+            |(stable, loader)| async {
-
+                {
-        {
+                    if versions.iter().any(|x| {
-            let mut loaders = loaders_mutex.write().await;
+                        x.id == DUMMY_REPLACE_STRING
-
+                            && x.loaders.iter().any(|x| x.id == loader)
-            // for loader in &list.loader {
+                    }) {
-            //     loaders.push((Box::new(loader.stable), loader.version.clone()))
+                        return Ok(None);
-            // }
-
-            loaders.push((Box::new(latest.stable), latest.version.clone()));
-
-            if !latest.stable {
-                if let Some(stable) = list.loader.iter().find(|x| x.stable) {
-                    loaders.push((Box::new(stable.stable), stable.version.clone()));
-                }
-            }
-
-            list.loader.retain(|x| loaders.iter().any(|val| val.1 == x.version))
-        }
-
-        let mut version_futures = Vec::new();
-
-        for game_version in list.game.iter_mut() {
-            let visited_artifacts_mutex = Arc::clone(&visited_artifacts_mutex);
-            let loaders_mutex = Arc::clone(&loaders_mutex);
-            let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex);
-
-            let versions_mutex = Arc::clone(&versions);
-            version_futures.push(async move {
-                let loader_version_mutex = Mutex::new(Vec::new());
-
-                let versions =
-                    futures::future::try_join_all(
-                        loaders_mutex.read().await.clone().into_iter().map(
-                            |(stable, loader)| async {
-                                {
-                                    if versions_mutex.lock().await.iter().any(|x| {
-                                        x.id == game_version.version
-                                            && x.loaders.iter().any(|x| x.id == loader)
-                                    }) {
-                                        return Ok(None);
-                                    }
-                                }
-
-                                let version =
-                                    fetch_fabric_version(&game_version.version, &loader).await?;
-
-                                Ok::<Option<(Box<bool>, String, PartialVersionInfo)>, Error>(Some(
-                                    (stable, loader, version),
-                                ))
-                            },
-                        ),
-                    )
-                    .await?
-                    .into_iter()
-                    .flatten();
-
-                futures::future::try_join_all(versions.map(|(stable, loader, version)| async {
-                    let libs = futures::future::try_join_all(version.libraries.into_iter().map(
-                        |mut lib| async {
-                            {
-                                let mut visited_assets = visited_artifacts_mutex.lock().await;
-
-                                if visited_assets.contains(&lib.name) {
-                                    lib.url = Some(format_url("maven/"));
-
-                                    return Ok(lib);
-                                } else {
-                                    visited_assets.push(lib.name.clone())
-                                }
-                            }
-
-                            let artifact_path = daedalus::get_path_from_artifact(&lib.name)?;
-
-                            let artifact = daedalus::download_file(
-                                &format!(
-                                    "{}{}",
-                                    lib.url.unwrap_or_else(|| {
-                                        "https://maven.fabricmc.net/".to_string()
-                                    }),
-                                    artifact_path
-                                ),
-                                None,
-                            )
-                            .await?;
-
-                            lib.url = Some(format_url("maven/"));
-
-                            upload_file_to_bucket(
-                                format!("{}/{}", "maven", artifact_path),
-                                artifact.to_vec(),
-                                Some("application/java-archive".to_string()),
-                                uploaded_files_mutex.as_ref(),
-                            )
-                            .await?;
-
-                            Ok::<Library, Error>(lib)
-                        },
-                    ))
-                    .await?;
-
-                    let version_path = format!(
-                        "fabric/v{}/versions/{}-{}.json",
-                        daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION,
-                        version.inherits_from,
-                        &loader
-                    );
-
-                    let inherits_from = version.inherits_from.clone();
-
-                    upload_file_to_bucket(
-                        version_path.clone(),
-                        serde_json::to_vec(&PartialVersionInfo {
-                            arguments: version.arguments,
-                            id: version.id,
-                            main_class: version.main_class,
-                            release_time: version.release_time,
-                            time: version.time,
-                            type_: version.type_,
-                            inherits_from: version.inherits_from,
-                            libraries: libs,
-                            minecraft_arguments: version.minecraft_arguments,
-                            processors: None,
-                            data: None,
-                        })?,
-                        Some("application/json".to_string()),
-                        uploaded_files_mutex.as_ref(),
-                    )
-                    .await?;
-
-                    {
-                        let mut loader_version_map = loader_version_mutex.lock().await;
-                        async move {
-                            loader_version_map.push(LoaderVersion {
-                                id: format!("{}-{}", inherits_from, loader),
-                                url: format_url(&version_path),
-                                stable: *stable,
-                            });
-                        }
-                        .await;
                     }
+                }
 
-                    Ok::<(), Error>(())
+                let version = fetch_fabric_version(
-                }))
+                    DUMMY_GAME_VERSION,
+                    &loader,
+                    semaphore.clone(),
+                )
                 .await?;
 
-                let mut versions = versions_mutex.lock().await;
+                Ok::<Option<(Box<bool>, String, PartialVersionInfo)>, Error>(
-                versions.push(Version {
+                    Some((stable, loader, version)),
-                    id: game_version.version.clone(),
+                )
-                    loaders: loader_version_mutex.into_inner(),
+            },
-                });
+        ),
+    )
+    .await?;
 
-                Ok::<(), Error>(())
+    let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new()));
-            });
+    futures::future::try_join_all(loader_versions.into_iter()
-        }
+        .flatten().map(
+        |(stable, loader, version)| async {
+            let libs = futures::future::try_join_all(
+                version.libraries.into_iter().map(|mut lib| async {
+                    {
+                        let mut visited_assets =
+                            visited_artifacts_mutex.lock().await;
 
-        let mut versions = version_futures.into_iter().peekable();
+                        if visited_assets.contains(&lib.name) {
-        let mut chunk_index = 0;
+                            lib.url = Some(format_url("maven/"));
-        while versions.peek().is_some() {
-            let now = Instant::now();
 
-            let chunk: Vec<_> = versions.by_ref().take(10).collect();
+                            return Ok(lib);
-            futures::future::try_join_all(chunk).await?;
+                        } else {
+                            visited_assets.push(lib.name.clone())
+                        }
+                    }
 
-            tokio::time::sleep(Duration::from_secs(1)).await;
+                    if lib.name.contains(DUMMY_GAME_VERSION) {
+                        lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING);
+                        lib.url = Some(format_url("maven/"));
+                        futures::future::try_join_all(list.game.clone().into_iter().map(|game_version| async {
+                            let semaphore = semaphore.clone();
+                            let uploaded_files_mutex = uploaded_files_mutex.clone();
+                            let lib_name = lib.name.clone();
+                            let lib_url = lib.url.clone();
 
-            chunk_index += 1;
+                            async move {
+                                let artifact_path =
+                                    daedalus::get_path_from_artifact(&lib_name.replace(DUMMY_REPLACE_STRING, &game_version.version))?;
 
-            let elapsed = now.elapsed();
+                                let artifact = download_file(
-            info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
+                                    &format!(
-        }
+                                        "{}{}",
+                                        lib_url.unwrap_or_else(|| {
+                                            "https://maven.fabricmc.net/".to_string()
+                                        }),
+                                        artifact_path
+                                    ),
+                                    None,
+                                    semaphore.clone(),
+                                )
+                                    .await?;
+
+                                upload_file_to_bucket(
+                                    format!("{}/{}", "maven", artifact_path),
+                                    artifact.to_vec(),
+                                    Some("application/java-archive".to_string()),
+                                    &uploaded_files_mutex,
+                                    semaphore.clone(),
+                                )
+                                    .await?;
+
+                                Ok::<(), Error>(())
+                            }.await?;
+
+                            Ok::<(), Error>(())
+                        })).await?;
+
+                        return Ok(lib);
+                    }
+
+                    let artifact_path =
+                        daedalus::get_path_from_artifact(&lib.name)?;
+
+                    let artifact = download_file(
+                        &format!(
+                            "{}{}",
+                            lib.url.unwrap_or_else(|| {
+                                "https://maven.fabricmc.net/".to_string()
+                            }),
+                            artifact_path
+                        ),
+                        None,
+                        semaphore.clone(),
+                    )
+                    .await?;
+
+                    lib.url = Some(format_url("maven/"));
+
+                    upload_file_to_bucket(
+                        format!("{}/{}", "maven", artifact_path),
+                        artifact.to_vec(),
+                        Some("application/java-archive".to_string()),
+                        &uploaded_files_mutex,
+                        semaphore.clone(),
+                    )
+                    .await?;
+
+                    Ok::<Library, Error>(lib)
+                }),
+            )
+            .await?;
+
+            let version_path = format!(
+                "fabric/v{}/versions/{}.json",
+                daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION,
+                &loader
+            );
+
+            upload_file_to_bucket(
+                version_path.clone(),
+                serde_json::to_vec(&PartialVersionInfo {
+                    arguments: version.arguments,
+                    id: version
+                        .id
+                        .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING),
+                    main_class: version.main_class,
+                    release_time: version.release_time,
+                    time: version.time,
+                    type_: version.type_,
+                    inherits_from: version
+                        .inherits_from
+                        .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING),
+                    libraries: libs,
+                    minecraft_arguments: version.minecraft_arguments,
+                    processors: None,
+                    data: None,
+                })?,
+                Some("application/json".to_string()),
+                &uploaded_files_mutex,
+                semaphore.clone(),
+            )
+            .await?;
+
+            {
+                let mut loader_version_map = loader_version_mutex.lock().await;
+                async move {
+                    loader_version_map.push(LoaderVersion {
+                        id: loader.to_string(),
+                        url: format_url(&version_path),
+                        stable: *stable,
+                    });
+                }
+                .await;
+            }
+
+            Ok::<(), Error>(())
+        },
+    ))
+    .await?;
+
+    versions.push(Version {
+        id: DUMMY_REPLACE_STRING.to_string(),
+        stable: true,
+        loaders: loader_version_mutex.into_inner(),
+    });
+
+    for version in &list.game {
+        versions.push(Version {
+            id: version.version.clone(),
+            stable: version.stable,
+            loaders: vec![],
+        });
     }
 
-    if let Ok(versions) = Arc::try_unwrap(versions) {
+    versions.sort_by(|x, y| {
-        let mut versions = versions.into_inner();
+        minecraft_versions
+            .versions
+            .iter()
+            .position(|z| x.id == z.id)
+            .unwrap_or_default()
+            .cmp(
+                &minecraft_versions
+                    .versions
+                    .iter()
+                    .position(|z| y.id == z.id)
+                    .unwrap_or_default(),
+            )
+    });
 
-        versions.sort_by(|x, y| {
+    for version in &mut versions {
-            minecraft_versions
+        version.loaders.sort_by(|x, y| {
-                .versions
+            list.loader
                 .iter()
-                .position(|z| x.id == z.id)
+                .position(|z| {
+                    x.id.split('-').next().unwrap_or_default() == &*z.version
+                })
                 .unwrap_or_default()
                 .cmp(
-                    &minecraft_versions
+                    &list
-                        .versions
+                        .loader
                         .iter()
-                        .position(|z| y.id == z.id)
+                        .position(|z| {
+                            y.id.split('-').next().unwrap_or_default()
+                                == z.version
+                        })
                         .unwrap_or_default(),
                 )
-        });
+        })
-
-        for version in &mut versions {
-            version.loaders.sort_by(|x, y| {
-                list.loader
-                    .iter()
-                    .position(|z| {
-                        x.id.split('-')
-                            .next()
-                            .unwrap_or_default()
-                            == &*z.version
-                    })
-                    .unwrap_or_default()
-                    .cmp(
-                        &list
-                            .loader
-                            .iter()
-                            .position(|z| {
-                                y.id.split('-')
-                                    .next()
-                                    .unwrap_or_default()
-                                    == z.version
-                            })
-                            .unwrap_or_default(),
-                    )
-            })
-        }
-
-        upload_file_to_bucket(
-            format!(
-                "fabric/v{}/manifest.json",
-                daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION,
-            ),
-            serde_json::to_vec(&Manifest {
-                game_versions: versions,
-            })?,
-            Some("application/json".to_string()),
-            uploaded_files_mutex.as_ref(),
-        )
-        .await?;
     }
 
+    upload_file_to_bucket(
+        format!(
+            "fabric/v{}/manifest.json",
+            daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION,
+        ),
+        serde_json::to_vec(&Manifest {
+            game_versions: versions,
+        })?,
+        Some("application/json".to_string()),
+        &uploaded_files_mutex,
+        semaphore,
+    )
+    .await?;
+
     if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) {
         uploaded_files.extend(uploaded_files_mutex.into_inner());
     }
@@ -273,6 +290,7 @@ const FABRIC_META_URL: &str = "https://meta.fabricmc.net/v2";
 async fn fetch_fabric_version(
     version_number: &str,
     loader_version: &str,
+    semaphore: Arc<Semaphore>,
 ) -> Result<PartialVersionInfo, Error> {
     Ok(serde_json::from_slice(
         &download_file(
@@ -281,6 +299,7 @@ async fn fetch_fabric_version(
                 FABRIC_META_URL, version_number, loader_version
             ),
             None,
+            semaphore,
         )
         .await?,
     )?)
@@ -319,11 +338,15 @@ struct FabricLoaderVersion {
     pub stable: bool,
 }
 /// Fetches the list of fabric versions
-async fn fetch_fabric_versions(url: Option<&str>) -> Result<FabricVersions, Error> {
+async fn fetch_fabric_versions(
+    url: Option<&str>,
+    semaphore: Arc<Semaphore>,
+) -> Result<FabricVersions, Error> {
     Ok(serde_json::from_slice(
         &download_file(
             url.unwrap_or(&*format!("{}/versions", FABRIC_META_URL)),
             None,
+            semaphore,
         )
         .await?,
     )?)

daedalus_client/src/forge.rs

@@ -1,8 +1,14 @@
-use crate::{format_url, upload_file_to_bucket, Error};
+use crate::{
+    download_file, download_file_mirrors, format_url, upload_file_to_bucket,
+    Error,
+};
 use chrono::{DateTime, Utc};
-use daedalus::download_file;
+use daedalus::minecraft::{
-use daedalus::minecraft::{Argument, ArgumentType, Library, VersionManifest, VersionType};
+    Argument, ArgumentType, Library, VersionManifest, VersionType,
-use daedalus::modded::{LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry};
+};
+use daedalus::modded::{
+    LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry,
+};
 use lazy_static::lazy_static;
 use log::info;
 use semver::{Version, VersionReq};
@@ -11,7 +17,7 @@ use std::collections::HashMap;
 use std::io::Read;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
-use tokio::sync::Mutex;
+use tokio::sync::{Mutex, Semaphore};
 
 lazy_static! {
     static ref FORGE_MANIFEST_V1_QUERY: VersionReq =
@@ -20,14 +26,16 @@ lazy_static! {
         VersionReq::parse(">=23.5.2851, <31.2.52").unwrap();
     static ref FORGE_MANIFEST_V2_QUERY_P2: VersionReq =
         VersionReq::parse(">=32.0.1, <37.0.0").unwrap();
-    static ref FORGE_MANIFEST_V3_QUERY: VersionReq = VersionReq::parse(">=37.0.0").unwrap();
+    static ref FORGE_MANIFEST_V3_QUERY: VersionReq =
+        VersionReq::parse(">=37.0.0").unwrap();
 }
 
 pub async fn retrieve_data(
     minecraft_versions: &VersionManifest,
     uploaded_files: &mut Vec<String>,
+    semaphore: Arc<Semaphore>,
 ) -> Result<(), Error> {
-    let maven_metadata = fetch_maven_metadata(None).await?;
+    let maven_metadata = fetch_maven_metadata(None, semaphore.clone()).await?;
     let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!(
         "forge/v{}/manifest.json",
         daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
@@ -35,11 +43,12 @@ pub async fn retrieve_data(
     .await
     .ok();
 
-    let old_versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest {
+    let old_versions =
-        old_manifest.game_versions
+        Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest {
-    } else {
+            old_manifest.game_versions
-        Vec::new()
+        } else {
-    }));
+            Vec::new()
+        }));
 
     let versions = Arc::new(Mutex::new(Vec::new()));
 
@@ -52,13 +61,14 @@ pub async fn retrieve_data(
         let mut loaders = Vec::new();
 
         for loader_version_full in loader_versions {
-            let loader_version = loader_version_full.split('-').into_iter().nth(1);
+            let loader_version = loader_version_full.split('-').nth(1);
 
             if let Some(loader_version_raw) = loader_version {
                 // This is a dirty hack to get around Forge not complying with SemVer, but whatever
                 // Most of this is a hack anyways :(
                 // Works for all forge versions!
-                let split = loader_version_raw.split('.').collect::<Vec<&str>>();
+                let split =
+                    loader_version_raw.split('.').collect::<Vec<&str>>();
                 let loader_version = if split.len() >= 4 {
                     if split[0].parse::<i32>().unwrap_or(0) < 6 {
                         format!("{}.{}.{}", split[0], split[1], split[3])
@@ -80,203 +90,209 @@ pub async fn retrieve_data(
                 }
             }
         }
-        version_futures.push(async {
-            let mut loaders_versions = Vec::new();
 
-            {
+        if !loaders.is_empty() {
-                let mut loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async {
+            version_futures.push(async {
-                    let versions_mutex = Arc::clone(&old_versions);
+                let loaders_versions = Vec::new();
-                    let visited_assets = Arc::clone(&visited_assets_mutex);
-                    let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex);
-                    let minecraft_version = minecraft_version.clone();
 
-                    async move {
+                {
-                        /// These forge versions are not worth supporting!
+                    let loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async {
-                        const WHITELIST : [&str; 1] = [
+                        let versions_mutex = Arc::clone(&old_versions);
-                            // Not supported due to `data` field being `[]` even though the type is a map
+                        let visited_assets = Arc::clone(&visited_assets_mutex);
-                            "1.12.2-14.23.5.2851"
+                        let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex);
-                        ];
+                        let semaphore = Arc::clone(&semaphore);
+                        let minecraft_version = minecraft_version.clone();
 
-                        if WHITELIST.contains(&&*loader_version_full) {
+                        async move {
-                            return Ok(None);
+                            /// These forge versions are not worth supporting!
-                        }
+                            const WHITELIST : [&str; 1] = [
+                                // Not supported due to `data` field being `[]` even though the type is a map
+                                "1.12.2-14.23.5.2851",
+                            ];
 
-                        {
+                            if WHITELIST.contains(&&*loader_version_full) {
-                            let versions = versions_mutex.lock().await;
+                                return Ok(None);
-                            let version = versions.iter().find(|x|
-                                x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full));
-
-                            if let Some(version) = version {
-                                return Ok::<Option<LoaderVersion>, Error>(Some(version.clone()));
                             }
-                        }
 
-                        info!("Forge - Installer Start {}", loader_version_full.clone());
+                            {
-                        let bytes = download_file(&format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?;
+                                let versions = versions_mutex.lock().await;
+                                let version = versions.iter().find(|x|
+                                    x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full));
 
-                        let reader = std::io::Cursor::new(bytes);
+                                if let Some(version) = version {
+                                    return Ok::<Option<LoaderVersion>, Error>(Some(version.clone()));
+                                }
+                            }
 
-                        if let Ok(archive) = zip::ZipArchive::new(reader) {
+                            info!("Forge - Installer Start {}", loader_version_full.clone());
-                            if FORGE_MANIFEST_V1_QUERY.matches(&version) {
+                            let bytes = download_file(&format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None, semaphore.clone()).await?;
-                                let mut archive_clone = archive.clone();
-                                let profile = tokio::task::spawn_blocking(move || {
-                                    let mut install_profile = archive_clone.by_name("install_profile.json")?;
 
-                                    let mut contents = String::new();
+                            let reader = std::io::Cursor::new(bytes);
-                                    install_profile.read_to_string(&mut contents)?;
 
-                                    Ok::<ForgeInstallerProfileV1, Error>(serde_json::from_str::<ForgeInstallerProfileV1>(&contents)?)
+                            if let Ok(archive) = zip::ZipArchive::new(reader) {
-                                }).await??;
+                                if FORGE_MANIFEST_V1_QUERY.matches(&version) {
+                                    let mut archive_clone = archive.clone();
+                                    let profile = tokio::task::spawn_blocking(move || {
+                                        let mut install_profile = archive_clone.by_name("install_profile.json")?;
 
-                                let mut archive_clone = archive.clone();
+                                        let mut contents = String::new();
-                                let file_path = profile.install.file_path.clone();
+                                        install_profile.read_to_string(&mut contents)?;
-                                let forge_universal_bytes = tokio::task::spawn_blocking(move || {
+
-                                    let mut forge_universal_file = archive_clone.by_name(&file_path)?;
+                                        Ok::<ForgeInstallerProfileV1, Error>(serde_json::from_str::<ForgeInstallerProfileV1>(&contents)?)
-                                    let mut forge_universal =  Vec::new();
+                                    }).await??;
-                                    forge_universal_file.read_to_end(&mut forge_universal)?;
+
+                                    let mut archive_clone = archive.clone();
+                                    let file_path = profile.install.file_path.clone();
+                                    let forge_universal_bytes = tokio::task::spawn_blocking(move || {
+                                        let mut forge_universal_file = archive_clone.by_name(&file_path)?;
+                                        let mut forge_universal =  Vec::new();
+                                        forge_universal_file.read_to_end(&mut forge_universal)?;
 
 
-                                    Ok::<bytes::Bytes, Error>(bytes::Bytes::from(forge_universal))
+                                        Ok::<bytes::Bytes, Error>(bytes::Bytes::from(forge_universal))
-                                }).await??;
+                                    }).await??;
-                                let forge_universal_path = profile.install.path.clone();
+                                    let forge_universal_path = profile.install.path.clone();
 
-                                let now = Instant::now();
+                                    let now = Instant::now();
-                                let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async {
+                                    let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async {
-                                    if let Some(url) = lib.url {
+                                        if let Some(url) = lib.url {
-                                        {
+                                            {
-                                            let mut visited_assets = visited_assets.lock().await;
+                                                let mut visited_assets = visited_assets.lock().await;
 
-                                            if visited_assets.contains(&lib.name) {
+                                                if visited_assets.contains(&lib.name) {
-                                                lib.url = Some(format_url("maven/"));
+                                                    lib.url = Some(format_url("maven/"));
 
-                                                return Ok::<Library, Error>(lib);
+                                                    return Ok::<Library, Error>(lib);
-                                            } else {
+                                                } else {
-                                                visited_assets.push(lib.name.clone())
+                                                    visited_assets.push(lib.name.clone())
+                                                }
                                             }
+
+                                            let artifact_path =
+                                                daedalus::get_path_from_artifact(&lib.name)?;
+
+                                            let artifact = if lib.name == forge_universal_path {
+                                                forge_universal_bytes.clone()
+                                            } else {
+                                                let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"];
+
+                                                download_file_mirrors(
+                                                    &artifact_path,
+                                                    &mirrors,
+                                                    None,
+                                                    semaphore.clone(),
+                                                )
+                                                    .await?
+                                            };
+
+                                            lib.url = Some(format_url("maven/"));
+
+                                            upload_file_to_bucket(
+                                                format!("{}/{}", "maven", artifact_path),
+                                                artifact.to_vec(),
+                                                Some("application/java-archive".to_string()),
+                                                uploaded_files_mutex.as_ref(),
+                                                semaphore.clone(),
+                                            ).await?;
                                         }
 
-                                        let artifact_path =
+                                        Ok::<Library, Error>(lib)
-                                            daedalus::get_path_from_artifact(&lib.name)?;
+                                    })).await?;
 
-                                        let artifact = if lib.name == forge_universal_path {
+                                    let elapsed = now.elapsed();
-                                            forge_universal_bytes.clone()
+                                    info!("Elapsed lib DL: {:.2?}", elapsed);
-                                        } else {
-                                            let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"];
 
-                                            daedalus::download_file_mirrors(
+                                    let new_profile = PartialVersionInfo {
-                                                &artifact_path,
+                                        id: profile.version_info.id,
-                                                &mirrors,
+                                        inherits_from: profile.install.minecraft,
-                                                None,
+                                        release_time: profile.version_info.release_time,
-                                            )
+                                        time: profile.version_info.time,
-                                                .await?
+                                        main_class: profile.version_info.main_class,
-                                        };
+                                        minecraft_arguments: profile.version_info.minecraft_arguments.clone(),
+                                        arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()),
+                                        libraries: libs,
+                                        type_: profile.version_info.type_,
+                                        data: None,
+                                        processors: None
+                                    };
 
-                                        lib.url = Some(format_url("maven/"));
+                                    let version_path = format!(
+                                        "forge/v{}/versions/{}.json",
+                                        daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
+                                        new_profile.id
+                                    );
 
-                                        upload_file_to_bucket(
+                                    upload_file_to_bucket(
-                                            format!("{}/{}", "maven", artifact_path),
+                                        version_path.clone(),
-                                            artifact.to_vec(),
+                                        serde_json::to_vec(&new_profile)?,
-                                            Some("application/java-archive".to_string()),
+                                        Some("application/json".to_string()),
-                                            uploaded_files_mutex.as_ref(),
+                                        uploaded_files_mutex.as_ref(),
-                                        ).await?;
+                                        semaphore.clone(),
+                                    ).await?;
+
+                                    return Ok(Some(LoaderVersion {
+                                        id: loader_version_full,
+                                        url: format_url(&version_path),
+                                        stable: false
+                                    }));
+                                } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) {
+                                    let mut archive_clone = archive.clone();
+                                    let mut profile = tokio::task::spawn_blocking(move || {
+                                        let mut install_profile = archive_clone.by_name("install_profile.json")?;
+
+                                        let mut contents = String::new();
+                                        install_profile.read_to_string(&mut contents)?;
+
+                                        Ok::<ForgeInstallerProfileV2, Error>(serde_json::from_str::<ForgeInstallerProfileV2>(&contents)?)
+                                    }).await??;
+
+                                    let mut archive_clone = archive.clone();
+                                    let version_info = tokio::task::spawn_blocking(move || {
+                                        let mut install_profile = archive_clone.by_name("version.json")?;
+
+                                        let mut contents = String::new();
+                                        install_profile.read_to_string(&mut contents)?;
+
+                                        Ok::<PartialVersionInfo, Error>(serde_json::from_str::<PartialVersionInfo>(&contents)?)
+                                    }).await??;
+
+
+                                    let mut libs : Vec<Library> = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library {
+                                        downloads: x.downloads,
+                                        extract: x.extract,
+                                        name: x.name,
+                                        url: x.url,
+                                        natives: x.natives,
+                                        rules: x.rules,
+                                        checksums: x.checksums,
+                                        include_in_classpath: false
+                                    })).collect();
+
+                                    let mut local_libs : HashMap<String, bytes::Bytes> = HashMap::new();
+
+                                    for lib in &libs {
+                                        if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) {
+                                            let mut archive_clone = archive.clone();
+                                            let lib_name_clone = lib.name.clone();
+
+                                            let lib_bytes = tokio::task::spawn_blocking(move || {
+                                                let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?;
+                                                let mut lib_bytes =  Vec::new();
+                                                lib_file.read_to_end(&mut lib_bytes)?;
+
+                                                Ok::<bytes::Bytes, Error>(bytes::Bytes::from(lib_bytes))
+                                            }).await??;
+
+                                            local_libs.insert(lib.name.clone(), lib_bytes);
+                                        }
                                     }
 
-                                    Ok::<Library, Error>(lib)
+                                    let path = profile.path.clone();
-                                })).await?;
+                                    let version = profile.version.clone();
 
-                                let elapsed = now.elapsed();
+                                    for entry in profile.data.values_mut() {
-                                info!("Elapsed lib DL: {:.2?}", elapsed);
+                                        if entry.client.starts_with('/') || entry.server.starts_with('/') {
-
+                                            macro_rules! read_data {
-                                let new_profile = PartialVersionInfo {
-                                    id: profile.version_info.id,
-                                    inherits_from: profile.install.minecraft,
-                                    release_time: profile.version_info.release_time,
-                                    time: profile.version_info.time,
-                                    main_class: profile.version_info.main_class,
-                                    minecraft_arguments: profile.version_info.minecraft_arguments.clone(),
-                                    arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()),
-                                    libraries: libs,
-                                    type_: profile.version_info.type_,
-                                    data: None,
-                                    processors: None
-                                };
-
-                                let version_path = format!(
-                                    "forge/v{}/versions/{}.json",
-                                    daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
-                                    new_profile.id
-                                );
-
-                                upload_file_to_bucket(
-                                    version_path.clone(),
-                                    serde_json::to_vec(&new_profile)?,
-                                    Some("application/json".to_string()),
-                                    uploaded_files_mutex.as_ref()
-                                ).await?;
-
-                                return Ok(Some(LoaderVersion {
-                                    id: loader_version_full,
-                                    url: format_url(&version_path),
-                                    stable: false
-                                }));
-                            } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) {
-                                let mut archive_clone = archive.clone();
-                                let mut profile = tokio::task::spawn_blocking(move || {
-                                    let mut install_profile = archive_clone.by_name("install_profile.json")?;
-
-                                    let mut contents = String::new();
-                                    install_profile.read_to_string(&mut contents)?;
-
-                                    Ok::<ForgeInstallerProfileV2, Error>(serde_json::from_str::<ForgeInstallerProfileV2>(&contents)?)
-                                }).await??;
-
-                                let mut archive_clone = archive.clone();
-                                let version_info = tokio::task::spawn_blocking(move || {
-                                    let mut install_profile = archive_clone.by_name("version.json")?;
-
-                                    let mut contents = String::new();
-                                    install_profile.read_to_string(&mut contents)?;
-
-                                    Ok::<PartialVersionInfo, Error>(serde_json::from_str::<PartialVersionInfo>(&contents)?)
-                                }).await??;
-
-
-                                let mut libs : Vec<daedalus::minecraft::Library> = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library {
-                                    downloads: x.downloads,
-                                    extract: x.extract,
-                                    name: x.name,
-                                    url: x.url,
-                                    natives: x.natives,
-                                    rules: x.rules,
-                                    checksums: x.checksums,
-                                    include_in_classpath: false
-                                })).collect();
-
-                                let mut local_libs : HashMap<String, bytes::Bytes> = HashMap::new();
-
-                                for lib in &libs {
-                                    if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) {
-                                        let mut archive_clone = archive.clone();
-                                        let lib_name_clone = lib.name.clone();
-
-                                        let lib_bytes = tokio::task::spawn_blocking(move || {
-                                            let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?;
-                                            let mut lib_bytes =  Vec::new();
-                                            lib_file.read_to_end(&mut lib_bytes)?;
-
-                                            Ok::<bytes::Bytes, Error>(bytes::Bytes::from(lib_bytes))
-                                        }).await??;
-
-                                        local_libs.insert(lib.name.clone(), lib_bytes);
-                                    }
-                                }
-
-                                let path = profile.path.clone();
-                                let version = profile.version.clone();
-
-                                for entry in profile.data.values_mut() {
-                                    if entry.client.starts_with('/') || entry.server.starts_with('/') {
-                                        macro_rules! read_data {
                                         ($value:expr) => {
                                             let mut archive_clone = archive.clone();
                                             let value_clone = $value.clone();
@@ -315,178 +331,178 @@ pub async fn retrieve_data(
                                         }
                                     }
 
-                                        if entry.client.starts_with('/') {
+                                            if entry.client.starts_with('/') {
-                                            read_data!(entry.client);
+                                                read_data!(entry.client);
-                                        }
-
-                                        // Do we really need to support server installs? Keeping this here
-                                        // just in case
-                                        //
-                                        // if entry.server.starts_with('/') {
-                                        //     read_data!(entry.server);
-                                        // }
-                                    }
-                                }
-
-                                let now = Instant::now();
-                                let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async {
-                                    let artifact_path =
-                                        daedalus::get_path_from_artifact(&lib.name)?;
-
-                                    {
-                                        let mut visited_assets = visited_assets.lock().await;
-
-                                        if visited_assets.contains(&lib.name) {
-                                            if let Some(ref mut downloads) = lib.downloads {
-                                                if let Some(ref mut artifact) = downloads.artifact {
-                                                    artifact.url = format_url(&format!("maven/{}", artifact_path));
-                                                }
-                                            } else if lib.url.is_some() {
-                                                lib.url = Some(format_url("maven/"));
                                             }
 
-                                            return Ok::<Library, Error>(lib);
+                                            if entry.server.starts_with('/') {
-                                        } else {
+                                                read_data!(entry.server);
-                                            visited_assets.push(lib.name.clone())
+                                            }
                                         }
                                     }
 
-                                    let artifact_bytes = if let Some(ref mut downloads) = lib.downloads {
+                                    let now = Instant::now();
-                                        if let Some(ref mut artifact) = downloads.artifact {
+                                    let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async {
-                                            let res = if artifact.url.is_empty() {
+                                        let artifact_path =
+                                            daedalus::get_path_from_artifact(&lib.name)?;
+
+                                        {
+                                            let mut visited_assets = visited_assets.lock().await;
+
+                                            if visited_assets.contains(&lib.name) {
+                                                if let Some(ref mut downloads) = lib.downloads {
+                                                    if let Some(ref mut artifact) = downloads.artifact {
+                                                        artifact.url = format_url(&format!("maven/{}", artifact_path));
+                                                    }
+                                                } else if lib.url.is_some() {
+                                                    lib.url = Some(format_url("maven/"));
+                                                }
+
+                                                return Ok::<Library, Error>(lib);
+                                            } else {
+                                                visited_assets.push(lib.name.clone())
+                                            }
+                                        }
+
+                                        let artifact_bytes = if let Some(ref mut downloads) = lib.downloads {
+                                            if let Some(ref mut artifact) = downloads.artifact {
+                                                let res = if artifact.url.is_empty() {
+                                                    local_libs.get(&lib.name).cloned()
+                                                } else {
+                                                    Some(download_file(
+                                                        &artifact.url,
+                                                        Some(&*artifact.sha1),
+                                                        semaphore.clone(),
+                                                    )
+                                                        .await?)
+                                                };
+
+                                                if res.is_some() {
+                                                    artifact.url = format_url(&format!("maven/{}", artifact_path));
+                                                }
+
+                                                res
+                                            } else { None }
+                                        } else if let Some(ref mut url) = lib.url {
+                                            let res = if url.is_empty() {
                                                 local_libs.get(&lib.name).cloned()
                                             } else {
-                                                Some(daedalus::download_file(
+                                                Some(download_file(
-                                                    &artifact.url,
+                                                    url,
-                                                    Some(&*artifact.sha1),
+                                                    None,
+                                                    semaphore.clone(),
                                                 )
                                                     .await?)
                                             };
 
                                             if res.is_some() {
-                                                artifact.url = format_url(&format!("maven/{}", artifact_path));
+                                                lib.url = Some(format_url("maven/"));
                                             }
 
                                             res
-                                        } else { None }
+                                        } else { None };
-                                    } else if let Some(ref mut url) = lib.url {
-                                        let res = if url.is_empty() {
-                                            local_libs.get(&lib.name).cloned()
-                                        } else {
-                                            Some(daedalus::download_file(
-                                                url,
-                                                None,
-                                            )
-                                                .await?)
-                                        };
 
-                                        if res.is_some() {
+                                        if let Some(bytes) = artifact_bytes {
-                                            lib.url = Some(format_url("maven/"));
+                                            upload_file_to_bucket(
+                                                format!("{}/{}", "maven", artifact_path),
+                                                bytes.to_vec(),
+                                                Some("application/java-archive".to_string()),
+                                                uploaded_files_mutex.as_ref(),
+                                                semaphore.clone(),
+                                            ).await?;
                                         }
 
-                                        res
+                                        Ok::<Library, Error>(lib)
-                                    } else { None };
+                                    })).await?;
 
-                                    if let Some(bytes) = artifact_bytes {
+                                    let elapsed = now.elapsed();
-                                        upload_file_to_bucket(
+                                    info!("Elapsed lib DL: {:.2?}", elapsed);
-                                            format!("{}/{}", "maven", artifact_path),
-                                            bytes.to_vec(),
-                                            Some("application/java-archive".to_string()),
-                                            uploaded_files_mutex.as_ref()
-                                        ).await?;
-                                    }
 
-                                    Ok::<Library, Error>(lib)
+                                    let new_profile = PartialVersionInfo {
-                                })).await?;
+                                        id: version_info.id,
+                                        inherits_from: version_info.inherits_from,
+                                        release_time: version_info.release_time,
+                                        time: version_info.time,
+                                        main_class: version_info.main_class,
+                                        minecraft_arguments: version_info.minecraft_arguments,
+                                        arguments: version_info.arguments,
+                                        libraries: libs,
+                                        type_: version_info.type_,
+                                        data: Some(profile.data),
+                                        processors: Some(profile.processors),
+                                    };
 
-                                let elapsed = now.elapsed();
+                                    let version_path = format!(
-                                info!("Elapsed lib DL: {:.2?}", elapsed);
+                                        "forge/v{}/versions/{}.json",
+                                        daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
+                                        new_profile.id
+                                    );
 
-                                let new_profile = PartialVersionInfo {
+                                    upload_file_to_bucket(
-                                    id: version_info.id,
+                                        version_path.clone(),
-                                    inherits_from: version_info.inherits_from,
+                                        serde_json::to_vec(&new_profile)?,
-                                    release_time: version_info.release_time,
+                                        Some("application/json".to_string()),
-                                    time: version_info.time,
+                                        uploaded_files_mutex.as_ref(),
-                                    main_class: version_info.main_class,
+                                        semaphore.clone(),
-                                    minecraft_arguments: version_info.minecraft_arguments,
+                                    ).await?;
-                                    arguments: version_info.arguments,
-                                    libraries: libs,
-                                    type_: version_info.type_,
-                                    data: Some(profile.data),
-                                    processors: Some(profile.processors),
-                                };
 
-                                let version_path = format!(
+                                    return Ok(Some(LoaderVersion {
-                                    "forge/v{}/versions/{}.json",
+                                        id: loader_version_full,
-                                    daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
+                                        url: format_url(&version_path),
-                                    new_profile.id
+                                        stable: false
-                                );
+                                    }));
-
+                                }
-                                upload_file_to_bucket(
-                                    version_path.clone(),
-                                    serde_json::to_vec(&new_profile)?,
-                                    Some("application/json".to_string()),
-                                    uploaded_files_mutex.as_ref()
-                                ).await?;
-
-                                return Ok(Some(LoaderVersion {
-                                    id: loader_version_full,
-                                    url: format_url(&version_path),
-                                    stable: false
-                                }));
                             }
+
+                            Ok(None)
+                        }.await
+                    });
+
+                    {
+                        let mut versions = loaders_futures.into_iter().peekable();
+                        let mut chunk_index = 0;
+                        while versions.peek().is_some() {
+                            let now = Instant::now();
+
+                            let chunk: Vec<_> = versions.by_ref().take(1).collect();
+                            futures::future::try_join_all(chunk).await?;
+
+                            chunk_index += 1;
+
+                            let elapsed = now.elapsed();
+                            info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
                         }
-
+                    }
-                        Ok(None)
+                    //futures::future::try_join_all(loaders_futures).await?;
-                    }.await
-                }).into_iter().peekable()/*.into_iter().flatten().collect()*/;
-
-                let mut chunk_index = 0;
-                while loaders_futures.peek().is_some() {
-                    info!("Loader Chunk {} Start", chunk_index);
-                    let now = Instant::now();
-
-                    let chunk: Vec<_> = loaders_futures.by_ref().take(10).collect();
-
-                    let res = futures::future::try_join_all(chunk).await?;
-                    loaders_versions.extend(res.into_iter().flatten());
-
-                    tokio::time::sleep(Duration::from_secs(1)).await;
-
-                    chunk_index += 1;
-
-                    let elapsed = now.elapsed();
-                    info!("Loader Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
                 }
-            }
 
-            versions.lock().await.push(daedalus::modded::Version {
+                versions.lock().await.push(daedalus::modded::Version {
-                id: minecraft_version,
+                    id: minecraft_version,
-                loaders: loaders_versions
+                    stable: true,
+                    loaders: loaders_versions
+                });
+
+                Ok::<(), Error>(())
             });
-
+        }
-            Ok::<(), Error>(())
-        });
     }
 
     {
-        let mut versions_peek = version_futures.into_iter().peekable();
+        let mut versions = version_futures.into_iter().peekable();
         let mut chunk_index = 0;
-        while versions_peek.peek().is_some() {
+        while versions.peek().is_some() {
-            info!("Chunk {} Start", chunk_index);
             let now = Instant::now();
 
-            let chunk: Vec<_> = versions_peek.by_ref().take(1).collect();
+            let chunk: Vec<_> = versions.by_ref().take(10).collect();
             futures::future::try_join_all(chunk).await?;
 
-            tokio::time::sleep(Duration::from_secs(1)).await;
-
             chunk_index += 1;
 
             let elapsed = now.elapsed();
             info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
         }
     }
+    //futures::future::try_join_all(version_futures).await?;
 
     if let Ok(versions) = Arc::try_unwrap(versions) {
         let mut versions = versions.into_inner();
@@ -495,13 +511,17 @@ pub async fn retrieve_data(
             minecraft_versions
                 .versions
                 .iter()
-                .position(|z| x.id == z.id)
+                .position(|z| {
+                    x.id.replace("1.7.10_pre4", "1.7.10-pre4") == z.id
+                })
                 .unwrap_or_default()
                 .cmp(
                     &minecraft_versions
                         .versions
                         .iter()
-                        .position(|z| y.id == z.id)
+                        .position(|z| {
+                            y.id.replace("1.7.10_pre4", "1.7.10-pre4") == z.id
+                        })
                         .unwrap_or_default(),
                 )
         });
@@ -534,6 +554,7 @@ pub async fn retrieve_data(
             })?,
             Some("application/json".to_string()),
             uploaded_files_mutex.as_ref(),
+            semaphore,
         )
         .await?;
     }
@@ -554,9 +575,15 @@ const DEFAULT_MAVEN_METADATA_URL: &str =
 /// the specified Minecraft version
 pub async fn fetch_maven_metadata(
     url: Option<&str>,
+    semaphore: Arc<Semaphore>,
 ) -> Result<HashMap<String, Vec<String>>, Error> {
     Ok(serde_json::from_slice(
-        &download_file(url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), None).await?,
+        &download_file(
+            url.unwrap_or(DEFAULT_MAVEN_METADATA_URL),
+            None,
+            semaphore,
+        )
+        .await?,
     )?)
 }
 

daedalus_client/src/main.rs

@@ -1,8 +1,10 @@
-use log::{error, warn};
+use log::{error, info, warn};
-use std::time::Duration;
-use s3::{Bucket, Region};
 use s3::creds::Credentials;
 use s3::error::S3Error;
+use s3::{Bucket, Region};
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::Semaphore;
 
 mod fabric;
 mod forge;
@@ -19,10 +21,7 @@ pub enum Error {
     #[error("Error while managing asynchronous tasks")]
     TaskError(#[from] tokio::task::JoinError),
     #[error("Error while uploading file to S3")]
-    S3Error {
+    S3Error { inner: S3Error, file: String },
-        inner: S3Error,
-        file: String,
-    },
     #[error("Error while parsing version as semver: {0}")]
     SemVerError(#[from] semver::Error),
     #[error("Error while reading zip file: {0}")]
@@ -31,6 +30,8 @@ pub enum Error {
     IoError(#[from] std::io::Error),
     #[error("Error while obtaining strong reference to Arc")]
     ArcError,
+    #[error("Error acquiring semaphore: {0}")]
+    AcquireError(#[from] tokio::sync::AcquireError),
 }
 
 #[tokio::main]
@@ -43,14 +44,20 @@ async fn main() {
         return;
     }
 
-    let mut timer = tokio::time::interval(Duration::from_secs(10 * 60));
+    let mut timer = tokio::time::interval(Duration::from_secs(30 * 60));
+    let semaphore = Arc::new(Semaphore::new(50));
 
     loop {
         timer.tick().await;
 
         let mut uploaded_files = Vec::new();
 
-        let versions = match minecraft::retrieve_data(&mut uploaded_files).await {
+        let versions = match minecraft::retrieve_data(
+            &mut uploaded_files,
+            semaphore.clone(),
+        )
+        .await
+        {
             Ok(res) => Some(res),
             Err(err) => {
                 error!("{:?}", err);
@@ -60,11 +67,23 @@ async fn main() {
         };
 
         if let Some(manifest) = versions {
-            match fabric::retrieve_data(&manifest, &mut uploaded_files).await {
+            match fabric::retrieve_data(
+                &manifest,
+                &mut uploaded_files,
+                semaphore.clone(),
+            )
+            .await
+            {
                 Ok(..) => {}
                 Err(err) => error!("{:?}", err),
             };
-            match forge::retrieve_data(&manifest, &mut uploaded_files).await {
+            match forge::retrieve_data(
+                &manifest,
+                &mut uploaded_files,
+                semaphore.clone(),
+            )
+            .await
+            {
                 Ok(..) => {}
                 Err(err) => error!("{:?}", err),
             };
@@ -93,7 +112,6 @@ fn check_env_vars() -> bool {
     }
 
     failed |= check_var::<String>("BASE_URL");
-    failed |= check_var::<String>("BASE_FOLDER");
 
     failed |= check_var::<String>("S3_ACCESS_TOKEN");
     failed |= check_var::<String>("S3_SECRET");
@@ -104,7 +122,6 @@ fn check_env_vars() -> bool {
     failed
 }
 
-
 lazy_static::lazy_static! {
     static ref CLIENT : Bucket = Bucket::new(
         &dotenvy::var("S3_BUCKET_NAME").unwrap(),
@@ -133,29 +150,29 @@ pub async fn upload_file_to_bucket(
     bytes: Vec<u8>,
     content_type: Option<String>,
     uploaded_files: &tokio::sync::Mutex<Vec<String>>,
+    semaphore: Arc<Semaphore>,
 ) -> Result<(), Error> {
-    let key = format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path);
+    let _permit = semaphore.acquire().await?;
+    info!("{} started uploading", path);
+    let key = path.clone();
 
     for attempt in 1..=4 {
         let result = if let Some(ref content_type) = content_type {
-            CLIENT.put_object_with_content_type(
+            CLIENT
-                key.clone(),
+                .put_object_with_content_type(key.clone(), &bytes, content_type)
-                &bytes,
+                .await
-                content_type,
-            ).await
         } else {
-            CLIENT.put_object(
+            CLIENT.put_object(key.clone(), &bytes).await
-                key.clone(),
+        }
-                &bytes,
+        .map_err(|err| Error::S3Error {
-            ).await
-        }.map_err(|err| Error::S3Error {
             inner: err,
-            file: format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path),
+            file: path.clone(),
         });
 
         match result {
             Ok(_) => {
                 {
+                    info!("{} done uploading", path);
                     let mut uploaded_files = uploaded_files.lock().await;
                     uploaded_files.push(key);
                 }
@@ -168,15 +185,39 @@ pub async fn upload_file_to_bucket(
             }
         }
     }
-
     unreachable!()
 }
 
 pub fn format_url(path: &str) -> String {
     format!(
-        "{}/{}/{}",
+        "{}/{}",
         &*dotenvy::var("BASE_URL").unwrap(),
-        &*dotenvy::var("BASE_FOLDER").unwrap(),
         path
     )
 }
+
+pub async fn download_file(
+    url: &str,
+    sha1: Option<&str>,
+    semaphore: Arc<Semaphore>,
+) -> Result<bytes::Bytes, Error> {
+    let _permit = semaphore.acquire().await?;
+    info!("{} started downloading", url);
+    let val = daedalus::download_file(url, sha1).await?;
+    info!("{} finished downloading", url);
+    Ok(val)
+}
+
+pub async fn download_file_mirrors(
+    base: &str,
+    mirrors: &[&str],
+    sha1: Option<&str>,
+    semaphore: Arc<Semaphore>,
+) -> Result<bytes::Bytes, Error> {
+    let _permit = semaphore.acquire().await?;
+    info!("{} started downloading", base);
+    let val = daedalus::download_file_mirrors(base, mirrors, sha1).await?;
+    info!("{} finished downloading", base);
+
+    Ok(val)
+}

daedalus_client/src/minecraft.rs

@@ -1,21 +1,26 @@
+use crate::download_file;
 use crate::{format_url, upload_file_to_bucket, Error};
-use daedalus::download_file;
 use daedalus::minecraft::VersionManifest;
 use log::info;
 use std::sync::Arc;
-use std::time::{Duration, Instant};
+use std::time::Instant;
-use tokio::sync::Mutex;
+use tokio::sync::{Mutex, Semaphore};
 
-pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionManifest, Error> {
+pub async fn retrieve_data(
-    let old_manifest =
+    uploaded_files: &mut Vec<String>,
-        daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&format!(
+    semaphore: Arc<Semaphore>,
+) -> Result<VersionManifest, Error> {
+    let old_manifest = daedalus::minecraft::fetch_version_manifest(Some(
+        &*format_url(&format!(
             "minecraft/v{}/manifest.json",
             daedalus::minecraft::CURRENT_FORMAT_VERSION
-        ))))
+        )),
-        .await
+    ))
-        .ok();
+    .await
+    .ok();
 
-    let mut manifest = daedalus::minecraft::fetch_version_manifest(None).await?;
+    let mut manifest =
+        daedalus::minecraft::fetch_version_manifest(None).await?;
     let cloned_manifest = Arc::new(Mutex::new(manifest.clone()));
 
     let visited_assets_mutex = Arc::new(Mutex::new(Vec::new()));
@@ -42,13 +47,16 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
             let visited_assets_mutex = Arc::clone(&visited_assets_mutex);
             let cloned_manifest_mutex = Arc::clone(&cloned_manifest);
             let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex);
+            let semaphore = Arc::clone(&semaphore);
 
-            let assets_hash = old_version.and_then(|x| x.assets_index_sha1.clone());
+            let assets_hash =
+                old_version.and_then(|x| x.assets_index_sha1.clone());
 
             async move {
                 let mut upload_futures = Vec::new();
 
-                let version_info = daedalus::minecraft::fetch_version_info(version).await?;
+                let version_info =
+                    daedalus::minecraft::fetch_version_info(version).await?;
 
                 let version_path = format!(
                     "minecraft/v{}/versions/{}.json",
@@ -63,14 +71,16 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
                 let assets_index_url = version_info.asset_index.url.clone();
 
                 {
-                    let mut cloned_manifest = cloned_manifest_mutex.lock().await;
+                    let mut cloned_manifest =
+                        cloned_manifest_mutex.lock().await;
 
                     let position = cloned_manifest
                         .versions
                         .iter()
                         .position(|x| version.id == x.id)
                         .unwrap();
-                    cloned_manifest.versions[position].url = format_url(&version_path);
+                    cloned_manifest.versions[position].url =
+                        format_url(&version_path);
                     cloned_manifest.versions[position].assets_index_sha1 =
                         Some(version_info.asset_index.sha1.clone());
                     cloned_manifest.versions[position].assets_index_url =
@@ -93,14 +103,18 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
                     }
 
                     if download_assets {
-                        visited_assets.push(version_info.asset_index.id.clone());
+                        visited_assets
+                            .push(version_info.asset_index.id.clone());
                     }
                 }
 
                 if download_assets {
-                    let assets_index =
+                    let assets_index = download_file(
-                        download_file(&assets_index_url, Some(&version_info.asset_index.sha1))
+                        &assets_index_url,
-                            .await?;
+                        Some(&version_info.asset_index.sha1),
+                        semaphore.clone(),
+                    )
+                    .await?;
 
                     {
                         upload_futures.push(upload_file_to_bucket(
@@ -108,6 +122,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
                             assets_index.to_vec(),
                             Some("application/json".to_string()),
                             uploaded_files_mutex.as_ref(),
+                            semaphore.clone(),
                         ));
                     }
                 }
@@ -118,6 +133,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
                         serde_json::to_vec(&version_info)?,
                         Some("application/json".to_string()),
                         uploaded_files_mutex.as_ref(),
+                        semaphore.clone(),
                     ));
                 }
 
@@ -140,14 +156,13 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
             let chunk: Vec<_> = versions.by_ref().take(100).collect();
             futures::future::try_join_all(chunk).await?;
 
-            tokio::time::sleep(Duration::from_secs(1)).await;
-
             chunk_index += 1;
 
             let elapsed = now.elapsed();
             info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
         }
     }
+    //futures::future::try_join_all(version_futures).await?;
 
     upload_file_to_bucket(
         format!(
@@ -157,6 +172,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
         serde_json::to_vec(&*cloned_manifest.lock().await)?,
         Some("application/json".to_string()),
         uploaded_files_mutex.as_ref(),
+        semaphore,
     )
     .await?;