Merge pull request #3 from modrinth/many-fixes

Many fixes

.env

@@ -1,7 +1,6 @@
 RUST_LOG=info,error
 
 BASE_URL=https://modrinth-cdn-staging.nyc3.digitaloceanspaces.com
-BASE_FOLDER=gamedata
 
 S3_ACCESS_TOKEN=none
 S3_SECRET=none

Dockerfile

@@ -1,4 +1,4 @@
-FROM rust:1.65
+FROM rust:1.68.2
 
 COPY ./ ./
 

daedalus/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "daedalus"
-version = "0.1.17"
+version = "0.1.18"
 authors = ["Jai A <jaiagr+gpg@pm.me>"]
 edition = "2018"
 license = "MIT"

daedalus/src/lib.rs

@@ -46,22 +46,34 @@ pub fn get_path_from_artifact(artifact: &str) -> Result<String, Error> {
     let name_items = artifact.split(':').collect::<Vec<&str>>();
 
     let package = name_items.first().ok_or_else(|| {
-        Error::ParseError(format!("Unable to find package for library {}", &artifact))
+        Error::ParseError(format!(
+            "Unable to find package for library {}",
+            &artifact
+        ))
     })?;
     let name = name_items.get(1).ok_or_else(|| {
-        Error::ParseError(format!("Unable to find name for library {}", &artifact))
+        Error::ParseError(format!(
+            "Unable to find name for library {}",
+            &artifact
+        ))
     })?;
 
     if name_items.len() == 3 {
         let version_ext = name_items
             .get(2)
             .ok_or_else(|| {
-                Error::ParseError(format!("Unable to find version for library {}", &artifact))
+                Error::ParseError(format!(
+                    "Unable to find version for library {}",
+                    &artifact
+                ))
             })?
             .split('@')
             .collect::<Vec<&str>>();
         let version = version_ext.first().ok_or_else(|| {
-            Error::ParseError(format!("Unable to find version for library {}", &artifact))
+            Error::ParseError(format!(
+                "Unable to find version for library {}",
+                &artifact
+            ))
         })?;
         let ext = version_ext.get(1);
 
@@ -76,18 +88,27 @@ pub fn get_path_from_artifact(artifact: &str) -> Result<String, Error> {
         ))
     } else {
         let version = name_items.get(2).ok_or_else(|| {
-            Error::ParseError(format!("Unable to find version for library {}", &artifact))
+            Error::ParseError(format!(
+                "Unable to find version for library {}",
+                &artifact
+            ))
         })?;
 
         let data_ext = name_items
             .get(3)
             .ok_or_else(|| {
-                Error::ParseError(format!("Unable to find data for library {}", &artifact))
+                Error::ParseError(format!(
+                    "Unable to find data for library {}",
+                    &artifact
+                ))
             })?
             .split('@')
             .collect::<Vec<&str>>();
         let data = data_ext.first().ok_or_else(|| {
-            Error::ParseError(format!("Unable to find data for library {}", &artifact))
+            Error::ParseError(format!(
+                "Unable to find data for library {}",
+                &artifact
+            ))
         })?;
         let ext = data_ext.get(1);
 
@@ -126,10 +147,21 @@ pub async fn download_file_mirrors(
 }
 
 /// Downloads a file with retry and checksum functionality
-pub async fn download_file(url: &str, sha1: Option<&str>) -> Result<bytes::Bytes, Error> {
+pub async fn download_file(
+    url: &str,
+    sha1: Option<&str>,
+) -> Result<bytes::Bytes, Error> {
+    let mut headers = reqwest::header::HeaderMap::new();
+    if let Ok(header) = reqwest::header::HeaderValue::from_str(&format!(
+        "modrinth/daedalus/{} (support@modrinth.com)",
+        env!("CARGO_PKG_VERSION")
+    )) {
+        headers.insert(reqwest::header::USER_AGENT, header);
+    }
     let client = reqwest::Client::builder()
         .tcp_keepalive(Some(std::time::Duration::from_secs(10)))
         .timeout(std::time::Duration::from_secs(15))
+        .default_headers(headers)
         .build()
         .map_err(|err| Error::FetchError {
             inner: err,
@@ -183,7 +215,9 @@ pub async fn download_file(url: &str, sha1: Option<&str>) -> Result<bytes::Bytes
 
 /// Computes a checksum of the input bytes
 pub async fn get_hash(bytes: bytes::Bytes) -> Result<String, Error> {
-    let hash = tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest()).await?;
+    let hash =
+        tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest())
+            .await?;
 
     Ok(hash)
 }

daedalus/src/modded.rs

@@ -15,6 +15,9 @@ pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0;
 /// The latest version of the format the fabric model structs deserialize to
 pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0;
 
+/// The dummy replace string library names, inheritsFrom, and version names should be replaced with
+pub const DUMMY_REPLACE_STRING: &str = "${modrinth.gameVersion}";
+
 /// A data variable entry that depends on the side of the installation
 #[cfg_attr(feature = "bincode", derive(Encode, Decode))]
 #[derive(Serialize, Deserialize, Debug)]
@@ -93,6 +96,8 @@ pub fn merge_partial_version(
     partial: PartialVersionInfo,
     merge: VersionInfo,
 ) -> VersionInfo {
+    let merge_id = merge.id.clone();
+
     VersionInfo {
         arguments: if let Some(partial_args) = partial.arguments {
             if let Some(merge_args) = merge.arguments {
@@ -126,12 +131,22 @@ pub fn merge_partial_version(
         asset_index: merge.asset_index,
         assets: merge.assets,
         downloads: merge.downloads,
-        id: partial.id,
+        id: merge.id,
         java_version: merge.java_version,
         libraries: partial
             .libraries
             .into_iter()
             .chain(merge.libraries)
+            .map(|x| Library {
+                downloads: x.downloads,
+                extract: x.extract,
+                name: x.name.replace(DUMMY_REPLACE_STRING, &merge_id),
+                url: x.url,
+                natives: x.natives,
+                rules: x.rules,
+                checksums: x.checksums,
+                include_in_classpath: x.include_in_classpath,
+            })
             .collect::<Vec<_>>(),
         main_class: if let Some(main_class) = partial.main_class {
             main_class
@@ -163,6 +178,8 @@ pub struct Manifest {
 pub struct Version {
     /// The minecraft version ID
     pub id: String,
+    /// Whether the release is stable or not
+    pub stable: bool,
     /// A map that contains loader versions for the game version
     pub loaders: Vec<LoaderVersion>,
 }

daedalus_client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "daedalus_client"
-version = "0.1.17"
+version = "0.1.18"
 authors = ["Jai A <jaiagr+gpg@pm.me>"]
 edition = "2018"
 

daedalus_client/src/fabric.rs

@@ -1,97 +1,85 @@
-use crate::{format_url, upload_file_to_bucket, Error};
+use crate::{download_file, format_url, upload_file_to_bucket, Error};
-use daedalus::download_file;
 use daedalus::minecraft::{Library, VersionManifest};
-use daedalus::modded::{LoaderVersion, Manifest, PartialVersionInfo, Version};
+use daedalus::modded::{
-use log::info;
+    LoaderVersion, Manifest, PartialVersionInfo, Version, DUMMY_REPLACE_STRING,
+};
 use serde::{Deserialize, Serialize};
 use std::sync::Arc;
-use std::time::{Duration, Instant};
+use tokio::sync::{Mutex, RwLock, Semaphore};
-use tokio::sync::{Mutex, RwLock};
 
 pub async fn retrieve_data(
     minecraft_versions: &VersionManifest,
     uploaded_files: &mut Vec<String>,
+    semaphore: Arc<Semaphore>,
 ) -> Result<(), Error> {
-    let mut list = fetch_fabric_versions(None).await?;
+    let mut list = fetch_fabric_versions(None, semaphore.clone()).await?;
-    let old_manifest = daedalus::modded::fetch_manifest(&format!(
+    let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!(
         "fabric/v{}/manifest.json",
         daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION,
-    ))
+    )))
     .await
     .ok();
 
-    let versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest {
+    let mut versions = if let Some(old_manifest) = old_manifest {
         old_manifest.game_versions
     } else {
         Vec::new()
-    }));
+    };
 
-    let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new()));
+    let loaders_mutex = RwLock::new(Vec::new());
-
-    if let Some(latest) = list.loader.get(0) {
-        let loaders_mutex = Arc::new(RwLock::new(Vec::new()));
-        let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new()));
 
     {
         let mut loaders = loaders_mutex.write().await;
 
-            // for loader in &list.loader {
+        for loader in &list.loader {
-            //     loaders.push((Box::new(loader.stable), loader.version.clone()))
+            loaders.push((Box::new(loader.stable), loader.version.clone()))
-            // }
-
-            loaders.push((Box::new(latest.stable), latest.version.clone()));
-
-            if !latest.stable {
-                if let Some(stable) = list.loader.iter().find(|x| x.stable) {
-                    loaders.push((Box::new(stable.stable), stable.version.clone()));
-                }
         }
 
-            list.loader.retain(|x| loaders.iter().any(|val| val.1 == x.version))
+        list.loader
+            .retain(|x| loaders.iter().any(|val| val.1 == x.version))
     }
 
-        let mut version_futures = Vec::new();
+    const DUMMY_GAME_VERSION: &str = "1.19.4-rc2";
 
-        for game_version in list.game.iter_mut() {
-            let visited_artifacts_mutex = Arc::clone(&visited_artifacts_mutex);
-            let loaders_mutex = Arc::clone(&loaders_mutex);
-            let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex);
-
-            let versions_mutex = Arc::clone(&versions);
-            version_futures.push(async move {
     let loader_version_mutex = Mutex::new(Vec::new());
+    let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new()));
 
-                let versions =
+    let loader_versions = futures::future::try_join_all(
-                    futures::future::try_join_all(
         loaders_mutex.read().await.clone().into_iter().map(
             |(stable, loader)| async {
                 {
-                                    if versions_mutex.lock().await.iter().any(|x| {
+                    if versions.iter().any(|x| {
-                                        x.id == game_version.version
+                        x.id == DUMMY_REPLACE_STRING
                             && x.loaders.iter().any(|x| x.id == loader)
                     }) {
                         return Ok(None);
                     }
                 }
 
-                                let version =
+                let version = fetch_fabric_version(
-                                    fetch_fabric_version(&game_version.version, &loader).await?;
+                    DUMMY_GAME_VERSION,
+                    &loader,
+                    semaphore.clone(),
+                )
+                .await?;
 
-                                Ok::<Option<(Box<bool>, String, PartialVersionInfo)>, Error>(Some(
+                Ok::<Option<(Box<bool>, String, PartialVersionInfo)>, Error>(
-                                    (stable, loader, version),
+                    Some((stable, loader, version)),
-                                ))
+                )
             },
         ),
     )
-                    .await?
+    .await?;
-                    .into_iter()
-                    .flatten();
 
-                futures::future::try_join_all(versions.map(|(stable, loader, version)| async {
+    let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new()));
-                    let libs = futures::future::try_join_all(version.libraries.into_iter().map(
+    futures::future::try_join_all(loader_versions.into_iter()
-                        |mut lib| async {
+        .flatten().map(
+        |(stable, loader, version)| async {
+            let libs = futures::future::try_join_all(
+                version.libraries.into_iter().map(|mut lib| async {
                     {
-                                let mut visited_assets = visited_artifacts_mutex.lock().await;
+                        let mut visited_assets =
+                            visited_artifacts_mutex.lock().await;
 
                         if visited_assets.contains(&lib.name) {
                             lib.url = Some(format_url("maven/"));
@@ -102,9 +90,54 @@ pub async fn retrieve_data(
                         }
                     }
 
-                            let artifact_path = daedalus::get_path_from_artifact(&lib.name)?;
+                    if lib.name.contains(DUMMY_GAME_VERSION) {
+                        lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING);
+                        lib.url = Some(format_url("maven/"));
+                        futures::future::try_join_all(list.game.clone().into_iter().map(|game_version| async {
+                            let semaphore = semaphore.clone();
+                            let uploaded_files_mutex = uploaded_files_mutex.clone();
+                            let lib_name = lib.name.clone();
+                            let lib_url = lib.url.clone();
 
-                            let artifact = daedalus::download_file(
+                            async move {
+                                let artifact_path =
+                                    daedalus::get_path_from_artifact(&lib_name.replace(DUMMY_REPLACE_STRING, &game_version.version))?;
+
+                                let artifact = download_file(
+                                    &format!(
+                                        "{}{}",
+                                        lib_url.unwrap_or_else(|| {
+                                            "https://maven.fabricmc.net/".to_string()
+                                        }),
+                                        artifact_path
+                                    ),
+                                    None,
+                                    semaphore.clone(),
+                                )
+                                    .await?;
+
+                                upload_file_to_bucket(
+                                    format!("{}/{}", "maven", artifact_path),
+                                    artifact.to_vec(),
+                                    Some("application/java-archive".to_string()),
+                                    &uploaded_files_mutex,
+                                    semaphore.clone(),
+                                )
+                                    .await?;
+
+                                Ok::<(), Error>(())
+                            }.await?;
+
+                            Ok::<(), Error>(())
+                        })).await?;
+
+                        return Ok(lib);
+                    }
+
+                    let artifact_path =
+                        daedalus::get_path_from_artifact(&lib.name)?;
+
+                    let artifact = download_file(
                         &format!(
                             "{}{}",
                             lib.url.unwrap_or_else(|| {
@@ -113,6 +146,7 @@ pub async fn retrieve_data(
                             artifact_path
                         ),
                         None,
+                        semaphore.clone(),
                     )
                     .await?;
 
@@ -122,41 +156,44 @@ pub async fn retrieve_data(
                         format!("{}/{}", "maven", artifact_path),
                         artifact.to_vec(),
                         Some("application/java-archive".to_string()),
-                                uploaded_files_mutex.as_ref(),
+                        &uploaded_files_mutex,
+                        semaphore.clone(),
                     )
                     .await?;
 
                     Ok::<Library, Error>(lib)
-                        },
+                }),
-                    ))
+            )
             .await?;
 
             let version_path = format!(
-                        "fabric/v{}/versions/{}-{}.json",
+                "fabric/v{}/versions/{}.json",
                 daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION,
-                        version.inherits_from,
                 &loader
             );
 
-                    let inherits_from = version.inherits_from.clone();
-
             upload_file_to_bucket(
                 version_path.clone(),
                 serde_json::to_vec(&PartialVersionInfo {
                     arguments: version.arguments,
-                            id: version.id,
+                    id: version
+                        .id
+                        .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING),
                     main_class: version.main_class,
                     release_time: version.release_time,
                     time: version.time,
                     type_: version.type_,
-                            inherits_from: version.inherits_from,
+                    inherits_from: version
+                        .inherits_from
+                        .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING),
                     libraries: libs,
                     minecraft_arguments: version.minecraft_arguments,
                     processors: None,
                     data: None,
                 })?,
                 Some("application/json".to_string()),
-                        uploaded_files_mutex.as_ref(),
+                &uploaded_files_mutex,
+                semaphore.clone(),
             )
             .await?;
 
@@ -164,7 +201,7 @@ pub async fn retrieve_data(
                 let mut loader_version_map = loader_version_mutex.lock().await;
                 async move {
                     loader_version_map.push(LoaderVersion {
-                                id: format!("{}-{}", inherits_from, loader),
+                        id: loader.to_string(),
                         url: format_url(&version_path),
                         stable: *stable,
                     });
@@ -173,39 +210,24 @@ pub async fn retrieve_data(
             }
 
             Ok::<(), Error>(())
-                }))
+        },
+    ))
     .await?;
 
-                let mut versions = versions_mutex.lock().await;
     versions.push(Version {
-                    id: game_version.version.clone(),
+        id: DUMMY_REPLACE_STRING.to_string(),
+        stable: true,
         loaders: loader_version_mutex.into_inner(),
     });
 
-                Ok::<(), Error>(())
+    for version in &list.game {
+        versions.push(Version {
+            id: version.version.clone(),
+            stable: version.stable,
+            loaders: vec![],
         });
     }
 
-        let mut versions = version_futures.into_iter().peekable();
-        let mut chunk_index = 0;
-        while versions.peek().is_some() {
-            let now = Instant::now();
-
-            let chunk: Vec<_> = versions.by_ref().take(10).collect();
-            futures::future::try_join_all(chunk).await?;
-
-            tokio::time::sleep(Duration::from_secs(1)).await;
-
-            chunk_index += 1;
-
-            let elapsed = now.elapsed();
-            info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
-        }
-    }
-
-    if let Ok(versions) = Arc::try_unwrap(versions) {
-        let mut versions = versions.into_inner();
-
     versions.sort_by(|x, y| {
         minecraft_versions
             .versions
@@ -226,10 +248,7 @@ pub async fn retrieve_data(
             list.loader
                 .iter()
                 .position(|z| {
-                        x.id.split('-')
+                    x.id.split('-').next().unwrap_or_default() == &*z.version
-                            .next()
-                            .unwrap_or_default()
-                            == &*z.version
                 })
                 .unwrap_or_default()
                 .cmp(
@@ -237,9 +256,7 @@ pub async fn retrieve_data(
                         .loader
                         .iter()
                         .position(|z| {
-                                y.id.split('-')
+                            y.id.split('-').next().unwrap_or_default()
-                                    .next()
-                                    .unwrap_or_default()
                                 == z.version
                         })
                         .unwrap_or_default(),
@@ -256,10 +273,10 @@ pub async fn retrieve_data(
             game_versions: versions,
         })?,
         Some("application/json".to_string()),
-            uploaded_files_mutex.as_ref(),
+        &uploaded_files_mutex,
+        semaphore,
     )
     .await?;
-    }
 
     if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) {
         uploaded_files.extend(uploaded_files_mutex.into_inner());
@@ -273,6 +290,7 @@ const FABRIC_META_URL: &str = "https://meta.fabricmc.net/v2";
 async fn fetch_fabric_version(
     version_number: &str,
     loader_version: &str,
+    semaphore: Arc<Semaphore>,
 ) -> Result<PartialVersionInfo, Error> {
     Ok(serde_json::from_slice(
         &download_file(
@@ -281,6 +299,7 @@ async fn fetch_fabric_version(
                 FABRIC_META_URL, version_number, loader_version
             ),
             None,
+            semaphore,
         )
         .await?,
     )?)
@@ -319,11 +338,15 @@ struct FabricLoaderVersion {
     pub stable: bool,
 }
 /// Fetches the list of fabric versions
-async fn fetch_fabric_versions(url: Option<&str>) -> Result<FabricVersions, Error> {
+async fn fetch_fabric_versions(
+    url: Option<&str>,
+    semaphore: Arc<Semaphore>,
+) -> Result<FabricVersions, Error> {
     Ok(serde_json::from_slice(
         &download_file(
             url.unwrap_or(&*format!("{}/versions", FABRIC_META_URL)),
             None,
+            semaphore,
         )
         .await?,
     )?)

daedalus_client/src/forge.rs

@@ -1,8 +1,14 @@
-use crate::{format_url, upload_file_to_bucket, Error};
+use crate::{
+    download_file, download_file_mirrors, format_url, upload_file_to_bucket,
+    Error,
+};
 use chrono::{DateTime, Utc};
-use daedalus::download_file;
+use daedalus::minecraft::{
-use daedalus::minecraft::{Argument, ArgumentType, Library, VersionManifest, VersionType};
+    Argument, ArgumentType, Library, VersionManifest, VersionType,
-use daedalus::modded::{LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry};
+};
+use daedalus::modded::{
+    LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry,
+};
 use lazy_static::lazy_static;
 use log::info;
 use semver::{Version, VersionReq};
@@ -11,7 +17,7 @@ use std::collections::HashMap;
 use std::io::Read;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
-use tokio::sync::Mutex;
+use tokio::sync::{Mutex, Semaphore};
 
 lazy_static! {
     static ref FORGE_MANIFEST_V1_QUERY: VersionReq =
@@ -20,14 +26,16 @@ lazy_static! {
         VersionReq::parse(">=23.5.2851, <31.2.52").unwrap();
     static ref FORGE_MANIFEST_V2_QUERY_P2: VersionReq =
         VersionReq::parse(">=32.0.1, <37.0.0").unwrap();
-    static ref FORGE_MANIFEST_V3_QUERY: VersionReq = VersionReq::parse(">=37.0.0").unwrap();
+    static ref FORGE_MANIFEST_V3_QUERY: VersionReq =
+        VersionReq::parse(">=37.0.0").unwrap();
 }
 
 pub async fn retrieve_data(
     minecraft_versions: &VersionManifest,
     uploaded_files: &mut Vec<String>,
+    semaphore: Arc<Semaphore>,
 ) -> Result<(), Error> {
-    let maven_metadata = fetch_maven_metadata(None).await?;
+    let maven_metadata = fetch_maven_metadata(None, semaphore.clone()).await?;
     let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!(
         "forge/v{}/manifest.json",
         daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
@@ -35,7 +43,8 @@ pub async fn retrieve_data(
     .await
     .ok();
 
-    let old_versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest {
+    let old_versions =
+        Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest {
             old_manifest.game_versions
         } else {
             Vec::new()
@@ -52,13 +61,14 @@ pub async fn retrieve_data(
         let mut loaders = Vec::new();
 
         for loader_version_full in loader_versions {
-            let loader_version = loader_version_full.split('-').into_iter().nth(1);
+            let loader_version = loader_version_full.split('-').nth(1);
 
             if let Some(loader_version_raw) = loader_version {
                 // This is a dirty hack to get around Forge not complying with SemVer, but whatever
                 // Most of this is a hack anyways :(
                 // Works for all forge versions!
-                let split = loader_version_raw.split('.').collect::<Vec<&str>>();
+                let split =
+                    loader_version_raw.split('.').collect::<Vec<&str>>();
                 let loader_version = if split.len() >= 4 {
                     if split[0].parse::<i32>().unwrap_or(0) < 6 {
                         format!("{}.{}.{}", split[0], split[1], split[3])
@@ -80,21 +90,24 @@ pub async fn retrieve_data(
                 }
             }
         }
+
+        if !loaders.is_empty() {
             version_futures.push(async {
-            let mut loaders_versions = Vec::new();
+                let loaders_versions = Vec::new();
 
                 {
-                let mut loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async {
+                    let loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async {
                         let versions_mutex = Arc::clone(&old_versions);
                         let visited_assets = Arc::clone(&visited_assets_mutex);
                         let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex);
+                        let semaphore = Arc::clone(&semaphore);
                         let minecraft_version = minecraft_version.clone();
 
                         async move {
                             /// These forge versions are not worth supporting!
                             const WHITELIST : [&str; 1] = [
                                 // Not supported due to `data` field being `[]` even though the type is a map
-                            "1.12.2-14.23.5.2851"
+                                "1.12.2-14.23.5.2851",
                             ];
 
                             if WHITELIST.contains(&&*loader_version_full) {
@@ -112,7 +125,7 @@ pub async fn retrieve_data(
                             }
 
                             info!("Forge - Installer Start {}", loader_version_full.clone());
-                        let bytes = download_file(&format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?;
+                            let bytes = download_file(&format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None, semaphore.clone()).await?;
 
                             let reader = std::io::Cursor::new(bytes);
 
@@ -163,10 +176,11 @@ pub async fn retrieve_data(
                                             } else {
                                                 let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"];
 
-                                            daedalus::download_file_mirrors(
+                                                download_file_mirrors(
                                                     &artifact_path,
                                                     &mirrors,
                                                     None,
+                                                    semaphore.clone(),
                                                 )
                                                     .await?
                                             };
@@ -178,6 +192,7 @@ pub async fn retrieve_data(
                                                 artifact.to_vec(),
                                                 Some("application/java-archive".to_string()),
                                                 uploaded_files_mutex.as_ref(),
+                                                semaphore.clone(),
                                             ).await?;
                                         }
 
@@ -211,7 +226,8 @@ pub async fn retrieve_data(
                                         version_path.clone(),
                                         serde_json::to_vec(&new_profile)?,
                                         Some("application/json".to_string()),
-                                    uploaded_files_mutex.as_ref()
+                                        uploaded_files_mutex.as_ref(),
+                                        semaphore.clone(),
                                     ).await?;
 
                                     return Ok(Some(LoaderVersion {
@@ -241,7 +257,7 @@ pub async fn retrieve_data(
                                     }).await??;
 
 
-                                let mut libs : Vec<daedalus::minecraft::Library> = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library {
+                                    let mut libs : Vec<Library> = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library {
                                         downloads: x.downloads,
                                         extract: x.extract,
                                         name: x.name,
@@ -319,12 +335,9 @@ pub async fn retrieve_data(
                                                 read_data!(entry.client);
                                             }
 
-                                        // Do we really need to support server installs? Keeping this here
+                                            if entry.server.starts_with('/') {
-                                        // just in case
+                                                read_data!(entry.server);
-                                        //
+                                            }
-                                        // if entry.server.starts_with('/') {
-                                        //     read_data!(entry.server);
-                                        // }
                                         }
                                     }
 
@@ -356,9 +369,10 @@ pub async fn retrieve_data(
                                                 let res = if artifact.url.is_empty() {
                                                     local_libs.get(&lib.name).cloned()
                                                 } else {
-                                                Some(daedalus::download_file(
+                                                    Some(download_file(
                                                         &artifact.url,
                                                         Some(&*artifact.sha1),
+                                                        semaphore.clone(),
                                                     )
                                                         .await?)
                                                 };
@@ -373,9 +387,10 @@ pub async fn retrieve_data(
                                             let res = if url.is_empty() {
                                                 local_libs.get(&lib.name).cloned()
                                             } else {
-                                            Some(daedalus::download_file(
+                                                Some(download_file(
                                                     url,
                                                     None,
+                                                    semaphore.clone(),
                                                 )
                                                     .await?)
                                             };
@@ -392,7 +407,8 @@ pub async fn retrieve_data(
                                                 format!("{}/{}", "maven", artifact_path),
                                                 bytes.to_vec(),
                                                 Some("application/java-archive".to_string()),
-                                            uploaded_files_mutex.as_ref()
+                                                uploaded_files_mutex.as_ref(),
+                                                semaphore.clone(),
                                             ).await?;
                                         }
 
@@ -426,7 +442,8 @@ pub async fn retrieve_data(
                                         version_path.clone(),
                                         serde_json::to_vec(&new_profile)?,
                                         Some("application/json".to_string()),
-                                    uploaded_files_mutex.as_ref()
+                                        uploaded_files_mutex.as_ref(),
+                                        semaphore.clone(),
                                     ).await?;
 
                                     return Ok(Some(LoaderVersion {
@@ -439,54 +456,53 @@ pub async fn retrieve_data(
 
                             Ok(None)
                         }.await
-                }).into_iter().peekable()/*.into_iter().flatten().collect()*/;
-
-                let mut chunk_index = 0;
-                while loaders_futures.peek().is_some() {
-                    info!("Loader Chunk {} Start", chunk_index);
-                    let now = Instant::now();
-
-                    let chunk: Vec<_> = loaders_futures.by_ref().take(10).collect();
-
-                    let res = futures::future::try_join_all(chunk).await?;
-                    loaders_versions.extend(res.into_iter().flatten());
-
-                    tokio::time::sleep(Duration::from_secs(1)).await;
-
-                    chunk_index += 1;
-
-                    let elapsed = now.elapsed();
-                    info!("Loader Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
-                }
-            }
-
-            versions.lock().await.push(daedalus::modded::Version {
-                id: minecraft_version,
-                loaders: loaders_versions
                     });
 
-            Ok::<(), Error>(())
-        });
-    }
-
                     {
-        let mut versions_peek = version_futures.into_iter().peekable();
+                        let mut versions = loaders_futures.into_iter().peekable();
                         let mut chunk_index = 0;
-        while versions_peek.peek().is_some() {
+                        while versions.peek().is_some() {
-            info!("Chunk {} Start", chunk_index);
                             let now = Instant::now();
 
-            let chunk: Vec<_> = versions_peek.by_ref().take(1).collect();
+                            let chunk: Vec<_> = versions.by_ref().take(1).collect();
                             futures::future::try_join_all(chunk).await?;
 
-            tokio::time::sleep(Duration::from_secs(1)).await;
-
                             chunk_index += 1;
 
                             let elapsed = now.elapsed();
                             info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
                         }
                     }
+                    //futures::future::try_join_all(loaders_futures).await?;
+                }
+
+                versions.lock().await.push(daedalus::modded::Version {
+                    id: minecraft_version,
+                    stable: true,
+                    loaders: loaders_versions
+                });
+
+                Ok::<(), Error>(())
+            });
+        }
+    }
+
+    {
+        let mut versions = version_futures.into_iter().peekable();
+        let mut chunk_index = 0;
+        while versions.peek().is_some() {
+            let now = Instant::now();
+
+            let chunk: Vec<_> = versions.by_ref().take(10).collect();
+            futures::future::try_join_all(chunk).await?;
+
+            chunk_index += 1;
+
+            let elapsed = now.elapsed();
+            info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
+        }
+    }
+    //futures::future::try_join_all(version_futures).await?;
 
     if let Ok(versions) = Arc::try_unwrap(versions) {
         let mut versions = versions.into_inner();
@@ -495,13 +511,17 @@ pub async fn retrieve_data(
             minecraft_versions
                 .versions
                 .iter()
-                .position(|z| x.id == z.id)
+                .position(|z| {
+                    x.id.replace("1.7.10_pre4", "1.7.10-pre4") == z.id
+                })
                 .unwrap_or_default()
                 .cmp(
                     &minecraft_versions
                         .versions
                         .iter()
-                        .position(|z| y.id == z.id)
+                        .position(|z| {
+                            y.id.replace("1.7.10_pre4", "1.7.10-pre4") == z.id
+                        })
                         .unwrap_or_default(),
                 )
         });
@@ -534,6 +554,7 @@ pub async fn retrieve_data(
             })?,
             Some("application/json".to_string()),
             uploaded_files_mutex.as_ref(),
+            semaphore,
         )
         .await?;
     }
@@ -554,9 +575,15 @@ const DEFAULT_MAVEN_METADATA_URL: &str =
 /// the specified Minecraft version
 pub async fn fetch_maven_metadata(
     url: Option<&str>,
+    semaphore: Arc<Semaphore>,
 ) -> Result<HashMap<String, Vec<String>>, Error> {
     Ok(serde_json::from_slice(
-        &download_file(url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), None).await?,
+        &download_file(
+            url.unwrap_or(DEFAULT_MAVEN_METADATA_URL),
+            None,
+            semaphore,
+        )
+        .await?,
     )?)
 }
 

daedalus_client/src/main.rs

@@ -1,8 +1,10 @@
-use log::{error, warn};
+use log::{error, info, warn};
-use std::time::Duration;
-use s3::{Bucket, Region};
 use s3::creds::Credentials;
 use s3::error::S3Error;
+use s3::{Bucket, Region};
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::sync::Semaphore;
 
 mod fabric;
 mod forge;
@@ -19,10 +21,7 @@ pub enum Error {
     #[error("Error while managing asynchronous tasks")]
     TaskError(#[from] tokio::task::JoinError),
     #[error("Error while uploading file to S3")]
-    S3Error {
+    S3Error { inner: S3Error, file: String },
-        inner: S3Error,
-        file: String,
-    },
     #[error("Error while parsing version as semver: {0}")]
     SemVerError(#[from] semver::Error),
     #[error("Error while reading zip file: {0}")]
@@ -31,6 +30,8 @@ pub enum Error {
     IoError(#[from] std::io::Error),
     #[error("Error while obtaining strong reference to Arc")]
     ArcError,
+    #[error("Error acquiring semaphore: {0}")]
+    AcquireError(#[from] tokio::sync::AcquireError),
 }
 
 #[tokio::main]
@@ -43,14 +44,20 @@ async fn main() {
         return;
     }
 
-    let mut timer = tokio::time::interval(Duration::from_secs(10 * 60));
+    let mut timer = tokio::time::interval(Duration::from_secs(30 * 60));
+    let semaphore = Arc::new(Semaphore::new(50));
 
     loop {
         timer.tick().await;
 
         let mut uploaded_files = Vec::new();
 
-        let versions = match minecraft::retrieve_data(&mut uploaded_files).await {
+        let versions = match minecraft::retrieve_data(
+            &mut uploaded_files,
+            semaphore.clone(),
+        )
+        .await
+        {
             Ok(res) => Some(res),
             Err(err) => {
                 error!("{:?}", err);
@@ -60,11 +67,23 @@ async fn main() {
         };
 
         if let Some(manifest) = versions {
-            match fabric::retrieve_data(&manifest, &mut uploaded_files).await {
+            match fabric::retrieve_data(
+                &manifest,
+                &mut uploaded_files,
+                semaphore.clone(),
+            )
+            .await
+            {
                 Ok(..) => {}
                 Err(err) => error!("{:?}", err),
             };
-            match forge::retrieve_data(&manifest, &mut uploaded_files).await {
+            match forge::retrieve_data(
+                &manifest,
+                &mut uploaded_files,
+                semaphore.clone(),
+            )
+            .await
+            {
                 Ok(..) => {}
                 Err(err) => error!("{:?}", err),
             };
@@ -93,7 +112,6 @@ fn check_env_vars() -> bool {
     }
 
     failed |= check_var::<String>("BASE_URL");
-    failed |= check_var::<String>("BASE_FOLDER");
 
     failed |= check_var::<String>("S3_ACCESS_TOKEN");
     failed |= check_var::<String>("S3_SECRET");
@@ -104,7 +122,6 @@ fn check_env_vars() -> bool {
     failed
 }
 
-
 lazy_static::lazy_static! {
     static ref CLIENT : Bucket = Bucket::new(
         &dotenvy::var("S3_BUCKET_NAME").unwrap(),
@@ -133,29 +150,29 @@ pub async fn upload_file_to_bucket(
     bytes: Vec<u8>,
     content_type: Option<String>,
     uploaded_files: &tokio::sync::Mutex<Vec<String>>,
+    semaphore: Arc<Semaphore>,
 ) -> Result<(), Error> {
-    let key = format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path);
+    let _permit = semaphore.acquire().await?;
+    info!("{} started uploading", path);
+    let key = path.clone();
 
     for attempt in 1..=4 {
         let result = if let Some(ref content_type) = content_type {
-            CLIENT.put_object_with_content_type(
+            CLIENT
-                key.clone(),
+                .put_object_with_content_type(key.clone(), &bytes, content_type)
-                &bytes,
+                .await
-                content_type,
-            ).await
         } else {
-            CLIENT.put_object(
+            CLIENT.put_object(key.clone(), &bytes).await
-                key.clone(),
+        }
-                &bytes,
+        .map_err(|err| Error::S3Error {
-            ).await
-        }.map_err(|err| Error::S3Error {
             inner: err,
-            file: format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path),
+            file: path.clone(),
         });
 
         match result {
             Ok(_) => {
                 {
+                    info!("{} done uploading", path);
                     let mut uploaded_files = uploaded_files.lock().await;
                     uploaded_files.push(key);
                 }
@@ -168,15 +185,39 @@ pub async fn upload_file_to_bucket(
             }
         }
     }
-
     unreachable!()
 }
 
 pub fn format_url(path: &str) -> String {
     format!(
-        "{}/{}/{}",
+        "{}/{}",
         &*dotenvy::var("BASE_URL").unwrap(),
-        &*dotenvy::var("BASE_FOLDER").unwrap(),
         path
     )
 }
+
+pub async fn download_file(
+    url: &str,
+    sha1: Option<&str>,
+    semaphore: Arc<Semaphore>,
+) -> Result<bytes::Bytes, Error> {
+    let _permit = semaphore.acquire().await?;
+    info!("{} started downloading", url);
+    let val = daedalus::download_file(url, sha1).await?;
+    info!("{} finished downloading", url);
+    Ok(val)
+}
+
+pub async fn download_file_mirrors(
+    base: &str,
+    mirrors: &[&str],
+    sha1: Option<&str>,
+    semaphore: Arc<Semaphore>,
+) -> Result<bytes::Bytes, Error> {
+    let _permit = semaphore.acquire().await?;
+    info!("{} started downloading", base);
+    let val = daedalus::download_file_mirrors(base, mirrors, sha1).await?;
+    info!("{} finished downloading", base);
+
+    Ok(val)
+}

daedalus_client/src/minecraft.rs

@@ -1,21 +1,26 @@
+use crate::download_file;
 use crate::{format_url, upload_file_to_bucket, Error};
-use daedalus::download_file;
 use daedalus::minecraft::VersionManifest;
 use log::info;
 use std::sync::Arc;
-use std::time::{Duration, Instant};
+use std::time::Instant;
-use tokio::sync::Mutex;
+use tokio::sync::{Mutex, Semaphore};
 
-pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionManifest, Error> {
+pub async fn retrieve_data(
-    let old_manifest =
+    uploaded_files: &mut Vec<String>,
-        daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&format!(
+    semaphore: Arc<Semaphore>,
+) -> Result<VersionManifest, Error> {
+    let old_manifest = daedalus::minecraft::fetch_version_manifest(Some(
+        &*format_url(&format!(
             "minecraft/v{}/manifest.json",
             daedalus::minecraft::CURRENT_FORMAT_VERSION
-        ))))
+        )),
+    ))
     .await
     .ok();
 
-    let mut manifest = daedalus::minecraft::fetch_version_manifest(None).await?;
+    let mut manifest =
+        daedalus::minecraft::fetch_version_manifest(None).await?;
     let cloned_manifest = Arc::new(Mutex::new(manifest.clone()));
 
     let visited_assets_mutex = Arc::new(Mutex::new(Vec::new()));
@@ -42,13 +47,16 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
             let visited_assets_mutex = Arc::clone(&visited_assets_mutex);
             let cloned_manifest_mutex = Arc::clone(&cloned_manifest);
             let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex);
+            let semaphore = Arc::clone(&semaphore);
 
-            let assets_hash = old_version.and_then(|x| x.assets_index_sha1.clone());
+            let assets_hash =
+                old_version.and_then(|x| x.assets_index_sha1.clone());
 
             async move {
                 let mut upload_futures = Vec::new();
 
-                let version_info = daedalus::minecraft::fetch_version_info(version).await?;
+                let version_info =
+                    daedalus::minecraft::fetch_version_info(version).await?;
 
                 let version_path = format!(
                     "minecraft/v{}/versions/{}.json",
@@ -63,14 +71,16 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
                 let assets_index_url = version_info.asset_index.url.clone();
 
                 {
-                    let mut cloned_manifest = cloned_manifest_mutex.lock().await;
+                    let mut cloned_manifest =
+                        cloned_manifest_mutex.lock().await;
 
                     let position = cloned_manifest
                         .versions
                         .iter()
                         .position(|x| version.id == x.id)
                         .unwrap();
-                    cloned_manifest.versions[position].url = format_url(&version_path);
+                    cloned_manifest.versions[position].url =
+                        format_url(&version_path);
                     cloned_manifest.versions[position].assets_index_sha1 =
                         Some(version_info.asset_index.sha1.clone());
                     cloned_manifest.versions[position].assets_index_url =
@@ -93,13 +103,17 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
                     }
 
                     if download_assets {
-                        visited_assets.push(version_info.asset_index.id.clone());
+                        visited_assets
+                            .push(version_info.asset_index.id.clone());
                     }
                 }
 
                 if download_assets {
-                    let assets_index =
+                    let assets_index = download_file(
-                        download_file(&assets_index_url, Some(&version_info.asset_index.sha1))
+                        &assets_index_url,
+                        Some(&version_info.asset_index.sha1),
+                        semaphore.clone(),
+                    )
                     .await?;
 
                     {
@@ -108,6 +122,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
                             assets_index.to_vec(),
                             Some("application/json".to_string()),
                             uploaded_files_mutex.as_ref(),
+                            semaphore.clone(),
                         ));
                     }
                 }
@@ -118,6 +133,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
                         serde_json::to_vec(&version_info)?,
                         Some("application/json".to_string()),
                         uploaded_files_mutex.as_ref(),
+                        semaphore.clone(),
                     ));
                 }
 
@@ -140,14 +156,13 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
             let chunk: Vec<_> = versions.by_ref().take(100).collect();
             futures::future::try_join_all(chunk).await?;
 
-            tokio::time::sleep(Duration::from_secs(1)).await;
-
             chunk_index += 1;
 
             let elapsed = now.elapsed();
             info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
         }
     }
+    //futures::future::try_join_all(version_futures).await?;
 
     upload_file_to_bucket(
         format!(
@@ -157,6 +172,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<VersionMa
         serde_json::to_vec(&*cloned_manifest.lock().await)?,
         Some("application/json".to_string()),
         uploaded_files_mutex.as_ref(),
+        semaphore,
     )
     .await?;