Add local libs to modrinth maven and other fixes

daedalus/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "daedalus"
-version = "0.1.3"
+version = "0.1.4"
 authors = ["Jai A <jaiagr+gpg@pm.me>"]
 edition = "2018"
 license = "MIT"

daedalus/src/lib.rs

@@ -51,23 +51,57 @@ pub fn get_path_from_artifact(artifact: &str) -> Result<String, Error> {
     let name = name_items.get(1).ok_or_else(|| {
         Error::ParseError(format!("Unable to find name for library {}", &artifact))
     })?;
-    let version_ext = name_items.get(2).ok_or_else(|| {
-        Error::ParseError(format!("Unable to find version for library {}", &artifact))
-    })?.split('@').collect::<Vec<&str>>();
-    let version = version_ext.get(0).ok_or_else(|| {
-        Error::ParseError(format!("Unable to find version for library {}", &artifact))
-    })?;
-    let ext = version_ext.get(1);
 
-    Ok(format!(
+    if name_items.len() == 3 {
-        "{}/{}/{}/{}-{}.{}",
+        let version_ext = name_items
-        package.replace(".", "/"),
+            .get(2)
-        name,
+            .ok_or_else(|| {
-        version,
+                Error::ParseError(format!("Unable to find version for library {}", &artifact))
-        name,
+            })?
-        version,
+            .split('@')
-        ext.unwrap_or(&"jar")
+            .collect::<Vec<&str>>();
-    ))
+        let version = version_ext.get(0).ok_or_else(|| {
+            Error::ParseError(format!("Unable to find version for library {}", &artifact))
+        })?;
+        let ext = version_ext.get(1);
+
+        Ok(format!(
+            "{}/{}/{}/{}-{}.{}",
+            package.replace(".", "/"),
+            name,
+            version,
+            name,
+            version,
+            ext.unwrap_or(&"jar")
+        ))
+    } else {
+        let version = name_items.get(2).ok_or_else(|| {
+            Error::ParseError(format!("Unable to find version for library {}", &artifact))
+        })?;
+
+        let data_ext = name_items
+            .get(3)
+            .ok_or_else(|| {
+                Error::ParseError(format!("Unable to find data for library {}", &artifact))
+            })?
+            .split('@')
+            .collect::<Vec<&str>>();
+        let data = data_ext.get(0).ok_or_else(|| {
+            Error::ParseError(format!("Unable to find data for library {}", &artifact))
+        })?;
+        let ext = data_ext.get(1);
+
+        Ok(format!(
+            "{}/{}/{}/{}-{}-{}.{}",
+            package.replace(".", "/"),
+            name,
+            version,
+            name,
+            version,
+            data,
+            ext.unwrap_or(&"jar")
+        ))
+    }
 }
 
 /// Downloads a file from specified mirrors

daedalus/src/modded.rs

@@ -76,7 +76,10 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) ->
             if let Some(merge_args) = merge.arguments {
                 let mut new_map = HashMap::new();
 
-                fn add_keys(new_map: &mut HashMap<ArgumentType, Vec<Argument>>, args: HashMap<ArgumentType, Vec<Argument>>) {
+                fn add_keys(
+                    new_map: &mut HashMap<ArgumentType, Vec<Argument>>,
+                    args: HashMap<ArgumentType, Vec<Argument>>,
+                ) {
                     for (type_, arguments) in args {
                         for arg in arguments {
                             if let Some(vec) = new_map.get_mut(&type_) {

daedalus_client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "daedalus_client"
-version = "0.1.0"
+version = "0.1.4"
 authors = ["Jai A <jaiagr+gpg@pm.me>"]
 edition = "2018"
 

daedalus_client/src/fabric.rs

@@ -2,12 +2,12 @@ use crate::{format_url, upload_file_to_bucket, Error};
 use daedalus::download_file;
 use daedalus::minecraft::Library;
 use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Version};
-use tokio::sync::Mutex;
+use log::info;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
-use log::info;
+use tokio::sync::Mutex;
 
 pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<(), Error> {
     let mut list = fetch_fabric_versions(None).await?;
@@ -125,7 +125,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<(), Error
                                 Some("application/java-archive".to_string()),
                                 uploaded_files_mutex.as_ref(),
                             )
-                                .await?;
+                            .await?;
 
                             Ok::<Library, Error>(lib)
                         },

daedalus_client/src/forge.rs

@@ -6,6 +6,7 @@ use daedalus::modded::{
     LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry,
 };
 use lazy_static::lazy_static;
+use log::info;
 use semver::{Version, VersionReq};
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -13,7 +14,6 @@ use std::io::Read;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
 use tokio::sync::Mutex;
-use log::info;
 
 lazy_static! {
     static ref FORGE_MANIFEST_V1_QUERY: VersionReq =
@@ -99,26 +99,30 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<(), Error
                     info!("Forge - Installer Start {}", loader_version_full.clone());
                     let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?;
 
-                    let reader = std::io::Cursor::new(&*bytes);
+                    let reader = std::io::Cursor::new(bytes);
 
-                    if let Ok(mut archive) = zip::ZipArchive::new(reader) {
+                    if let Ok(archive) = zip::ZipArchive::new(reader) {
                         if FORGE_MANIFEST_V1_QUERY.matches(&version) {
-                            let profile = {
+                            let mut archive_clone = archive.clone();
-                                let mut install_profile = archive.by_name("install_profile.json")?;
+                            let profile = tokio::task::spawn_blocking(move || {
+                                let mut install_profile = archive_clone.by_name("install_profile.json")?;
 
                                 let mut contents = String::new();
                                 install_profile.read_to_string(&mut contents)?;
 
-                                serde_json::from_str::<ForgeInstallerProfileV1>(&*contents)?
+                                Ok::<ForgeInstallerProfileV1, Error>(serde_json::from_str::<ForgeInstallerProfileV1>(&*contents)?)
-                            };
+                            }).await??;
 
-                            let forge_universal_bytes = {
+                            let mut archive_clone = archive.clone();
-                                let mut forge_universal_file = archive.by_name(&*profile.install.file_path)?;
+                            let file_path = profile.install.file_path.clone();
+                            let forge_universal_bytes = tokio::task::spawn_blocking(move || {
+                                let mut forge_universal_file = archive_clone.by_name(&*file_path)?;
                                 let mut forge_universal =  Vec::new();
                                 forge_universal_file.read_to_end(&mut forge_universal)?;
 
-                                bytes::Bytes::from(forge_universal)
+
-                            };
+                                Ok::<bytes::Bytes, Error>(bytes::Bytes::from(forge_universal))
+                            }).await??;
                             let forge_universal_path = profile.install.path.clone();
 
                             let now = Instant::now();
@@ -205,73 +209,170 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<(), Error
                                 loaders: map
                             })
                         } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) {
-                            let profile = {
+                            let mut archive_clone = archive.clone();
-                                let mut install_profile = archive.by_name("install_profile.json")?;
+                            let mut profile = tokio::task::spawn_blocking(move || {
+                                let mut install_profile = archive_clone.by_name("install_profile.json")?;
 
                                 let mut contents = String::new();
                                 install_profile.read_to_string(&mut contents)?;
 
-                                serde_json::from_str::<ForgeInstallerProfileV2>(&*contents)?
+                                Ok::<ForgeInstallerProfileV2, Error>(serde_json::from_str::<ForgeInstallerProfileV2>(&*contents)?)
-                            };
+                            }).await??;
 
-                            let version_info = {
+                            let mut archive_clone = archive.clone();
-                                let mut install_profile = archive.by_name("version.json")?;
+                            let version_info = tokio::task::spawn_blocking(move || {
+                                let mut install_profile = archive_clone.by_name("version.json")?;
 
                                 let mut contents = String::new();
                                 install_profile.read_to_string(&mut contents)?;
-                                serde_json::from_str::<PartialVersionInfo>(&*contents)?
-                            };
 
-                            let forge_universal_bytes = {
+                                Ok::<PartialVersionInfo, Error>(serde_json::from_str::<PartialVersionInfo>(&*contents)?)
-                                if let Some(path) = &profile.path {
+                            }).await??;
-                                    let mut forge_universal_file = archive.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*path)?))?;
-                                    let mut forge_universal =  Vec::new();
-                                    forge_universal_file.read_to_end(&mut forge_universal)?;
 
-                                    Some(bytes::Bytes::from(forge_universal))
+
-                                } else {
+                            let mut libs : Vec<daedalus::minecraft::Library> = profile.libraries.into_iter().chain(version_info.libraries).collect();
-                                    None
+
+                            let mut local_libs : HashMap<String, bytes::Bytes> = HashMap::new();
+
+                            for lib in &libs {
+                                if lib.downloads.as_ref().map(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).flatten().unwrap_or(false) {
+                                    let mut archive_clone = archive.clone();
+                                    let lib_name_clone = lib.name.clone();
+
+                                    let lib_bytes = tokio::task::spawn_blocking(move || {
+                                        let mut lib_file = archive_clone.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*lib_name_clone)?))?;
+                                        let mut lib_bytes =  Vec::new();
+                                        lib_file.read_to_end(&mut lib_bytes)?;
+
+                                        Ok::<bytes::Bytes, Error>(bytes::Bytes::from(lib_bytes))
+                                    }).await??;
+
+                                    local_libs.insert(lib.name.clone(), lib_bytes);
                                 }
-                            };
+                            }
 
-                            let now = Instant::now();
+                            let path = profile.path.clone();
-                            let libs = futures::future::try_join_all(profile.libraries.into_iter().chain(version_info.libraries).map(|mut lib| async {
+                            let version = profile.version.clone();
-                                if let Some(ref mut downloads) = lib.downloads {
-                                    if let Some(ref mut artifact) = downloads.artifact {
-                                        {
-                                            let mut visited_assets = visited_assets.lock().await;
 
-                                            if visited_assets.contains(&lib.name) {
+                            for entry in profile.data.values_mut() {
-                                                artifact.url = format_url(&*format!("maven/{}", artifact.path));
+                                if entry.client.starts_with('/') || entry.server.starts_with('/') {
+                                    macro_rules! read_data {
+                                        ($value:expr) => {
+                                            let mut archive_clone = archive.clone();
+                                            let value_clone = $value.clone();
+                                            let lib_bytes = tokio::task::spawn_blocking(move || {
+                                                let mut lib_file = archive_clone.by_name(&value_clone[1..value_clone.len()])?;
+                                                let mut lib_bytes =  Vec::new();
+                                                lib_file.read_to_end(&mut lib_bytes)?;
 
-                                                return Ok::<Library, Error>(lib);
+                                                Ok::<bytes::Bytes, Error>(bytes::Bytes::from(lib_bytes))
-                                            } else {
+                                            }).await??;
-                                                visited_assets.push(lib.name.clone())
+
+                                            let split = $value.split('/').last();
+
+                                            if let Some(last) = split {
+                                                let mut file = last.split('.');
+
+                                                if let Some(file_name) = file.next() {
+                                                    if let Some(ext) = file.next() {
+                                                        let path = format!("{}:{}@{}", path.as_deref().unwrap_or(&*format!("net.minecraftforge:forge:{}", version)), file_name, ext);
+                                                        $value = format!("[{}]", &path);
+                                                        local_libs.insert(path.clone(), bytes::Bytes::from(lib_bytes));
+
+                                                        libs.push(Library {
+                                                            downloads: None,
+                                                            extract: None,
+                                                            name: path,
+                                                            url: Some("".to_string()),
+                                                            natives: None,
+                                                            rules: None,
+                                                            checksums: None
+                                                        });
+                                                    }
+                                                }
                                             }
                                         }
+                                    }
 
-                                        let artifact_path =
+                                    if entry.client.starts_with('/') {
-                                            daedalus::get_path_from_artifact(&*lib.name)?;
+                                        read_data!(entry.client);
+                                    }
 
-                                        let artifact_bytes = if artifact.url.is_empty() {
+                                    // Do we really need to support server installs? Keeping this here
-                                            forge_universal_bytes.clone().unwrap_or_default()
+                                    // just in case
+                                    //
+                                    // if entry.server.starts_with('/') {
+                                    //     read_data!(entry.server);
+                                    // }
+                                }
+                            }
+
+                            let now = Instant::now();
+                            let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async {
+                                let artifact_path =
+                                    daedalus::get_path_from_artifact(&*lib.name)?;
+
+                                {
+                                    let mut visited_assets = visited_assets.lock().await;
+
+                                    if visited_assets.contains(&lib.name) {
+                                        if let Some(ref mut downloads) = lib.downloads {
+                                            if let Some(ref mut artifact) = downloads.artifact {
+                                                artifact.url = format_url(&*format!("maven/{}", artifact_path));
+                                            }
+                                        } else if lib.url.is_some() {
+                                            lib.url = Some(format_url(&*format!("maven/{}", artifact_path)));
+                                        }
+
+                                        return Ok::<Library, Error>(lib);
+                                    } else {
+                                        visited_assets.push(lib.name.clone())
+                                    }
+                                }
+
+                                let artifact_bytes = if let Some(ref mut downloads) = lib.downloads {
+                                    if let Some(ref mut artifact) = downloads.artifact {
+                                       let res = if artifact.url.is_empty() {
+                                            local_libs.get(&lib.name).cloned()
                                         } else {
-                                            daedalus::download_file(
+                                            Some(daedalus::download_file(
                                                 &*artifact.url,
                                                 Some(&*artifact.sha1),
                                             )
-                                                .await?
+                                                .await?)
                                         };
 
-                                        artifact.url = format_url(&*format!("maven/{}", artifact.path));
+                                        if res.is_some() {
+                                            artifact.url = format_url(&*format!("maven/{}", artifact_path));
+                                        }
 
-                                        upload_file_to_bucket(
+                                        res
-                                            format!("{}/{}", "maven", artifact_path),
+                                    } else { None }
-                                            artifact_bytes.to_vec(),
+                                } else if let Some(ref mut url) = lib.url {
-                                            Some("application/java-archive".to_string()),
+                                    let res = if url.is_empty() {
-                                            uploaded_files_mutex.as_ref()
+                                        local_libs.get(&lib.name).cloned()
-                                        ).await?;
+                                    } else {
+                                        Some(daedalus::download_file(
+                                            url,
+                                            None,
+                                        )
+                                            .await?)
+                                    };
+
+                                    if res.is_some() {
+                                        lib.url = Some(format_url(&*format!("maven/{}", artifact_path)));
                                     }
+
+                                    res
+                                } else { None };
+
+                                if let Some(bytes) = artifact_bytes {
+                                    upload_file_to_bucket(
+                                        format!("{}/{}", "maven", artifact_path),
+                                        bytes.to_vec(),
+                                        Some("application/java-archive".to_string()),
+                                        uploaded_files_mutex.as_ref()
+                                    ).await?;
                                 }
 
                                 Ok::<Library, Error>(lib)
@@ -334,7 +435,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<(), Error
             info!("Chunk {} Start", chunk_index);
             let now = Instant::now();
 
-            let chunk: Vec<_> = versions_peek.by_ref().take(100).collect();
+            let chunk: Vec<_> = versions_peek.by_ref().take(10).collect();
             futures::future::try_join_all(chunk).await?;
 
             tokio::time::sleep(Duration::from_secs(1)).await;
@@ -356,7 +457,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<(), Error
                 game_versions: versions.into_inner(),
             })?,
             Some("application/json".to_string()),
-            uploaded_files_mutex.as_ref()
+            uploaded_files_mutex.as_ref(),
         )
         .await?;
     }

daedalus_client/src/main.rs

@@ -62,7 +62,7 @@ async fn main() {
                 Err(err) => error!("{:?}", err),
             };
 
-            match  purge_digitalocean_cache(uploaded_files).await {
+            match purge_digitalocean_cache(uploaded_files).await {
                 Ok(..) => {}
                 Err(err) => error!("{:?}", err),
             };
@@ -139,27 +139,39 @@ pub async fn upload_file_to_bucket(
 ) -> Result<(), Error> {
     let key = format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path);
 
-    CLIENT
+    for attempt in 1..=4 {
-        .put_object(PutObjectRequest {
+        let result = CLIENT
-            bucket: dotenv::var("S3_BUCKET_NAME").unwrap(),
+            .put_object(PutObjectRequest {
-            key: key.clone(),
+                bucket: dotenv::var("S3_BUCKET_NAME").unwrap(),
-            body: Some(bytes.into()),
+                key: key.clone(),
-            acl: Some("public-read".to_string()),
+                body: Some(bytes.clone().into()),
-            content_type,
+                acl: Some("public-read".to_string()),
-            ..Default::default()
+                content_type: content_type.clone(),
-        })
+                ..Default::default()
-        .await
+            })
-        .map_err(|err| Error::S3Error {
+            .await
-            inner: err,
+            .map_err(|err| Error::S3Error {
-            file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path),
+                inner: err,
-        })?;
+                file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path),
+            });
 
-    {
+        match result {
-        let mut uploaded_files = uploaded_files.lock().await;
+            Ok(_) => {
-        uploaded_files.push(key);
+                {
+                    let mut uploaded_files = uploaded_files.lock().await;
+                    uploaded_files.push(key);
+                }
+
+                return Ok(());
+            }
+            Err(_) if attempt <= 3 => continue,
+            Err(_) => {
+                result?;
+            }
+        }
     }
 
-    Ok(())
+    unreachable!()
 }
 
 pub fn format_url(path: &str) -> String {
@@ -181,9 +193,9 @@ pub async fn purge_digitalocean_cache(files: Vec<String>) -> Result<(), Error> {
         .ok()
         .map(|x| x.parse::<bool>().ok())
         .flatten()
-        .unwrap_or(false) {
+        .unwrap_or(false)
-
+    {
-        return Ok(())
+        return Ok(());
     }
 
     let client = reqwest::Client::new();
@@ -193,11 +205,16 @@ pub async fn purge_digitalocean_cache(files: Vec<String>) -> Result<(), Error> {
             "https://api.digitalocean.com/v2/cdn/endpoints/{}/cache",
             &*dotenv::var("DO_ENDPOINT_ID").unwrap()
         ))
-        .header("Authorization", &*format!("Bearer {}", &*dotenv::var("DO_ACCESS_KEY").unwrap()))
+        .header(
+            "Authorization",
+            &*format!("Bearer {}", &*dotenv::var("DO_ACCESS_KEY").unwrap()),
+        )
         .json(&PurgeCacheRequest { files })
-        .send().await.map_err(|err| Error::FetchError {
+        .send()
+        .await
+        .map_err(|err| Error::FetchError {
             inner: err,
-            item: "purging digital ocean cache".to_string()
+            item: "purging digital ocean cache".to_string(),
         })?;
 
     Ok(())

daedalus_client/src/minecraft.rs

@@ -1,9 +1,9 @@
 use crate::{format_url, upload_file_to_bucket, Error};
 use daedalus::download_file;
-use tokio::sync::Mutex;
+use log::info;
 use std::sync::Arc;
 use std::time::{Duration, Instant};
-use log::info;
+use tokio::sync::Mutex;
 
 pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<(), Error> {
     let old_manifest =
@@ -107,7 +107,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<(), Error
                             assets_path,
                             assets_index.to_vec(),
                             Some("application/json".to_string()),
-                            uploaded_files_mutex.as_ref()
+                            uploaded_files_mutex.as_ref(),
                         ));
                     }
                 }
@@ -117,7 +117,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<(), Error
                         version_path,
                         serde_json::to_vec(&version_info)?,
                         Some("application/json".to_string()),
-                        uploaded_files_mutex.as_ref()
+                        uploaded_files_mutex.as_ref(),
                     ));
                 }
 
@@ -156,7 +156,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<(), Error
         ),
         serde_json::to_vec(&*cloned_manifest.lock().await)?,
         Some("application/json".to_string()),
-        uploaded_files_mutex.as_ref()
+        uploaded_files_mutex.as_ref(),
     )
     .await?;