Make java stuff public, fix forge erroring out due to ratelimiting

daedalus/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "daedalus"
-version = "0.1.8"
+version = "0.1.9"
 authors = ["Jai A <jaiagr+gpg@pm.me>"]
 edition = "2018"
 license = "MIT"

daedalus/src/minecraft.rs

@@ -233,9 +233,9 @@ pub struct LibraryExtract {
 /// Information about the java version the game needs
 pub struct JavaVersion {
     /// The component needed for the Java installation
-    component: String,
+    pub component: String,
     /// The major Java version number
-    major_version: u32,
+    pub major_version: u32,
 }
 
 #[derive(Serialize, Deserialize, Debug)]

daedalus_client/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "daedalus_client"
-version = "0.1.8"
+version = "0.1.9"
 authors = ["Jai A <jaiagr+gpg@pm.me>"]
 edition = "2018"
 

daedalus_client/src/forge.rs

@@ -27,6 +27,7 @@ pub async fn retrieve_data(
     minecraft_versions: &VersionManifest,
     uploaded_files: &mut Vec<String>,
 ) -> Result<(), Error> {
+    println!("forg");
     let maven_metadata = fetch_maven_metadata(None).await?;
     let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!(
         "forge/v{}/manifest.json",
@@ -81,7 +82,10 @@ pub async fn retrieve_data(
             }
         }
         version_futures.push(async {
-            let loaders_versions = futures::future::try_join_all(loaders.into_iter().map(|(loader_version_full, version)| async {
+            let mut loaders_versions = Vec::new();
+
+            {
+                let mut loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async {
                     let versions_mutex = Arc::clone(&old_versions);
                     let visited_assets = Arc::clone(&visited_assets_mutex);
                     let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex);
@@ -426,7 +430,26 @@ pub async fn retrieve_data(
 
                         Ok(None)
                     }.await
-            })).await?.into_iter().flatten().collect();
+                }).into_iter().peekable()/*.into_iter().flatten().collect()*/;
+
+                let mut chunk_index = 0;
+                while loaders_futures.peek().is_some() {
+                    info!("Loader Chunk {} Start", chunk_index);
+                    let now = Instant::now();
+
+                    let chunk: Vec<_> = loaders_futures.by_ref().take(10).collect();
+
+                    let res = futures::future::try_join_all(chunk).await?;
+                    loaders_versions.extend(res.into_iter().flatten());
+
+                    tokio::time::sleep(Duration::from_secs(1)).await;
+
+                    chunk_index += 1;
+
+                    let elapsed = now.elapsed();
+                    info!("Loader Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
+                }
+            }
 
             versions.lock().await.push(daedalus::modded::Version {
                 id: minecraft_version,
@@ -444,7 +467,7 @@ pub async fn retrieve_data(
             info!("Chunk {} Start", chunk_index);
             let now = Instant::now();
 
-            let chunk: Vec<_> = versions_peek.by_ref().take(10).collect();
+            let chunk: Vec<_> = versions_peek.by_ref().take(1).collect();
             futures::future::try_join_all(chunk).await?;
 
             tokio::time::sleep(Duration::from_secs(1)).await;

daedalus_client/src/main.rs

@@ -48,7 +48,7 @@ async fn main() {
 
     loop {
         timer.tick().await;
-        tokio::spawn(async {
+
         let mut uploaded_files = Vec::new();
 
         let versions = match minecraft::retrieve_data(&mut uploaded_files).await {
@@ -61,10 +61,10 @@ async fn main() {
         };
 
         if let Some(manifest) = versions {
-                match fabric::retrieve_data(&manifest, &mut uploaded_files).await {
+            // match fabric::retrieve_data(&manifest, &mut uploaded_files).await {
-                    Ok(..) => {}
+            //     Ok(..) => {}
-                    Err(err) => error!("{:?}", err),
+            //     Err(err) => error!("{:?}", err),
-                };
+            // };
             match forge::retrieve_data(&manifest, &mut uploaded_files).await {
                 Ok(..) => {}
                 Err(err) => error!("{:?}", err),
@@ -75,7 +75,6 @@ async fn main() {
             Ok(..) => {}
             Err(err) => error!("{:?}", err),
         };
-        });
     }
 }