use crate::download_file; use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::get_hash; use daedalus::minecraft::{ merge_partial_library, Library, PartialLibrary, VersionManifest, }; use log::info; use serde::Deserialize; use std::sync::Arc; use std::time::Instant; use tokio::sync::{Mutex, Semaphore}; pub async fn retrieve_data( uploaded_files: &mut Vec, semaphore: Arc, ) -> Result { let old_manifest = daedalus::minecraft::fetch_version_manifest(Some( &*format_url(&format!( "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION )), )) .await .ok(); let mut manifest = daedalus::minecraft::fetch_version_manifest(None).await?; let cloned_manifest = Arc::new(Mutex::new(old_manifest.clone().unwrap_or(manifest.clone()))); let patches = fetch_library_patches()?; let cloned_patches = Arc::new(&patches); let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); let now = Instant::now(); let mut version_futures = Vec::new(); for version in manifest.versions.iter_mut() { version_futures.push(async { let old_version = if let Some(old_manifest) = &old_manifest { old_manifest.versions.iter().find(|x| x.id == version.id) } else { None }; if old_version.is_some() { return Ok(()); } let visited_assets_mutex = Arc::clone(&visited_assets_mutex); let cloned_manifest_mutex = Arc::clone(&cloned_manifest); let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); let semaphore = Arc::clone(&semaphore); let patches = Arc::clone(&cloned_patches); let assets_hash = old_version.and_then(|x| x.assets_index_sha1.clone()); async move { let mut upload_futures = Vec::new(); let mut version_info = daedalus::minecraft::fetch_version_info(version).await?; fn patch_library( patches: &Vec, mut library: Library, ) -> Vec { let mut val = Vec::new(); let actual_patches = patches .iter() .filter(|x| x.match_.contains(&library.name)) .collect::>(); if !actual_patches.is_empty() { for patch in actual_patches { if let Some(override_) = &patch.override_ { library = merge_partial_library( override_.clone(), library, ); } if let Some(additional_libraries) = &patch.additional_libraries { for additional_library in additional_libraries { if patch .patch_additional_libraries .unwrap_or(false) { let mut libs = patch_library( patches, additional_library.clone(), ); val.append(&mut libs) } else { val.push(additional_library.clone()); } } } } val.push(library); } else { val.push(library); } val } let mut new_libraries = Vec::new(); for library in version_info.libraries.clone() { let mut libs = patch_library(&patches, library); new_libraries.append(&mut libs) } version_info.libraries = new_libraries; let version_info_hash = get_hash(bytes::Bytes::from( serde_json::to_vec(&version_info)?, )) .await?; let version_path = format!( "minecraft/v{}/versions/{}.json", daedalus::minecraft::CURRENT_FORMAT_VERSION, version.id ); let assets_path = format!( "minecraft/v{}/assets/{}.json", daedalus::minecraft::CURRENT_FORMAT_VERSION, version_info.asset_index.id ); let assets_index_url = version_info.asset_index.url.clone(); { let mut cloned_manifest = cloned_manifest_mutex.lock().await; let position = cloned_manifest .versions .iter() .position(|x| version.id == x.id) .unwrap(); cloned_manifest.versions[position].url = format_url(&version_path); cloned_manifest.versions[position].assets_index_sha1 = Some(version_info.asset_index.sha1.clone()); cloned_manifest.versions[position].assets_index_url = Some(format_url(&assets_path)); cloned_manifest.versions[position].sha1 = version_info_hash; } let mut download_assets = false; { let mut visited_assets = visited_assets_mutex.lock().await; if !visited_assets.contains(&version_info.asset_index.id) { if let Some(assets_hash) = assets_hash { if version_info.asset_index.sha1 != assets_hash { download_assets = true; } } else { download_assets = true; } } if download_assets { visited_assets .push(version_info.asset_index.id.clone()); } } if download_assets { let assets_index = download_file( &assets_index_url, Some(&version_info.asset_index.sha1), semaphore.clone(), ) .await?; { upload_futures.push(upload_file_to_bucket( assets_path, assets_index.to_vec(), Some("application/json".to_string()), uploaded_files_mutex.as_ref(), semaphore.clone(), )); } } { upload_futures.push(upload_file_to_bucket( version_path, serde_json::to_vec(&version_info)?, Some("application/json".to_string()), uploaded_files_mutex.as_ref(), semaphore.clone(), )); } futures::future::try_join_all(upload_futures).await?; Ok::<(), Error>(()) } .await?; Ok::<(), Error>(()) }) } { let mut versions = version_futures.into_iter().peekable(); let mut chunk_index = 0; while versions.peek().is_some() { let now = Instant::now(); let chunk: Vec<_> = versions.by_ref().take(100).collect(); futures::future::try_join_all(chunk).await?; chunk_index += 1; let elapsed = now.elapsed(); info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); } } //futures::future::try_join_all(version_futures).await?; upload_file_to_bucket( format!( "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION ), serde_json::to_vec(&*cloned_manifest.lock().await)?, Some("application/json".to_string()), uploaded_files_mutex.as_ref(), semaphore, ) .await?; if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { uploaded_files.extend(uploaded_files_mutex.into_inner()); } let elapsed = now.elapsed(); info!("Elapsed: {:.2?}", elapsed); Ok(Arc::try_unwrap(cloned_manifest) .map_err(|_| Error::ArcError)? .into_inner()) } #[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// A version of the fabric loader struct LibraryPatch { #[serde(rename = "_comment")] pub _comment: String, #[serde(rename = "match")] pub match_: Vec, pub additional_libraries: Option>, #[serde(rename = "override")] pub override_: Option, pub patch_additional_libraries: Option, } /// Fetches the list of fabric versions fn fetch_library_patches() -> Result, Error> { let patches = include_bytes!("../library-patches.json"); Ok(serde_json::from_slice(patches)?) }