From 89e56ae279392f41abfc8e35658099449071a3e6 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 25 Apr 2023 19:36:21 -0700 Subject: [PATCH] Support for ARM + Quilt --- LICENSE | 2 +- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 82 ++++++- daedalus/src/modded.rs | 2 + daedalus_client/Cargo.toml | 2 +- daedalus_client/src/fabric.rs | 29 ++- daedalus_client/src/forge.rs | 17 +- daedalus_client/src/main.rs | 13 +- daedalus_client/src/minecraft.rs | 61 +++++- daedalus_client/src/quilt.rs | 362 +++++++++++++++++++++++++++++++ 10 files changed, 542 insertions(+), 30 deletions(-) create mode 100644 daedalus_client/src/quilt.rs diff --git a/LICENSE b/LICENSE index 1bc38c7a1..75e4f7799 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright © 2022 Rinth, Inc. +Copyright © 2023 Rinth, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 09c54758c..94cbde710 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.19" +version = "0.1.20" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 247274a54..49c87fe4c 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -149,11 +149,12 @@ pub struct Download { } #[cfg_attr(feature = "bincode", derive(Encode, Decode))] -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// Download information of a library pub struct LibraryDownload { + #[serde(skip_serializing_if = "Option::is_none")] /// The path that the library should be saved to - pub path: String, + pub path: Option, /// The SHA1 hash of the library pub sha1: String, /// The size of the library @@ -163,7 +164,7 @@ pub struct LibraryDownload { } #[cfg_attr(feature = "bincode", derive(Encode, Decode))] -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// A list of files that should be downloaded for libraries pub struct LibraryDownloads { #[serde(skip_serializing_if = "Option::is_none")] @@ -188,17 +189,25 @@ pub enum RuleAction { #[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] -#[serde(rename_all = "snake_case")] +#[serde(rename_all = "kebab-case")] /// An enum representing the different types of operating systems pub enum Os { - /// MacOS + /// MacOS (x86) Osx, - /// Windows + /// M1-Based Macs + OsxArm64, + /// Windows (x86) Windows, - /// Linux and its derivatives + /// Windows ARM + WindowsArm64, + /// Linux (x86) and its derivatives Linux, + /// Linux ARM 64 + LinuxArm64, + /// Linux ARM 32 + LinuxArm32, /// The OS is unknown - Unknown, + Unknown } #[cfg_attr(feature = "bincode", derive(Encode, Decode))] @@ -243,7 +252,7 @@ pub struct Rule { } #[cfg_attr(feature = "bincode", derive(Encode, Decode))] -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// Information delegating the extraction of the library pub struct LibraryExtract { #[serde(skip_serializing_if = "Option::is_none")] @@ -263,7 +272,7 @@ pub struct JavaVersion { } #[cfg_attr(feature = "bincode", derive(Encode, Decode))] -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// A library which the game relies on to run pub struct Library { #[serde(skip_serializing_if = "Option::is_none")] @@ -291,6 +300,59 @@ pub struct Library { pub include_in_classpath: bool, } +#[derive(Deserialize, Debug, Clone)] +/// A partial library which should be merged with a full library +pub struct PartialLibrary { + /// The files the library has + pub downloads: Option, + /// Rules of the extraction of the file + pub extract: Option, + /// The maven name of the library. The format is `groupId:artifactId:version` + pub name: Option, + /// The URL to the repository where the library can be downloaded + pub url: Option, + /// Native files that the library relies on + pub natives: Option>, + /// Rules deciding whether the library should be downloaded or not + pub rules: Option>, + /// SHA1 Checksums for validating the library's integrity. Only present for forge libraries + pub checksums: Option>, + /// Whether the library should be included in the classpath at the game's launch + pub include_in_classpath: Option, +} + +pub fn merge_partial_library( + partial: PartialLibrary, + mut merge: Library, +) -> Library { + if let Some(downloads) = partial.downloads { + merge.downloads = Some(downloads) + } + if let Some(extract) = partial.extract { + merge.extract = Some(extract) + } + if let Some(name) = partial.name { + merge.name = name + } + if let Some(url) = partial.url { + merge.url = Some(url) + } + if let Some(natives) = partial.natives { + merge.natives = Some(natives) + } + if let Some(rules) = partial.rules { + merge.rules = Some(rules) + } + if let Some(checksums) = partial.checksums { + merge.checksums = Some(checksums) + } + if let Some(include_in_classpath) = partial.include_in_classpath { + merge.include_in_classpath = include_in_classpath + } + + merge +} + fn default_include_in_classpath() -> bool { true } diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 2519719c8..8f91798fc 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -14,6 +14,8 @@ use bincode::{Decode, Encode}; pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0; /// The latest version of the format the fabric model structs deserialize to pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0; +/// The latest version of the format the quilt model structs deserialize to +pub const CURRENT_QUILT_FORMAT_VERSION: usize = 0; /// The dummy replace string library names, inheritsFrom, and version names should be replaced with pub const DUMMY_REPLACE_STRING: &str = "${modrinth.gameVersion}"; diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 8b531712c..42b93a80b 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.19" +version = "0.1.20" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 6aa8f91fe..d57cfd851 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -215,18 +215,27 @@ pub async fn retrieve_data( )) .await?; - versions.push(Version { - id: DUMMY_REPLACE_STRING.to_string(), - stable: true, - loaders: loader_version_mutex.into_inner(), - }); + let mut loader_version_mutex = loader_version_mutex.into_inner(); + if !loader_version_mutex.is_empty() { + if let Some(version) = versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) { + version.loaders.append(&mut loader_version_mutex); + } else { + versions.push(Version { + id: DUMMY_REPLACE_STRING.to_string(), + stable: true, + loaders: loader_version_mutex, + }); + } + } for version in &list.game { - versions.push(Version { - id: version.version.clone(), - stable: version.stable, - loaders: vec![], - }); + if !versions.iter().any(|x| x.id == version.version) { + versions.push(Version { + id: version.version.clone(), + stable: version.stable, + loaders: vec![], + }); + } } versions.sort_by(|x, y| { diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 7415cc651..fc3bf2f76 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -105,9 +105,16 @@ pub async fn retrieve_data( async move { /// These forge versions are not worth supporting! - const WHITELIST : [&str; 1] = [ + const WHITELIST : &[&str] = &[ // Not supported due to `data` field being `[]` even though the type is a map "1.12.2-14.23.5.2851", + // Malformed Archives + "1.6.1-8.9.0.749", + "1.6.1-8.9.0.751", + "1.6.4-9.11.1.960", + "1.6.4-9.11.1.961", + "1.6.4-9.11.1.963", + "1.6.4-9.11.1.964", ]; if WHITELIST.contains(&&*loader_version_full) { @@ -459,19 +466,20 @@ pub async fn retrieve_data( }); { + let len = loaders_futures.len(); let mut versions = loaders_futures.into_iter().peekable(); let mut chunk_index = 0; while versions.peek().is_some() { let now = Instant::now(); - let chunk: Vec<_> = versions.by_ref().take(10).collect(); + let chunk: Vec<_> = versions.by_ref().take(1).collect(); let res = futures::future::try_join_all(chunk).await?; loaders_versions.extend(res.into_iter().flatten()); chunk_index += 1; let elapsed = now.elapsed(); - info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + info!("Loader Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed); } } //futures::future::try_join_all(loaders_futures).await?; @@ -489,6 +497,7 @@ pub async fn retrieve_data( } { + let len = version_futures.len(); let mut versions = version_futures.into_iter().peekable(); let mut chunk_index = 0; while versions.peek().is_some() { @@ -500,7 +509,7 @@ pub async fn retrieve_data( chunk_index += 1; let elapsed = now.elapsed(); - info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + info!("Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed); } } //futures::future::try_join_all(version_futures).await?; diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 92fee4cd7..c4bb24719 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -9,6 +9,7 @@ use tokio::sync::Semaphore; mod fabric; mod forge; mod minecraft; +mod quilt; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -45,7 +46,7 @@ async fn main() { } let mut timer = tokio::time::interval(Duration::from_secs(60 * 60)); - let semaphore = Arc::new(Semaphore::new(50)); + let semaphore = Arc::new(Semaphore::new(10)); loop { timer.tick().await; @@ -87,6 +88,16 @@ async fn main() { Ok(..) => {} Err(err) => error!("{:?}", err), }; + match quilt::retrieve_data( + &manifest, + &mut uploaded_files, + semaphore.clone(), + ) + .await + { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; } } } diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 024ce3d05..282fd9346 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,10 +1,12 @@ use crate::download_file; use crate::{format_url, upload_file_to_bucket, Error}; -use daedalus::minecraft::VersionManifest; +use daedalus::minecraft::{Library, merge_partial_library, PartialLibrary, VersionManifest}; use log::info; use std::sync::Arc; use std::time::Instant; use tokio::sync::{Mutex, Semaphore}; +use serde::Deserialize; +use daedalus::get_hash; pub async fn retrieve_data( uploaded_files: &mut Vec, @@ -23,6 +25,9 @@ pub async fn retrieve_data( daedalus::minecraft::fetch_version_manifest(None).await?; let cloned_manifest = Arc::new(Mutex::new(manifest.clone())); + let patches = fetch_library_patches(None, semaphore.clone()).await?; + let cloned_patches = Arc::new(&patches); + let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); @@ -48,6 +53,7 @@ pub async fn retrieve_data( let cloned_manifest_mutex = Arc::clone(&cloned_manifest); let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); let semaphore = Arc::clone(&semaphore); + let patches = Arc::clone(&cloned_patches); let assets_hash = old_version.and_then(|x| x.assets_index_sha1.clone()); @@ -55,9 +61,30 @@ pub async fn retrieve_data( async move { let mut upload_futures = Vec::new(); - let version_info = + let mut version_info = daedalus::minecraft::fetch_version_info(version).await?; + let mut new_libraries = Vec::new(); + for library in version_info.libraries { + if let Some(patch) = patches.iter().find(|x| x.match_.contains(&library.name)) { + if let Some(additional_libraries) = &patch.additional_libraries { + new_libraries.push(library); + for additional_library in additional_libraries { + new_libraries.push(additional_library.clone()); + } + } else if let Some(override_) = &patch.override_ { + new_libraries.push(merge_partial_library(override_.clone(), library)); + } else { + new_libraries.push(library); + } + } else { + new_libraries.push(library); + } + } + version_info.libraries = new_libraries; + + let version_info_hash = get_hash(bytes::Bytes::from(serde_json::to_vec(&version_info)?)).await?; + let version_path = format!( "minecraft/v{}/versions/{}.json", daedalus::minecraft::CURRENT_FORMAT_VERSION, @@ -85,6 +112,7 @@ pub async fn retrieve_data( Some(version_info.asset_index.sha1.clone()); cloned_manifest.versions[position].assets_index_url = Some(format_url(&assets_path)); + cloned_manifest.versions[position].sha1 = version_info_hash; } let mut download_assets = false; @@ -187,3 +215,32 @@ pub async fn retrieve_data( .map_err(|_| Error::ArcError)? .into_inner()) } + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +/// A version of the fabric loader +struct LibraryPatch { + #[serde(rename = "_comment")] + pub _comment: String, + #[serde(rename = "match")] + pub match_: Vec, + pub additional_libraries: Option>, + #[serde(rename = "override")] + pub override_: Option, + pub patch_additional_libraries: Option, +} + +/// Fetches the list of fabric versions +async fn fetch_library_patches( + url: Option<&str>, + semaphore: Arc, +) -> Result, Error> { + Ok(serde_json::from_slice( + &download_file( + url.unwrap_or(&format_url("library-patches.json")), + None, + semaphore, + ) + .await?, + )?) +} diff --git a/daedalus_client/src/quilt.rs b/daedalus_client/src/quilt.rs new file mode 100644 index 000000000..9b122967b --- /dev/null +++ b/daedalus_client/src/quilt.rs @@ -0,0 +1,362 @@ +use crate::{download_file, format_url, upload_file_to_bucket, Error}; +use daedalus::minecraft::{Library, VersionManifest}; +use daedalus::modded::{ + LoaderVersion, Manifest, PartialVersionInfo, Version, DUMMY_REPLACE_STRING, +}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use tokio::sync::{Mutex, RwLock, Semaphore}; + +pub async fn retrieve_data( + minecraft_versions: &VersionManifest, + uploaded_files: &mut Vec, + semaphore: Arc, +) -> Result<(), Error> { + let mut list = fetch_quilt_versions(None, semaphore.clone()).await?; + let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( + "quilt/v{}/manifest.json", + daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, + ))) + .await + .ok(); + + let mut versions = if let Some(old_manifest) = old_manifest { + old_manifest.game_versions + } else { + Vec::new() + }; + + let loaders_mutex = RwLock::new(Vec::new()); + + { + let mut loaders = loaders_mutex.write().await; + + for loader in &list.loader { + loaders.push((Box::new(false), loader.version.clone())) + } + + list.loader + .retain(|x| loaders.iter().any(|val| val.1 == x.version)) + } + + const DUMMY_GAME_VERSION: &str = "1.19.4-rc2"; + + let loader_version_mutex = Mutex::new(Vec::new()); + let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); + + let loader_versions = futures::future::try_join_all( + loaders_mutex.read().await.clone().into_iter().map( + |(stable, loader)| async { + { + if versions.iter().any(|x| { + x.id == DUMMY_REPLACE_STRING + && x.loaders.iter().any(|x| x.id == loader) + }) { + return Ok(None); + } + } + + let version = fetch_quilt_version( + DUMMY_GAME_VERSION, + &loader, + semaphore.clone(), + ) + .await?; + + Ok::, String, PartialVersionInfo)>, Error>( + Some((stable, loader, version)), + ) + }, + ), + ) + .await?; + + let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); + futures::future::try_join_all(loader_versions.into_iter() + .flatten().map( + |(stable, loader, version)| async { + let libs = futures::future::try_join_all( + version.libraries.into_iter().map(|mut lib| async { + { + let mut visited_assets = + visited_artifacts_mutex.lock().await; + + if visited_assets.contains(&lib.name) { + lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); + lib.url = Some(format_url("maven/")); + + return Ok(lib); + } else { + visited_assets.push(lib.name.clone()) + } + } + + if lib.name.contains(DUMMY_GAME_VERSION) { + lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); + futures::future::try_join_all(list.game.clone().into_iter().map(|game_version| async { + let semaphore = semaphore.clone(); + let uploaded_files_mutex = uploaded_files_mutex.clone(); + let lib_name = lib.name.clone(); + let lib_url = lib.url.clone(); + + async move { + let artifact_path = + daedalus::get_path_from_artifact(&lib_name.replace(DUMMY_REPLACE_STRING, &game_version.version))?; + + let artifact = download_file( + &format!( + "{}{}", + lib_url.unwrap_or_else(|| { + "https://maven.quiltmc.org/".to_string() + }), + artifact_path + ), + None, + semaphore.clone(), + ) + .await?; + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + &uploaded_files_mutex, + semaphore.clone(), + ) + .await?; + + Ok::<(), Error>(()) + }.await?; + + Ok::<(), Error>(()) + })).await?; + lib.url = Some(format_url("maven/")); + + return Ok(lib); + } + + let artifact_path = + daedalus::get_path_from_artifact(&lib.name)?; + + let artifact = download_file( + &format!( + "{}{}", + lib.url.unwrap_or_else(|| { + "https://maven.quiltmc.org/".to_string() + }), + artifact_path + ), + None, + semaphore.clone(), + ) + .await?; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + &uploaded_files_mutex, + semaphore.clone(), + ) + .await?; + + Ok::(lib) + }), + ) + .await?; + + let version_path = format!( + "quilt/v{}/versions/{}.json", + daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, + &loader + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&PartialVersionInfo { + arguments: version.arguments, + id: version + .id + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), + main_class: version.main_class, + release_time: version.release_time, + time: version.time, + type_: version.type_, + inherits_from: version + .inherits_from + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), + libraries: libs, + minecraft_arguments: version.minecraft_arguments, + processors: None, + data: None, + })?, + Some("application/json".to_string()), + &uploaded_files_mutex, + semaphore.clone(), + ) + .await?; + + { + let mut loader_version_map = loader_version_mutex.lock().await; + async move { + loader_version_map.push(LoaderVersion { + id: loader.to_string(), + url: format_url(&version_path), + stable: *stable, + }); + } + .await; + } + + Ok::<(), Error>(()) + }, + )) + .await?; + + let mut loader_version_mutex = loader_version_mutex.into_inner(); + if !loader_version_mutex.is_empty() { + if let Some(version) = versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) { + version.loaders.append(&mut loader_version_mutex); + } else { + versions.push(Version { + id: DUMMY_REPLACE_STRING.to_string(), + stable: true, + loaders: loader_version_mutex, + }); + } + } + + for version in &list.game { + if !versions.iter().any(|x| x.id == version.version) { + versions.push(Version { + id: version.version.clone(), + stable: version.stable, + loaders: vec![], + }); + } + } + + versions.sort_by(|x, y| { + minecraft_versions + .versions + .iter() + .position(|z| x.id == z.id) + .unwrap_or_default() + .cmp( + &minecraft_versions + .versions + .iter() + .position(|z| y.id == z.id) + .unwrap_or_default(), + ) + }); + + for version in &mut versions { + version.loaders.sort_by(|x, y| { + list.loader + .iter() + .position(|z| { + x.id.split('-').next().unwrap_or_default() == &*z.version + }) + .unwrap_or_default() + .cmp( + &list + .loader + .iter() + .position(|z| { + y.id.split('-').next().unwrap_or_default() + == z.version + }) + .unwrap_or_default(), + ) + }) + } + + upload_file_to_bucket( + format!( + "quilt/v{}/manifest.json", + daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, + ), + serde_json::to_vec(&Manifest { + game_versions: versions, + })?, + Some("application/json".to_string()), + &uploaded_files_mutex, + semaphore, + ) + .await?; + + if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { + uploaded_files.extend(uploaded_files_mutex.into_inner()); + } + + Ok(()) +} + +const QUILT_META_URL: &str = "https://meta.quiltmc.org/v3"; + +async fn fetch_quilt_version( + version_number: &str, + loader_version: &str, + semaphore: Arc, +) -> Result { + Ok(serde_json::from_slice( + &download_file( + &format!( + "{}/versions/loader/{}/{}/profile/json", + QUILT_META_URL, version_number, loader_version + ), + None, + semaphore, + ) + .await?, + )?) +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// Versions of quilt components +struct QuiltVersions { + /// Versions of Minecraft that quilt supports + pub game: Vec, + /// Available versions of the quilt loader + pub loader: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// A version of Minecraft that quilt supports +struct QuiltGameVersion { + /// The version number of the game + pub version: String, + /// Whether the Minecraft version is stable or not + pub stable: bool, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// A version of the quilt loader +struct QuiltLoaderVersion { + /// The separator to get the build number + pub separator: String, + /// The build number + pub build: u32, + /// The maven artifact + pub maven: String, + /// The version number of the quilt loader + pub version: String, +} + +/// Fetches the list of quilt versions +async fn fetch_quilt_versions( + url: Option<&str>, + semaphore: Arc, +) -> Result { + Ok(serde_json::from_slice( + &download_file( + url.unwrap_or(&*format!("{}/versions", QUILT_META_URL)), + None, + semaphore, + ) + .await?, + )?) +}