tomasalias/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Merge pull request #6 from modrinth/quilt-m1-support

Browse Source 
Support for ARM + Quilt
This commit is contained in:
Geometrically
2023-04-26 12:05:46 -07:00
committed by GitHub
parent fd19bb7cd5 3afda71349
commit 6512dbae1c
10 changed files with 577 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
LICENSE
View File

@@ -1,4 +1,4 @@
Copyright © 2022 Rinth, Inc.
Copyright © 2023 Rinth, Inc.
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:

2
daedalus/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "daedalus"
version = "0.1.19"
version = "0.1.20"
authors = ["Jai A <jaiagr+gpg@pm.me>"]
edition = "2018"
license = "MIT"

81
daedalus/src/minecraft.rs
View File

@@ -149,11 +149,12 @@ pub struct Download {
}
#[cfg_attr(feature = "bincode", derive(Encode, Decode))]
#[derive(Serialize, Deserialize, Debug)]
#[derive(Serialize, Deserialize, Debug, Clone)]
/// Download information of a library
pub struct LibraryDownload {
#[serde(skip_serializing_if = "Option::is_none")]
/// The path that the library should be saved to
pub path: String,
pub path: Option<String>,
/// The SHA1 hash of the library
pub sha1: String,
/// The size of the library
@@ -163,7 +164,7 @@ pub struct LibraryDownload {
}
#[cfg_attr(feature = "bincode", derive(Encode, Decode))]
#[derive(Serialize, Deserialize, Debug)]
#[derive(Serialize, Deserialize, Debug, Clone)]
/// A list of files that should be downloaded for libraries
pub struct LibraryDownloads {
#[serde(skip_serializing_if = "Option::is_none")]
@@ -188,15 +189,23 @@ pub enum RuleAction {
#[cfg_attr(feature = "bincode", derive(Encode, Decode))]
#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)]
#[serde(rename_all = "snake_case")]
#[serde(rename_all = "kebab-case")]
/// An enum representing the different types of operating systems
pub enum Os {
/// MacOS
/// MacOS (x86)
Osx,
/// Windows
/// M1-Based Macs
OsxArm64,
/// Windows (x86)
Windows,
/// Linux and its derivatives
/// Windows ARM
WindowsArm64,
/// Linux (x86) and its derivatives
Linux,
/// Linux ARM 64
LinuxArm64,
/// Linux ARM 32
LinuxArm32,
/// The OS is unknown
Unknown,
}
@@ -243,7 +252,7 @@ pub struct Rule {
}
#[cfg_attr(feature = "bincode", derive(Encode, Decode))]
#[derive(Serialize, Deserialize, Debug)]
#[derive(Serialize, Deserialize, Debug, Clone)]
/// Information delegating the extraction of the library
pub struct LibraryExtract {
#[serde(skip_serializing_if = "Option::is_none")]
@@ -263,7 +272,7 @@ pub struct JavaVersion {
}
#[cfg_attr(feature = "bincode", derive(Encode, Decode))]
#[derive(Serialize, Deserialize, Debug)]
#[derive(Serialize, Deserialize, Debug, Clone)]
/// A library which the game relies on to run
pub struct Library {
#[serde(skip_serializing_if = "Option::is_none")]
@@ -291,6 +300,60 @@ pub struct Library {
pub include_in_classpath: bool,
}
#[derive(Deserialize, Debug, Clone)]
/// A partial library which should be merged with a full library
pub struct PartialLibrary {
/// The files the library has
pub downloads: Option<LibraryDownloads>,
/// Rules of the extraction of the file
pub extract: Option<LibraryExtract>,
/// The maven name of the library. The format is `groupId:artifactId:version`
pub name: Option<String>,
/// The URL to the repository where the library can be downloaded
pub url: Option<String>,
/// Native files that the library relies on
pub natives: Option<HashMap<Os, String>>,
/// Rules deciding whether the library should be downloaded or not
pub rules: Option<Vec<Rule>>,
/// SHA1 Checksums for validating the library's integrity. Only present for forge libraries
pub checksums: Option<Vec<String>>,
/// Whether the library should be included in the classpath at the game's launch
pub include_in_classpath: Option<bool>,
}
/// Merges a partial library to make a complete library
pub fn merge_partial_library(
partial: PartialLibrary,
mut merge: Library,
) -> Library {
if let Some(downloads) = partial.downloads {
merge.downloads = Some(downloads)
}
if let Some(extract) = partial.extract {
merge.extract = Some(extract)
}
if let Some(name) = partial.name {
merge.name = name
}
if let Some(url) = partial.url {
merge.url = Some(url)
}
if let Some(natives) = partial.natives {
merge.natives = Some(natives)
}
if let Some(rules) = partial.rules {
merge.rules = Some(rules)
}
if let Some(checksums) = partial.checksums {
merge.checksums = Some(checksums)
}
if let Some(include_in_classpath) = partial.include_in_classpath {
merge.include_in_classpath = include_in_classpath
}
merge
}
fn default_include_in_classpath() -> bool {
true
}

2
daedalus/src/modded.rs
View File

@@ -14,6 +14,8 @@ use bincode::{Decode, Encode};
pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0;
/// The latest version of the format the fabric model structs deserialize to
pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0;
/// The latest version of the format the quilt model structs deserialize to
pub const CURRENT_QUILT_FORMAT_VERSION: usize = 0;
/// The dummy replace string library names, inheritsFrom, and version names should be replaced with
pub const DUMMY_REPLACE_STRING: &str = "${modrinth.gameVersion}";

2
daedalus_client/Cargo.toml
View File

@@ -1,6 +1,6 @@
[package]
name = "daedalus_client"
version = "0.1.19"
version = "0.1.20"
authors = ["Jai A <jaiagr+gpg@pm.me>"]
edition = "2018"

31
daedalus_client/src/fabric.rs
View File

@@ -215,18 +215,29 @@ pub async fn retrieve_data(
))
.await?;
versions.push(Version {
id: DUMMY_REPLACE_STRING.to_string(),
stable: true,
loaders: loader_version_mutex.into_inner(),
});
let mut loader_version_mutex = loader_version_mutex.into_inner();
if !loader_version_mutex.is_empty() {
if let Some(version) =
versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING)
{
version.loaders.append(&mut loader_version_mutex);
} else {
versions.push(Version {
id: DUMMY_REPLACE_STRING.to_string(),
stable: true,
loaders: loader_version_mutex,
});
}
}
for version in &list.game {
versions.push(Version {
id: version.version.clone(),
stable: version.stable,
loaders: vec![],
});
if !versions.iter().any(|x| x.id == version.version) {
versions.push(Version {
id: version.version.clone(),
stable: version.stable,
loaders: vec![],
});
}
}
versions.sort_by(|x, y| {

17
daedalus_client/src/forge.rs
View File

@@ -105,9 +105,16 @@ pub async fn retrieve_data(
async move {
/// These forge versions are not worth supporting!
const WHITELIST : [&str; 1] = [
const WHITELIST : &[&str] = &[
// Not supported due to `data` field being `[]` even though the type is a map
"1.12.2-14.23.5.2851",
// Malformed Archives
"1.6.1-8.9.0.749",
"1.6.1-8.9.0.751",
"1.6.4-9.11.1.960",
"1.6.4-9.11.1.961",
"1.6.4-9.11.1.963",
"1.6.4-9.11.1.964",
];
if WHITELIST.contains(&&*loader_version_full) {
@@ -459,19 +466,20 @@ pub async fn retrieve_data(
});
{
let len = loaders_futures.len();
let mut versions = loaders_futures.into_iter().peekable();
let mut chunk_index = 0;
while versions.peek().is_some() {
let now = Instant::now();
let chunk: Vec<_> = versions.by_ref().take(10).collect();
let chunk: Vec<_> = versions.by_ref().take(1).collect();
let res = futures::future::try_join_all(chunk).await?;
loaders_versions.extend(res.into_iter().flatten());
chunk_index += 1;
let elapsed = now.elapsed();
info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
info!("Loader Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed);
}
}
//futures::future::try_join_all(loaders_futures).await?;
@@ -489,6 +497,7 @@ pub async fn retrieve_data(
}
{
let len = version_futures.len();
let mut versions = version_futures.into_iter().peekable();
let mut chunk_index = 0;
while versions.peek().is_some() {
@@ -500,7 +509,7 @@ pub async fn retrieve_data(
chunk_index += 1;
let elapsed = now.elapsed();
info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
info!("Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed);
}
}
//futures::future::try_join_all(version_futures).await?;

13
daedalus_client/src/main.rs
View File

@@ -9,6 +9,7 @@ use tokio::sync::Semaphore;
mod fabric;
mod forge;
mod minecraft;
mod quilt;
#[derive(thiserror::Error, Debug)]
pub enum Error {
@@ -45,7 +46,7 @@ async fn main() {
}
let mut timer = tokio::time::interval(Duration::from_secs(60 * 60));
let semaphore = Arc::new(Semaphore::new(50));
let semaphore = Arc::new(Semaphore::new(10));
loop {
timer.tick().await;
@@ -87,6 +88,16 @@ async fn main() {
Ok(..) => {}
Err(err) => error!("{:?}", err),
};
match quilt::retrieve_data(
&manifest,
&mut uploaded_files,
semaphore.clone(),
)
.await
{
Ok(..) => {}
Err(err) => error!("{:?}", err),
};
}
}
}

92
daedalus_client/src/minecraft.rs
View File

@@ -1,7 +1,11 @@
use crate::download_file;
use crate::{format_url, upload_file_to_bucket, Error};
use daedalus::minecraft::VersionManifest;
use daedalus::get_hash;
use daedalus::minecraft::{
merge_partial_library, Library, PartialLibrary, VersionManifest,
};
use log::info;
use serde::Deserialize;
use std::sync::Arc;
use std::time::Instant;
use tokio::sync::{Mutex, Semaphore};
@@ -23,6 +27,9 @@ pub async fn retrieve_data(
daedalus::minecraft::fetch_version_manifest(None).await?;
let cloned_manifest = Arc::new(Mutex::new(manifest.clone()));
let patches = fetch_library_patches(None, semaphore.clone()).await?;
let cloned_patches = Arc::new(&patches);
let visited_assets_mutex = Arc::new(Mutex::new(Vec::new()));
let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new()));
@@ -48,6 +55,7 @@ pub async fn retrieve_data(
let cloned_manifest_mutex = Arc::clone(&cloned_manifest);
let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex);
let semaphore = Arc::clone(&semaphore);
let patches = Arc::clone(&cloned_patches);
let assets_hash =
old_version.and_then(|x| x.assets_index_sha1.clone());
@@ -55,9 +63,59 @@ pub async fn retrieve_data(
async move {
let mut upload_futures = Vec::new();
let version_info =
let mut version_info =
daedalus::minecraft::fetch_version_info(version).await?;
fn patch_library(patches: &Vec<LibraryPatch>, mut library: Library) -> Vec<Library> {
let mut val = Vec::new();
let actual_patches = patches
.iter()
.filter(|x| x.match_.contains(&library.name))
.collect::<Vec<_>>();
if !actual_patches.is_empty()
{
for patch in actual_patches {
if let Some(additional_libraries) =
&patch.additional_libraries
{
for additional_library in additional_libraries {
if patch.patch_additional_libraries.unwrap_or(false) {
let mut libs = patch_library(patches, additional_library.clone());
val.append(&mut libs)
} else {
val.push(additional_library.clone());
}
}
} else if let Some(override_) = &patch.override_ {
library = merge_partial_library(
override_.clone(),
library,
);
}
}
val.push(library);
} else {
val.push(library);
}
val
}
let mut new_libraries = Vec::new();
for library in version_info.libraries {
let mut libs = patch_library(&patches, library);
new_libraries.append(&mut libs)
}
version_info.libraries = new_libraries;
let version_info_hash = get_hash(bytes::Bytes::from(
serde_json::to_vec(&version_info)?,
))
.await?;
let version_path = format!(
"minecraft/v{}/versions/{}.json",
daedalus::minecraft::CURRENT_FORMAT_VERSION,
@@ -85,6 +143,7 @@ pub async fn retrieve_data(
Some(version_info.asset_index.sha1.clone());
cloned_manifest.versions[position].assets_index_url =
Some(format_url(&assets_path));
cloned_manifest.versions[position].sha1 = version_info_hash;
}
let mut download_assets = false;
@@ -187,3 +246,32 @@ pub async fn retrieve_data(
.map_err(|_| Error::ArcError)?
.into_inner())
}
#[derive(Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
/// A version of the fabric loader
struct LibraryPatch {
#[serde(rename = "_comment")]
pub _comment: String,
#[serde(rename = "match")]
pub match_: Vec<String>,
pub additional_libraries: Option<Vec<Library>>,
#[serde(rename = "override")]
pub override_: Option<PartialLibrary>,
pub patch_additional_libraries: Option<bool>,
}
/// Fetches the list of fabric versions
async fn fetch_library_patches(
url: Option<&str>,
semaphore: Arc<Semaphore>,
) -> Result<Vec<LibraryPatch>, Error> {
Ok(serde_json::from_slice(
&download_file(
url.unwrap_or(&format_url("library-patches.json")),
None,
semaphore,
)
.await?,
)?)
}

364
daedalus_client/src/quilt.rs Normal file
View File

@@ -0,0 +1,364 @@
use crate::{download_file, format_url, upload_file_to_bucket, Error};
use daedalus::minecraft::{Library, VersionManifest};
use daedalus::modded::{
LoaderVersion, Manifest, PartialVersionInfo, Version, DUMMY_REPLACE_STRING,
};
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use tokio::sync::{Mutex, RwLock, Semaphore};
pub async fn retrieve_data(
minecraft_versions: &VersionManifest,
uploaded_files: &mut Vec<String>,
semaphore: Arc<Semaphore>,
) -> Result<(), Error> {
let mut list = fetch_quilt_versions(None, semaphore.clone()).await?;
let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!(
"quilt/v{}/manifest.json",
daedalus::modded::CURRENT_QUILT_FORMAT_VERSION,
)))
.await
.ok();
let mut versions = if let Some(old_manifest) = old_manifest {
old_manifest.game_versions
} else {
Vec::new()
};
let loaders_mutex = RwLock::new(Vec::new());
{
let mut loaders = loaders_mutex.write().await;
for loader in &list.loader {
loaders.push((Box::new(false), loader.version.clone()))
}
list.loader
.retain(|x| loaders.iter().any(|val| val.1 == x.version))
}
const DUMMY_GAME_VERSION: &str = "1.19.4-rc2";
let loader_version_mutex = Mutex::new(Vec::new());
let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new()));
let loader_versions = futures::future::try_join_all(
loaders_mutex.read().await.clone().into_iter().map(
|(stable, loader)| async {
{
if versions.iter().any(|x| {
x.id == DUMMY_REPLACE_STRING
&& x.loaders.iter().any(|x| x.id == loader)
}) {
return Ok(None);
}
}
let version = fetch_quilt_version(
DUMMY_GAME_VERSION,
&loader,
semaphore.clone(),
)
.await?;
Ok::<Option<(Box<bool>, String, PartialVersionInfo)>, Error>(
Some((stable, loader, version)),
)
},
),
)
.await?;
let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new()));
futures::future::try_join_all(loader_versions.into_iter()
.flatten().map(
|(stable, loader, version)| async {
let libs = futures::future::try_join_all(
version.libraries.into_iter().map(|mut lib| async {
{
let mut visited_assets =
visited_artifacts_mutex.lock().await;
if visited_assets.contains(&lib.name) {
lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING);
lib.url = Some(format_url("maven/"));
return Ok(lib);
} else {
visited_assets.push(lib.name.clone())
}
}
if lib.name.contains(DUMMY_GAME_VERSION) {
lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING);
futures::future::try_join_all(list.game.clone().into_iter().map(|game_version| async {
let semaphore = semaphore.clone();
let uploaded_files_mutex = uploaded_files_mutex.clone();
let lib_name = lib.name.clone();
let lib_url = lib.url.clone();
async move {
let artifact_path =
daedalus::get_path_from_artifact(&lib_name.replace(DUMMY_REPLACE_STRING, &game_version.version))?;
let artifact = download_file(
&format!(
"{}{}",
lib_url.unwrap_or_else(|| {
"https://maven.quiltmc.org/".to_string()
}),
artifact_path
),
None,
semaphore.clone(),
)
.await?;
upload_file_to_bucket(
format!("{}/{}", "maven", artifact_path),
artifact.to_vec(),
Some("application/java-archive".to_string()),
&uploaded_files_mutex,
semaphore.clone(),
)
.await?;
Ok::<(), Error>(())
}.await?;
Ok::<(), Error>(())
})).await?;
lib.url = Some(format_url("maven/"));
return Ok(lib);
}
let artifact_path =
daedalus::get_path_from_artifact(&lib.name)?;
let artifact = download_file(
&format!(
"{}{}",
lib.url.unwrap_or_else(|| {
"https://maven.quiltmc.org/".to_string()
}),
artifact_path
),
None,
semaphore.clone(),
)
.await?;
lib.url = Some(format_url("maven/"));
upload_file_to_bucket(
format!("{}/{}", "maven", artifact_path),
artifact.to_vec(),
Some("application/java-archive".to_string()),
&uploaded_files_mutex,
semaphore.clone(),
)
.await?;
Ok::<Library, Error>(lib)
}),
)
.await?;
let version_path = format!(
"quilt/v{}/versions/{}.json",
daedalus::modded::CURRENT_QUILT_FORMAT_VERSION,
&loader
);
upload_file_to_bucket(
version_path.clone(),
serde_json::to_vec(&PartialVersionInfo {
arguments: version.arguments,
id: version
.id
.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING),
main_class: version.main_class,
release_time: version.release_time,
time: version.time,
type_: version.type_,
inherits_from: version
.inherits_from
.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING),
libraries: libs,
minecraft_arguments: version.minecraft_arguments,
processors: None,
data: None,
})?,
Some("application/json".to_string()),
&uploaded_files_mutex,
semaphore.clone(),
)
.await?;
{
let mut loader_version_map = loader_version_mutex.lock().await;
async move {
loader_version_map.push(LoaderVersion {
id: loader.to_string(),
url: format_url(&version_path),
stable: *stable,
});
}
.await;
}
Ok::<(), Error>(())
},
))
.await?;
let mut loader_version_mutex = loader_version_mutex.into_inner();
if !loader_version_mutex.is_empty() {
if let Some(version) =
versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING)
{
version.loaders.append(&mut loader_version_mutex);
} else {
versions.push(Version {
id: DUMMY_REPLACE_STRING.to_string(),
stable: true,
loaders: loader_version_mutex,
});
}
}
for version in &list.game {
if !versions.iter().any(|x| x.id == version.version) {
versions.push(Version {
id: version.version.clone(),
stable: version.stable,
loaders: vec![],
});
}
}
versions.sort_by(|x, y| {
minecraft_versions
.versions
.iter()
.position(|z| x.id == z.id)
.unwrap_or_default()
.cmp(
&minecraft_versions
.versions
.iter()
.position(|z| y.id == z.id)
.unwrap_or_default(),
)
});
for version in &mut versions {
version.loaders.sort_by(|x, y| {
list.loader
.iter()
.position(|z| {
x.id.split('-').next().unwrap_or_default() == &*z.version
})
.unwrap_or_default()
.cmp(
&list
.loader
.iter()
.position(|z| {
y.id.split('-').next().unwrap_or_default()
== z.version
})
.unwrap_or_default(),
)
})
}
upload_file_to_bucket(
format!(
"quilt/v{}/manifest.json",
daedalus::modded::CURRENT_QUILT_FORMAT_VERSION,
),
serde_json::to_vec(&Manifest {
game_versions: versions,
})?,
Some("application/json".to_string()),
&uploaded_files_mutex,
semaphore,
)
.await?;
if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) {
uploaded_files.extend(uploaded_files_mutex.into_inner());
}
Ok(())
}
const QUILT_META_URL: &str = "https://meta.quiltmc.org/v3";
async fn fetch_quilt_version(
version_number: &str,
loader_version: &str,
semaphore: Arc<Semaphore>,
) -> Result<PartialVersionInfo, Error> {
Ok(serde_json::from_slice(
&download_file(
&format!(
"{}/versions/loader/{}/{}/profile/json",
QUILT_META_URL, version_number, loader_version
),
None,
semaphore,
)
.await?,
)?)
}
#[derive(Serialize, Deserialize, Debug, Clone)]
/// Versions of quilt components
struct QuiltVersions {
/// Versions of Minecraft that quilt supports
pub game: Vec<QuiltGameVersion>,
/// Available versions of the quilt loader
pub loader: Vec<QuiltLoaderVersion>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
/// A version of Minecraft that quilt supports
struct QuiltGameVersion {
/// The version number of the game
pub version: String,
/// Whether the Minecraft version is stable or not
pub stable: bool,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
/// A version of the quilt loader
struct QuiltLoaderVersion {
/// The separator to get the build number
pub separator: String,
/// The build number
pub build: u32,
/// The maven artifact
pub maven: String,
/// The version number of the quilt loader
pub version: String,
}
/// Fetches the list of quilt versions
async fn fetch_quilt_versions(
url: Option<&str>,
semaphore: Arc<Semaphore>,
) -> Result<QuiltVersions, Error> {
Ok(serde_json::from_slice(
&download_file(
url.unwrap_or(&*format!("{}/versions", QUILT_META_URL)),
None,
semaphore,
)
.await?,
)?)
}