eightequal/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Support for ARM + Quilt

Browse Source
This commit is contained in:
Jai A
2023-04-25 19:36:21 -07:00
parent fd19bb7cd5
commit 89e56ae279
10 changed files with 542 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
daedalus_client/src/fabric.rs
View File

@@ -215,18 +215,27 @@ pub async fn retrieve_data(
))
.await?;
versions.push(Version {
id: DUMMY_REPLACE_STRING.to_string(),
stable: true,
loaders: loader_version_mutex.into_inner(),
});
let mut loader_version_mutex = loader_version_mutex.into_inner();
if !loader_version_mutex.is_empty() {
if let Some(version) = versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) {
version.loaders.append(&mut loader_version_mutex);
} else {
versions.push(Version {
id: DUMMY_REPLACE_STRING.to_string(),
stable: true,
loaders: loader_version_mutex,
});
}
}
for version in &list.game {
versions.push(Version {
id: version.version.clone(),
stable: version.stable,
loaders: vec![],
});
if !versions.iter().any(|x| x.id == version.version) {
versions.push(Version {
id: version.version.clone(),
stable: version.stable,
loaders: vec![],
});
}
}
versions.sort_by(|x, y| {

17
daedalus_client/src/forge.rs
View File

@@ -105,9 +105,16 @@ pub async fn retrieve_data(
async move {
/// These forge versions are not worth supporting!
const WHITELIST : [&str; 1] = [
const WHITELIST : &[&str] = &[
// Not supported due to `data` field being `[]` even though the type is a map
"1.12.2-14.23.5.2851",
// Malformed Archives
"1.6.1-8.9.0.749",
"1.6.1-8.9.0.751",
"1.6.4-9.11.1.960",
"1.6.4-9.11.1.961",
"1.6.4-9.11.1.963",
"1.6.4-9.11.1.964",
];
if WHITELIST.contains(&&*loader_version_full) {
@@ -459,19 +466,20 @@ pub async fn retrieve_data(
});
{
let len = loaders_futures.len();
let mut versions = loaders_futures.into_iter().peekable();
let mut chunk_index = 0;
while versions.peek().is_some() {
let now = Instant::now();
let chunk: Vec<_> = versions.by_ref().take(10).collect();
let chunk: Vec<_> = versions.by_ref().take(1).collect();
let res = futures::future::try_join_all(chunk).await?;
loaders_versions.extend(res.into_iter().flatten());
chunk_index += 1;
let elapsed = now.elapsed();
info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
info!("Loader Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed);
}
}
//futures::future::try_join_all(loaders_futures).await?;
@@ -489,6 +497,7 @@ pub async fn retrieve_data(
}
{
let len = version_futures.len();
let mut versions = version_futures.into_iter().peekable();
let mut chunk_index = 0;
while versions.peek().is_some() {
@@ -500,7 +509,7 @@ pub async fn retrieve_data(
chunk_index += 1;
let elapsed = now.elapsed();
info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
info!("Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed);
}
}
//futures::future::try_join_all(version_futures).await?;

13
daedalus_client/src/main.rs
View File

@@ -9,6 +9,7 @@ use tokio::sync::Semaphore;
mod fabric;
mod forge;
mod minecraft;
mod quilt;
#[derive(thiserror::Error, Debug)]
pub enum Error {
@@ -45,7 +46,7 @@ async fn main() {
}
let mut timer = tokio::time::interval(Duration::from_secs(60 * 60));
let semaphore = Arc::new(Semaphore::new(50));
let semaphore = Arc::new(Semaphore::new(10));
loop {
timer.tick().await;
@@ -87,6 +88,16 @@ async fn main() {
Ok(..) => {}
Err(err) => error!("{:?}", err),
};
match quilt::retrieve_data(
&manifest,
&mut uploaded_files,
semaphore.clone(),
)
.await
{
Ok(..) => {}
Err(err) => error!("{:?}", err),
};
}
}
}

61
daedalus_client/src/minecraft.rs
View File

@@ -1,10 +1,12 @@
use crate::download_file;
use crate::{format_url, upload_file_to_bucket, Error};
use daedalus::minecraft::VersionManifest;
use daedalus::minecraft::{Library, merge_partial_library, PartialLibrary, VersionManifest};
use log::info;
use std::sync::Arc;
use std::time::Instant;
use tokio::sync::{Mutex, Semaphore};
use serde::Deserialize;
use daedalus::get_hash;
pub async fn retrieve_data(
uploaded_files: &mut Vec<String>,
@@ -23,6 +25,9 @@ pub async fn retrieve_data(
daedalus::minecraft::fetch_version_manifest(None).await?;
let cloned_manifest = Arc::new(Mutex::new(manifest.clone()));
let patches = fetch_library_patches(None, semaphore.clone()).await?;
let cloned_patches = Arc::new(&patches);
let visited_assets_mutex = Arc::new(Mutex::new(Vec::new()));
let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new()));
@@ -48,6 +53,7 @@ pub async fn retrieve_data(
let cloned_manifest_mutex = Arc::clone(&cloned_manifest);
let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex);
let semaphore = Arc::clone(&semaphore);
let patches = Arc::clone(&cloned_patches);
let assets_hash =
old_version.and_then(|x| x.assets_index_sha1.clone());
@@ -55,9 +61,30 @@ pub async fn retrieve_data(
async move {
let mut upload_futures = Vec::new();
let version_info =
let mut version_info =
daedalus::minecraft::fetch_version_info(version).await?;
let mut new_libraries = Vec::new();
for library in version_info.libraries {
if let Some(patch) = patches.iter().find(|x| x.match_.contains(&library.name)) {
if let Some(additional_libraries) = &patch.additional_libraries {
new_libraries.push(library);
for additional_library in additional_libraries {
new_libraries.push(additional_library.clone());
}
} else if let Some(override_) = &patch.override_ {
new_libraries.push(merge_partial_library(override_.clone(), library));
} else {
new_libraries.push(library);
}
} else {
new_libraries.push(library);
}
}
version_info.libraries = new_libraries;
let version_info_hash = get_hash(bytes::Bytes::from(serde_json::to_vec(&version_info)?)).await?;
let version_path = format!(
"minecraft/v{}/versions/{}.json",
daedalus::minecraft::CURRENT_FORMAT_VERSION,
@@ -85,6 +112,7 @@ pub async fn retrieve_data(
Some(version_info.asset_index.sha1.clone());
cloned_manifest.versions[position].assets_index_url =
Some(format_url(&assets_path));
cloned_manifest.versions[position].sha1 = version_info_hash;
}
let mut download_assets = false;
@@ -187,3 +215,32 @@ pub async fn retrieve_data(
.map_err(|_| Error::ArcError)?
.into_inner())
}
#[derive(Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
/// A version of the fabric loader
struct LibraryPatch {
#[serde(rename = "_comment")]
pub _comment: String,
#[serde(rename = "match")]
pub match_: Vec<String>,
pub additional_libraries: Option<Vec<Library>>,
#[serde(rename = "override")]
pub override_: Option<PartialLibrary>,
pub patch_additional_libraries: Option<bool>,
}
/// Fetches the list of fabric versions
async fn fetch_library_patches(
url: Option<&str>,
semaphore: Arc<Semaphore>,
) -> Result<Vec<LibraryPatch>, Error> {
Ok(serde_json::from_slice(
&download_file(
url.unwrap_or(&format_url("library-patches.json")),
None,
semaphore,
)
.await?,
)?)
}

362
daedalus_client/src/quilt.rs Normal file
View File

@@ -0,0 +1,362 @@
use crate::{download_file, format_url, upload_file_to_bucket, Error};
use daedalus::minecraft::{Library, VersionManifest};
use daedalus::modded::{
LoaderVersion, Manifest, PartialVersionInfo, Version, DUMMY_REPLACE_STRING,
};
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use tokio::sync::{Mutex, RwLock, Semaphore};
pub async fn retrieve_data(
minecraft_versions: &VersionManifest,
uploaded_files: &mut Vec<String>,
semaphore: Arc<Semaphore>,
) -> Result<(), Error> {
let mut list = fetch_quilt_versions(None, semaphore.clone()).await?;
let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!(
"quilt/v{}/manifest.json",
daedalus::modded::CURRENT_QUILT_FORMAT_VERSION,
)))
.await
.ok();
let mut versions = if let Some(old_manifest) = old_manifest {
old_manifest.game_versions
} else {
Vec::new()
};
let loaders_mutex = RwLock::new(Vec::new());
{
let mut loaders = loaders_mutex.write().await;
for loader in &list.loader {
loaders.push((Box::new(false), loader.version.clone()))
}
list.loader
.retain(|x| loaders.iter().any(|val| val.1 == x.version))
}
const DUMMY_GAME_VERSION: &str = "1.19.4-rc2";
let loader_version_mutex = Mutex::new(Vec::new());
let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new()));
let loader_versions = futures::future::try_join_all(
loaders_mutex.read().await.clone().into_iter().map(
|(stable, loader)| async {
{
if versions.iter().any(|x| {
x.id == DUMMY_REPLACE_STRING
&& x.loaders.iter().any(|x| x.id == loader)
}) {
return Ok(None);
}
}
let version = fetch_quilt_version(
DUMMY_GAME_VERSION,
&loader,
semaphore.clone(),
)
.await?;
Ok::<Option<(Box<bool>, String, PartialVersionInfo)>, Error>(
Some((stable, loader, version)),
)
},
),
)
.await?;
let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new()));
futures::future::try_join_all(loader_versions.into_iter()
.flatten().map(
|(stable, loader, version)| async {
let libs = futures::future::try_join_all(
version.libraries.into_iter().map(|mut lib| async {
{
let mut visited_assets =
visited_artifacts_mutex.lock().await;
if visited_assets.contains(&lib.name) {
lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING);
lib.url = Some(format_url("maven/"));
return Ok(lib);
} else {
visited_assets.push(lib.name.clone())
}
}
if lib.name.contains(DUMMY_GAME_VERSION) {
lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING);
futures::future::try_join_all(list.game.clone().into_iter().map(|game_version| async {
let semaphore = semaphore.clone();
let uploaded_files_mutex = uploaded_files_mutex.clone();
let lib_name = lib.name.clone();
let lib_url = lib.url.clone();
async move {
let artifact_path =
daedalus::get_path_from_artifact(&lib_name.replace(DUMMY_REPLACE_STRING, &game_version.version))?;
let artifact = download_file(
&format!(
"{}{}",
lib_url.unwrap_or_else(|| {
"https://maven.quiltmc.org/".to_string()
}),
artifact_path
),
None,
semaphore.clone(),
)
.await?;
upload_file_to_bucket(
format!("{}/{}", "maven", artifact_path),
artifact.to_vec(),
Some("application/java-archive".to_string()),
&uploaded_files_mutex,
semaphore.clone(),
)
.await?;
Ok::<(), Error>(())
}.await?;
Ok::<(), Error>(())
})).await?;
lib.url = Some(format_url("maven/"));
return Ok(lib);
}
let artifact_path =
daedalus::get_path_from_artifact(&lib.name)?;
let artifact = download_file(
&format!(
"{}{}",
lib.url.unwrap_or_else(|| {
"https://maven.quiltmc.org/".to_string()
}),
artifact_path
),
None,
semaphore.clone(),
)
.await?;
lib.url = Some(format_url("maven/"));
upload_file_to_bucket(
format!("{}/{}", "maven", artifact_path),
artifact.to_vec(),
Some("application/java-archive".to_string()),
&uploaded_files_mutex,
semaphore.clone(),
)
.await?;
Ok::<Library, Error>(lib)
}),
)
.await?;
let version_path = format!(
"quilt/v{}/versions/{}.json",
daedalus::modded::CURRENT_QUILT_FORMAT_VERSION,
&loader
);
upload_file_to_bucket(
version_path.clone(),
serde_json::to_vec(&PartialVersionInfo {
arguments: version.arguments,
id: version
.id
.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING),
main_class: version.main_class,
release_time: version.release_time,
time: version.time,
type_: version.type_,
inherits_from: version
.inherits_from
.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING),
libraries: libs,
minecraft_arguments: version.minecraft_arguments,
processors: None,
data: None,
})?,
Some("application/json".to_string()),
&uploaded_files_mutex,
semaphore.clone(),
)
.await?;
{
let mut loader_version_map = loader_version_mutex.lock().await;
async move {
loader_version_map.push(LoaderVersion {
id: loader.to_string(),
url: format_url(&version_path),
stable: *stable,
});
}
.await;
}
Ok::<(), Error>(())
},
))
.await?;
let mut loader_version_mutex = loader_version_mutex.into_inner();
if !loader_version_mutex.is_empty() {
if let Some(version) = versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) {
version.loaders.append(&mut loader_version_mutex);
} else {
versions.push(Version {
id: DUMMY_REPLACE_STRING.to_string(),
stable: true,
loaders: loader_version_mutex,
});
}
}
for version in &list.game {
if !versions.iter().any(|x| x.id == version.version) {
versions.push(Version {
id: version.version.clone(),
stable: version.stable,
loaders: vec![],
});
}
}
versions.sort_by(|x, y| {
minecraft_versions
.versions
.iter()
.position(|z| x.id == z.id)
.unwrap_or_default()
.cmp(
&minecraft_versions
.versions
.iter()
.position(|z| y.id == z.id)
.unwrap_or_default(),
)
});
for version in &mut versions {
version.loaders.sort_by(|x, y| {
list.loader
.iter()
.position(|z| {
x.id.split('-').next().unwrap_or_default() == &*z.version
})
.unwrap_or_default()
.cmp(
&list
.loader
.iter()
.position(|z| {
y.id.split('-').next().unwrap_or_default()
== z.version
})
.unwrap_or_default(),
)
})
}
upload_file_to_bucket(
format!(
"quilt/v{}/manifest.json",
daedalus::modded::CURRENT_QUILT_FORMAT_VERSION,
),
serde_json::to_vec(&Manifest {
game_versions: versions,
})?,
Some("application/json".to_string()),
&uploaded_files_mutex,
semaphore,
)
.await?;
if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) {
uploaded_files.extend(uploaded_files_mutex.into_inner());
}
Ok(())
}
const QUILT_META_URL: &str = "https://meta.quiltmc.org/v3";
async fn fetch_quilt_version(
version_number: &str,
loader_version: &str,
semaphore: Arc<Semaphore>,
) -> Result<PartialVersionInfo, Error> {
Ok(serde_json::from_slice(
&download_file(
&format!(
"{}/versions/loader/{}/{}/profile/json",
QUILT_META_URL, version_number, loader_version
),
None,
semaphore,
)
.await?,
)?)
}
#[derive(Serialize, Deserialize, Debug, Clone)]
/// Versions of quilt components
struct QuiltVersions {
/// Versions of Minecraft that quilt supports
pub game: Vec<QuiltGameVersion>,
/// Available versions of the quilt loader
pub loader: Vec<QuiltLoaderVersion>,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
/// A version of Minecraft that quilt supports
struct QuiltGameVersion {
/// The version number of the game
pub version: String,
/// Whether the Minecraft version is stable or not
pub stable: bool,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
/// A version of the quilt loader
struct QuiltLoaderVersion {
/// The separator to get the build number
pub separator: String,
/// The build number
pub build: u32,
/// The maven artifact
pub maven: String,
/// The version number of the quilt loader
pub version: String,
}
/// Fetches the list of quilt versions
async fn fetch_quilt_versions(
url: Option<&str>,
semaphore: Arc<Semaphore>,
) -> Result<QuiltVersions, Error> {
Ok(serde_json::from_slice(
&download_file(
url.unwrap_or(&*format!("{}/versions", QUILT_META_URL)),
None,
semaphore,
)
.await?,
)?)
}