From 4a7f4bde4a4384d92b79768cecaf147a7033e6f8 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 5 Oct 2021 22:52:17 -0700 Subject: [PATCH 01/76] Working mirroring of minecraft metadata --- .env | 11 ++ .gitignore | 121 ++++++++++++ .idea/.gitignore | 8 + .idea/daedalus.iml | 14 ++ .idea/modules.xml | 8 + .idea/runConfigurations.xml | 10 + .idea/vcs.xml | 6 + Cargo.toml | 6 + daedalus/Cargo.toml | 17 ++ daedalus/src/lib.rs | 99 ++++++++++ daedalus/src/minecraft.rs | 328 +++++++++++++++++++++++++++++++ daedalus_client/Cargo.toml | 21 ++ daedalus_client/src/main.rs | 125 ++++++++++++ daedalus_client/src/minecraft.rs | 172 ++++++++++++++++ 14 files changed, 946 insertions(+) create mode 100644 .env create mode 100644 .gitignore create mode 100644 .idea/.gitignore create mode 100644 .idea/daedalus.iml create mode 100644 .idea/modules.xml create mode 100644 .idea/runConfigurations.xml create mode 100644 .idea/vcs.xml create mode 100644 Cargo.toml create mode 100644 daedalus/Cargo.toml create mode 100644 daedalus/src/lib.rs create mode 100644 daedalus/src/minecraft.rs create mode 100644 daedalus_client/Cargo.toml create mode 100644 daedalus_client/src/main.rs create mode 100644 daedalus_client/src/minecraft.rs diff --git a/.env b/.env new file mode 100644 index 00000000..4e9b803b --- /dev/null +++ b/.env @@ -0,0 +1,11 @@ +BASE_URL=https://modrinth-cdn-staging.nyc3.digitaloceanspaces.com +BASE_FOLDER=gamedata + +S3_ACCESS_TOKEN=none +S3_SECRET=none +S3_URL=none +S3_REGION=none +S3_BUCKET_NAME=none + +DO_INTEGRATION=false +DO_ACCESS_KEY=none \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..dae6eff4 --- /dev/null +++ b/.gitignore @@ -0,0 +1,121 @@ +### Intellij ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### Intellij Patch ### +# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 + +# *.iml +# modules.xml +# .idea/misc.xml +# *.ipr + +# Sonarlint plugin +# https://plugins.jetbrains.com/plugin/7973-sonarlint +.idea/**/sonarlint/ + +# SonarQube Plugin +# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin +.idea/**/sonarIssues.xml + +# Markdown Navigator plugin +# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced +.idea/**/markdown-navigator.xml +.idea/**/markdown-navigator-enh.xml +.idea/**/markdown-navigator/ + +# Cache file creation bug +# See https://youtrack.jetbrains.com/issue/JBR-2257 +.idea/$CACHE_FILE$ + +# CodeStream plugin +# https://plugins.jetbrains.com/plugin/12206-codestream +.idea/codestream.xml + +### Rust ### +# Generated by Cargo +# will have compiled files and executables +debug/ +target/ + +# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries +# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html +Cargo.lock + +# These are backup files generated by rustfmt +**/*.rs.bk + +# MSVC Windows builds of rustc generate these, which store debugging information +*.pdb diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 00000000..73f69e09 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,8 @@ +# Default ignored files +/shelf/ +/workspace.xml +# Datasource local storage ignored files +/dataSources/ +/dataSources.local.xml +# Editor-based HTTP Client requests +/httpRequests/ diff --git a/.idea/daedalus.iml b/.idea/daedalus.iml new file mode 100644 index 00000000..ec7bb013 --- /dev/null +++ b/.idea/daedalus.iml @@ -0,0 +1,14 @@ + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 00000000..c7359481 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/runConfigurations.xml b/.idea/runConfigurations.xml new file mode 100644 index 00000000..797acea5 --- /dev/null +++ b/.idea/runConfigurations.xml @@ -0,0 +1,10 @@ + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 00000000..35eb1ddf --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 00000000..dc360c2e --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,6 @@ +[workspace] + +members = [ + "daedalus", + "daedalus_client" +] \ No newline at end of file diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml new file mode 100644 index 00000000..a52c207f --- /dev/null +++ b/daedalus/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "daedalus" +version = "0.1.0" +authors = ["Jai A "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +reqwest = { version = "0.11", features = ["json"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +chrono = { version = "0.4", features = ["serde"] } +bytes = "1" +thiserror = "1.0" +tokio = { version = "1", features = ["full"] } +sha1 = { version = "0.6.0", features = ["std"]} \ No newline at end of file diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs new file mode 100644 index 00000000..8aea3e1e --- /dev/null +++ b/daedalus/src/lib.rs @@ -0,0 +1,99 @@ +//! # Daedalus +//! +//! Daedalus is a library which provides models and methods to fetch metadata about games + +#![warn(missing_docs, unused_import_braces, missing_debug_implementations)] + +/// Models and methods for fetching metadata for Minecraft +pub mod minecraft; + +#[derive(thiserror::Error, Debug)] +/// An error type representing possible errors when fetching metadata +pub enum Error { + #[error("Failed to validate file checksum at url {url} with hash {hash} after {tries} tries")] + /// A checksum was failed to validate for a file + ChecksumFailure { + /// The checksum's hash + hash: String, + /// The URL of the file attempted to be downloaded + url: String, + /// The amount of tries that the file was downloaded until failure + tries: u32, + }, + /// There was an error while deserializing metadata + #[error("Error while deserializing JSON")] + SerdeError(#[from] serde_json::Error), + /// There was a network error when fetching an object + #[error("Unable to fetch {item}")] + FetchError { + /// The internal reqwest error + inner: reqwest::Error, + /// The item that was failed to be fetched + item: String, + }, + /// There was an error when managing async tasks + #[error("Error while managing asynchronous tasks")] + TaskError(#[from] tokio::task::JoinError), +} + +/// Downloads a file with retry and checksum functionality +pub async fn download_file(url: &str, sha1: Option<&str>) -> Result { + let client = reqwest::Client::builder() + .tcp_keepalive(Some(std::time::Duration::from_secs(10))) + .build() + .map_err(|err| Error::FetchError { + inner: err, + item: url.to_string(), + })?; + + for attempt in 1..=4 { + let result = client.get(url).send().await; + + match result { + Ok(x) => { + let bytes = x.bytes().await; + + if let Ok(bytes) = bytes { + if let Some(sha1) = sha1 { + if &*get_hash(bytes.clone()).await? != sha1 { + if attempt <= 3 { + continue; + } else { + return Err(Error::ChecksumFailure { + hash: sha1.to_string(), + url: url.to_string(), + tries: attempt, + }); + } + } + } + + return Ok(bytes); + } else if attempt <= 3 { + continue; + } else if let Err(err) = bytes { + return Err(Error::FetchError { + inner: err, + item: url.to_string(), + }); + } + } + Err(_) if attempt <= 3 => continue, + Err(err) => { + return Err(Error::FetchError { + inner: err, + item: url.to_string(), + }) + } + } + } + + unreachable!() +} + +/// Computes a checksum of the input bytes +pub async fn get_hash(bytes: bytes::Bytes) -> Result { + let hash = tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest()).await?; + + Ok(hash) +} diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs new file mode 100644 index 00000000..89008da1 --- /dev/null +++ b/daedalus/src/minecraft.rs @@ -0,0 +1,328 @@ +use crate::{download_file, Error}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// The latest version of the format the model structs deserialize to +pub const CURRENT_FORMAT_VERSION: usize = 0; + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "snake_case")] +/// The version type +pub enum VersionType { + /// A major version, which is stable for all players to use + Release, + /// An experimental version, which is unstable and used for feature previews and beta testing + Snapshot, + /// The oldest versions before the game was released + OldAlpha, + /// Early versions of the game + OldBeta, +} + +impl VersionType { + /// Converts the version type to a string + pub fn as_str(&self) -> &'static str { + match self { + VersionType::Release => "release", + VersionType::Snapshot => "snapshot", + VersionType::OldAlpha => "old_alpha", + VersionType::OldBeta => "old_beta", + } + } +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +/// A game version of Minecraft +pub struct Version { + /// A unique identifier of the version + pub id: String, + #[serde(rename = "type")] + /// The release type of the version + pub type_: VersionType, + /// A link to additional information about the version + pub url: String, + /// The latest time a file in this version was updated + pub time: DateTime, + /// The time this version was released + pub release_time: DateTime, + /// The SHA1 hash of the additional information about the version + pub sha1: String, + /// Whether the version supports the latest player safety features + pub compliance_level: u32, + /// (Modrinth Provided) The link to the assets index for this version + /// This is only available when using the Modrinth mirror + pub assets_index_url: Option, + /// (Modrinth Provided) The SHA1 hash of the assets index for this version + /// This is only available when using the Modrinth mirror + pub assets_index_sha1: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// The latest snapshot and release of the game +pub struct LatestVersion { + /// The version id of the latest release + pub release: String, + /// The version id of the latest snapshot + pub snapshot: String, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// Data of all game versions of Minecraft +pub struct VersionManifest { + /// A struct containing the latest snapshot and release of the game + pub latest: LatestVersion, + /// A list of game versions of Minecraft + pub versions: Vec, +} + +/// The URL to the version manifest +pub const VERSION_MANIFEST_URL: &str = + "https://launchermeta.mojang.com/mc/game/version_manifest_v2.json"; + +/// Fetches a version manifest from the specified URL. If no URL is specified, the default is used. +pub async fn fetch_version_manifest(url: Option<&str>) -> Result { + Ok(serde_json::from_slice( + &download_file(url.unwrap_or(VERSION_MANIFEST_URL), None).await?, + )?) +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +/// Information about the assets of the game +pub struct AssetIndex { + /// The game version ID the assets are for + pub id: String, + /// The SHA1 hash of the assets index + pub sha1: String, + /// The size of the assets index + pub size: u32, + /// The size of the game version's assets + pub total_size: u32, + /// A URL to a file which contains information about the version's assets + pub url: String, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)] +#[serde(rename_all = "snake_case")] +/// The type of download +pub enum DownloadType { + /// The download is for the game client + Client, + /// The download is mappings for the game + ClientMappings, + /// The download is for the game server + Server, + /// The download is mappings for the game server + ServerMappings, + /// The download is for the windows server + WindowsServer, +} + +#[derive(Serialize, Deserialize, Debug)] +/// Download information of a file +pub struct Download { + /// The SHA1 hash of the file + pub sha1: String, + /// The size of the file + pub size: u32, + /// The URL where the file can be downloaded + pub url: String, +} + +#[derive(Serialize, Deserialize, Debug)] +/// Download information of a library +pub struct LibraryDownload { + /// The path that the library should be saved to + pub path: String, + /// The SHA1 hash of the library + pub sha1: String, + /// The size of the library + pub size: u32, + /// The URL where the library can be downloaded + pub url: String, +} + +#[derive(Serialize, Deserialize, Debug)] +/// A list of files that should be downloaded for libraries +pub struct LibraryDownloads { + /// The primary library artifact + pub artifact: Option, + /// Conditional files that may be needed to be downloaded alongside the library + /// The HashMap key specifies a classifier as additional information for downloading files + pub classifiers: Option>, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "snake_case")] +/// The action a rule can follow +pub enum RuleAction { + /// The rule's status allows something to be done + Allow, + /// The rule's status disallows something to be done + Disallow, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)] +#[serde(rename_all = "snake_case")] +/// An enum representing the different types of operating systems +pub enum Os { + /// MacOS + Osx, + /// Windows + Windows, + /// Linux and its derivatives + Linux, + /// The OS is unknown + Unknown, +} + +#[derive(Serialize, Deserialize, Debug)] +/// A rule which depends on what OS the user is on +pub struct OsRule { + /// The name of the OS + pub name: Option, + /// The version of the OS. This is normally a RegEx + pub version: Option, + /// The architecture of the OS + pub arch: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +/// A rule which depends on the toggled features of the launcher +pub struct FeatureRule { + /// Whether the user is in demo mode + pub is_demo_user: Option, + /// Whether the user is using the demo resolution + pub has_demo_resolution: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +/// A rule deciding whether a file is downloaded, an argument is used, etc. +pub struct Rule { + /// The action the rule takes + pub action: RuleAction, + /// The OS rule + pub os: Option, + /// The feature rule + pub features: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +/// Information delegating the extraction of the library +pub struct LibraryExtract { + /// Files/Folders to be excluded from the extraction of the library + pub exclude: Option>, +} + +#[derive(Serialize, Deserialize, Debug)] +/// A library which the game relies on to run +pub struct Library { + /// The files the library has + pub downloads: LibraryDownloads, + /// Rules of the extraction of the file + pub extract: Option, + /// The maven name of the library. The format is `groupId:artifactId:version` + pub name: String, + /// Native files that the library relies on + pub natives: Option>, + /// Rules deciding whether the library should be downloaded or not + pub rules: Option>, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(untagged)] +/// A container for an argument or multiple arguments +pub enum ArgumentValue { + /// The container has one argument + Single(String), + /// The container has multiple arguments + Many(Vec), +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(untagged)] +/// A command line argument passed to a program +pub enum Argument { + /// An argument which is applied no matter what + Normal(String), + /// An argument which is only applied if certain conditions are met + Ruled { + /// The rules deciding whether the argument(s) is used or not + rules: Vec, + /// The container of the argument(s) that should be applied accordingly + value: ArgumentValue, + }, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)] +#[serde(rename_all = "snake_case")] +/// The type of argument +pub enum ArgumentType { + /// The argument is passed to the game + Game, + /// The argument is passed to the JVM + Jvm, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +/// Information about a version +pub struct VersionInfo { + /// Arguments passed to the game or JVM + pub arguments: Option>>, + /// Assets for the game + pub asset_index: AssetIndex, + /// The version ID of the assets + pub assets: String, + /// Game downloads of the version + pub downloads: HashMap, + /// The version ID of the version + pub id: String, + /// Libraries that the version depends on + pub libraries: Vec, + /// The classpath to the main class to launch the game + pub main_class: String, + /// (Legacy) Arguments passed to the game + pub minecraft_arguments: Option, + /// The minimum version of the Minecraft Launcher that can run this version of the game + pub minimum_launcher_version: u32, + /// The time that the version was released + pub release_time: DateTime, + /// The latest time a file in this version was updated + pub time: DateTime, + #[serde(rename = "type")] + /// The type of version + pub type_: VersionType, +} + +/// Fetches detailed information about a version from the manifest +pub async fn fetch_version_info(version: &Version) -> Result { + Ok(serde_json::from_slice( + &download_file(&version.url, Some(&version.sha1)).await?, + )?) +} + +#[derive(Serialize, Deserialize, Debug)] +/// An asset of the game +pub struct Asset { + /// The SHA1 hash of the asset file + pub hash: String, + /// The size of the asset file + pub size: u32, +} + +#[derive(Serialize, Deserialize, Debug)] +/// An index containing all assets the game needs +pub struct AssetsIndex { + /// A hashmap containing the filename (key) and asset (value) + pub objects: HashMap, +} + +/// Fetches the assets index from the version info +pub async fn fetch_assets_index(version: &VersionInfo) -> Result { + Ok(serde_json::from_slice( + &download_file(&version.asset_index.url, Some(&version.asset_index.sha1)).await?, + )?) +} diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml new file mode 100644 index 00000000..4f2fa019 --- /dev/null +++ b/daedalus_client/Cargo.toml @@ -0,0 +1,21 @@ +[package] +name = "daedalus_client" +version = "0.1.0" +authors = ["Jai A "] +edition = "2018" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +daedalus = { path = "../daedalus" } +tokio = { version = "1", features = ["full"] } +futures = "0.3.17" +dotenv = "0.15.0" +log = "0.4.8" +serde_json = "1.0" +lazy_static = "1.4.0" +thiserror = "1.0" +reqwest = "0.11.4" + +rusoto_core = "0.47.0" +rusoto_s3 = "0.47.0" diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs new file mode 100644 index 00000000..bb10303a --- /dev/null +++ b/daedalus_client/src/main.rs @@ -0,0 +1,125 @@ +use log::{error, info, warn}; +use rusoto_core::credential::StaticProvider; +use rusoto_core::{HttpClient, Region, RusotoError}; +use rusoto_s3::{S3Client, PutObjectError}; +use rusoto_s3::{PutObjectRequest, S3}; + +mod minecraft; + +#[derive(thiserror::Error, Debug)] +pub enum Error { + #[error("{0}")] + DaedalusError(#[from] daedalus::Error), + #[error("Error while deserializing JSON")] + SerdeError(#[from] serde_json::Error), + #[error("Unable to fetch {item}")] + FetchError { + inner: reqwest::Error, + item: String, + }, + #[error("Error while managing asynchronous tasks")] + TaskError(#[from] tokio::task::JoinError), + #[error("Error while uploading file to S3")] + S3Error { + inner: RusotoError, + file: String, + }, +} + +#[tokio::main] +async fn main() { + if check_env_vars() { + error!("Some environment variables are missing!"); + + return; + } + + minecraft::retrieve_data().await.unwrap(); +} + +fn check_env_vars() -> bool { + let mut failed = false; + + fn check_var(var: &str) -> bool { + if dotenv::var(var) + .ok() + .and_then(|s| s.parse::().ok()) + .is_none() + { + warn!( + "Variable `{}` missing in dotenv or not of type `{}`", + var, + std::any::type_name::() + ); + true + } else { + false + } + } + + failed |= check_var::("BASE_URL"); + failed |= check_var::("BASE_FOLDER"); + + failed |= check_var::("S3_ACCESS_TOKEN"); + failed |= check_var::("S3_SECRET"); + failed |= check_var::("S3_URL"); + failed |= check_var::("S3_REGION"); + failed |= check_var::("S3_BUCKET_NAME"); + + failed |= check_var::("DO_INTEGRATION"); + + let do_integration = dotenv::var("DO_INTEGRATION") + .ok() + .map(|x| x.parse::().ok()) + .flatten(); + + if do_integration.unwrap_or(false) { + failed |= check_var::("DO_ACCESS_KEY"); + } + + failed +} + +lazy_static::lazy_static! { + static ref CLIENT : S3Client = S3Client::new_with( + HttpClient::new().unwrap(), + StaticProvider::new( + dotenv::var("S3_ACCESS_TOKEN").unwrap(), + dotenv::var("S3_SECRET").unwrap(), + None, + None, + ), + Region::Custom { + name: dotenv::var("S3_REGION").unwrap(), + endpoint: dotenv::var("S3_URL").unwrap(), + }, + ); +} + +pub async fn upload_file_to_bucket(path: String, bytes: Vec, content_type: Option) -> Result<(), Error> { + CLIENT + .put_object(PutObjectRequest { + bucket: dotenv::var("S3_BUCKET_NAME").unwrap(), + key: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path), + body: Some(bytes.into()), + acl: Some("public-read".to_string()), + content_type, + ..Default::default() + }) + .await + .map_err(|err| Error::S3Error { + inner: err, + file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path) + })?; + + Ok(()) +} + +pub fn format_url(path: &str) -> String { + format!( + "{}/{}/{}", + &*dotenv::var("BASE_URL").unwrap(), + &*dotenv::var("BASE_FOLDER").unwrap(), + path + ) +} diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs new file mode 100644 index 00000000..1336e9ff --- /dev/null +++ b/daedalus_client/src/minecraft.rs @@ -0,0 +1,172 @@ +use crate::{format_url, upload_file_to_bucket, Error}; +use daedalus::download_file; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; + +pub async fn retrieve_data() -> Result<(), Error> { + let old_manifest = + daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&*format!( + "minecraft/v{}/version_manifest.json", + daedalus::minecraft::CURRENT_FORMAT_VERSION + )))) + .await + .ok(); + + let mut manifest = daedalus::minecraft::fetch_version_manifest(None) + .await?; + let cloned_manifest = Arc::new(Mutex::new(manifest.clone())); + + let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); + + let now = Instant::now(); + + let mut versions = manifest + .versions + .iter_mut() + .map(|version| async { + let old_version = if let Some(old_manifest) = &old_manifest { + old_manifest.versions.iter().find(|x| x.id == version.id) + } else { + None + }; + + if let Some(old_version) = old_version { + if old_version.sha1 == version.sha1 { + return Ok(()); + } + } + + let visited_assets_mutex = Arc::clone(&visited_assets_mutex); + let cloned_manifest_mutex = Arc::clone(&cloned_manifest); + + let assets_hash = old_version.map(|x| x.assets_index_sha1.clone()).flatten(); + + async move { + let mut upload_futures = Vec::new(); + + let now = Instant::now(); + let mut version_println = daedalus::minecraft::fetch_version_info(version) + .await + ?; + let elapsed = now.elapsed(); + println!("Version {} Elapsed: {:.2?}", version.id, elapsed); + + let version_path = format!( + "minecraft/v{}/versions/{}.json", + daedalus::minecraft::CURRENT_FORMAT_VERSION, + version.id + ); + let assets_path = format!( + "minecraft/v{}/assets/{}.json", + daedalus::minecraft::CURRENT_FORMAT_VERSION, + version_println.asset_index.id + ); + let assets_index_url = version_println.asset_index.url.clone(); + + { + let mut cloned_manifest = match cloned_manifest_mutex.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + + let position = cloned_manifest + .versions + .iter() + .position(|x| version.id == x.id) + .unwrap(); + cloned_manifest.versions[position].url = format_url(&version_path); + cloned_manifest.versions[position].assets_index_sha1 = + Some(version_println.asset_index.sha1.clone()); + cloned_manifest.versions[position].assets_index_url = + Some(format_url(&assets_path)); + version_println.asset_index.url = format_url(&assets_path); + } + + let mut download_assets = false; + + { + let mut visited_assets = match visited_assets_mutex.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + + if !visited_assets.contains(&version_println.asset_index.id) { + if let Some(assets_hash) = assets_hash { + if version_println.asset_index.sha1 != assets_hash { + download_assets = true; + } + } else { + download_assets = true; + } + } + + if download_assets { + visited_assets.push(version_println.asset_index.id.clone()); + } + } + + if download_assets { + let assets_index = + download_file(&assets_index_url, Some(&version_println.asset_index.sha1)) + .await?; + + { + upload_futures + .push(upload_file_to_bucket(assets_path, assets_index.to_vec(), Some("application/json".to_string()))); + } + } + + { + upload_futures.push(upload_file_to_bucket( + version_path, + serde_json::to_vec(&version_println)?, + Some("application/json".to_string()) + )); + } + + let now = Instant::now(); + futures::future::try_join_all(upload_futures).await?; + let elapsed = now.elapsed(); + println!("Spaces Upload {} Elapsed: {:.2?}", version.id, elapsed); + + Ok::<(), Error>(()) + } + .await?; + + Ok::<(), Error>(()) + }) + .peekable(); + + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); + + let chunk: Vec<_> = versions.by_ref().take(100).collect(); + futures::future::try_join_all(chunk).await?; + + std::thread::sleep(Duration::from_secs(1)); + + chunk_index += 1; + + let elapsed = now.elapsed(); + println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } + + upload_file_to_bucket( + format!( + "minecraft/v{}/version_manifest.json", + daedalus::minecraft::CURRENT_FORMAT_VERSION + ), + serde_json::to_vec(&*match cloned_manifest.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + })?, + Some("application/json".to_string()) + ) + .await?; + + let elapsed = now.elapsed(); + println!("Elapsed: {:.2?}", elapsed); + + Ok(()) +} From 32850f6770f13ff78dad6cb11ead9968e39b1c93 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sat, 9 Oct 2021 14:04:45 -0700 Subject: [PATCH 02/76] Fabric support --- daedalus/src/fabric.rs | 132 ++++++++++++++++++++++ daedalus/src/lib.rs | 28 +++++ daedalus/src/minecraft.rs | 21 +++- daedalus_client/src/fabric.rs | 188 +++++++++++++++++++++++++++++++ daedalus_client/src/main.rs | 18 +-- daedalus_client/src/minecraft.rs | 43 ++++--- 6 files changed, 397 insertions(+), 33 deletions(-) create mode 100644 daedalus/src/fabric.rs create mode 100644 daedalus_client/src/fabric.rs diff --git a/daedalus/src/fabric.rs b/daedalus/src/fabric.rs new file mode 100644 index 00000000..052486ff --- /dev/null +++ b/daedalus/src/fabric.rs @@ -0,0 +1,132 @@ +use crate::minecraft::{Argument, ArgumentType, Library, VersionInfo, VersionType}; +use crate::{download_file, Error}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// The latest version of the format the model structs deserialize to +pub const CURRENT_FORMAT_VERSION: usize = 0; + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +/// A partial version returned by fabric meta +pub struct PartialVersionInfo { + /// The version ID of the version + pub id: String, + /// The version ID this partial version inherits from + pub inherits_from: String, + /// The time that the version was released + pub release_time: DateTime, + /// The latest time a file in this version was updated + pub time: DateTime, + /// The classpath to the main class to launch the game + pub main_class: String, + /// Arguments passed to the game or JVM + pub arguments: Option>>, + /// Libraries that the version depends on + pub libraries: Vec, + #[serde(rename = "type")] + /// The type of version + pub type_: VersionType, +} + +/// Merges a partial version into a complete one +pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> VersionInfo { + VersionInfo { + arguments: if let Some(partial_args) = partial.arguments { + if let Some(merge_args) = merge.arguments { + Some(partial_args.into_iter().chain(merge_args).collect()) + } else { + Some(partial_args) + } + } else { + merge.arguments + }, + asset_index: merge.asset_index, + assets: merge.assets, + downloads: merge.downloads, + id: merge.id, + libraries: partial + .libraries + .into_iter() + .chain(merge.libraries) + .collect::>(), + main_class: partial.main_class, + minecraft_arguments: merge.minecraft_arguments, + minimum_launcher_version: merge.minimum_launcher_version, + release_time: partial.release_time, + time: partial.time, + type_: partial.type_, + } +} + +/// The default servers for fabric meta +pub const FABRIC_META_URL: &str = "https://meta.fabricmc.net/v2"; + +/// Fetches the manifest of a fabric loader version and game version +pub async fn fetch_fabric_version( + version_number: &str, + loader_version: &str, +) -> Result { + Ok(serde_json::from_slice( + &download_file( + &*format!( + "{}/versions/loader/{}/{}/profile/json", + FABRIC_META_URL, version_number, loader_version + ), + None, + ) + .await?, + )?) +} + +/// Fetches the manifest of a game version's URL +pub async fn fetch_fabric_game_version(url: &str) -> Result { + Ok(serde_json::from_slice(&download_file(url, None).await?)?) +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// Versions of fabric components +pub struct FabricVersions { + /// Versions of Minecraft that fabric supports + pub game: Vec, + /// Available versions of the fabric loader + pub loader: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// A version of Minecraft that fabric supports +pub struct FabricGameVersion { + /// The version number of the game + pub version: String, + /// Whether the Minecraft version is stable or not + pub stable: bool, + /// (Modrinth Provided) The URLs to download this version's profile with a loader + /// The key of the map is the loader version, and the value is the URL + pub urls: Option>, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// A version of the fabric loader +pub struct FabricLoaderVersion { + /// The separator to get the build number + pub separator: String, + /// The build number + pub build: u32, + /// The maven artifact + pub maven: String, + /// The version number of the fabric loader + pub version: String, + /// Whether the loader is stable or not + pub stable: bool, +} +/// Fetches the list of fabric versions +pub async fn fetch_fabric_versions(url: Option<&str>) -> Result { + Ok(serde_json::from_slice( + &download_file( + url.unwrap_or(&*format!("{}/versions", FABRIC_META_URL)), + None, + ) + .await?, + )?) +} diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index 8aea3e1e..92ff542f 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -4,6 +4,8 @@ #![warn(missing_docs, unused_import_braces, missing_debug_implementations)] +/// Models and methods for fetching metadata for the Fabric mod loader +pub mod fabric; /// Models and methods for fetching metadata for Minecraft pub mod minecraft; @@ -34,6 +36,32 @@ pub enum Error { /// There was an error when managing async tasks #[error("Error while managing asynchronous tasks")] TaskError(#[from] tokio::task::JoinError), + /// Error while parsing input + #[error("{0}")] + ParseError(String), +} + +/// Converts a maven artifact to a path +pub fn get_path_from_artifact(artifact: &str) -> Result { + let name_items = artifact.split(':').collect::>(); + + let package = name_items.get(0).ok_or_else(|| { + Error::ParseError(format!("Unable to find package for library {}", &artifact)) + })?; + let name = name_items.get(1).ok_or_else(|| { + Error::ParseError(format!("Unable to find name for library {}", &artifact)) + })?; + let version = name_items.get(2).ok_or_else(|| { + Error::ParseError(format!("Unable to find version for library {}", &artifact)) + })?; + + Ok(format!( + "{}/{}/{}-{}.jar", + package.replace(".", "/"), + version, + name, + version + )) } /// Downloads a file with retry and checksum functionality diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 89008da1..e90e8f4d 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -69,7 +69,7 @@ pub struct LatestVersion { } #[derive(Serialize, Deserialize, Debug, Clone)] -/// Data of all game versions of Minecraft +/// Data of all game versions of Minecrafat pub struct VersionManifest { /// A struct containing the latest snapshot and release of the game pub latest: LatestVersion, @@ -181,10 +181,13 @@ pub enum Os { #[derive(Serialize, Deserialize, Debug)] /// A rule which depends on what OS the user is on pub struct OsRule { + #[serde(skip_serializing_if = "Option::is_none")] /// The name of the OS pub name: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// The version of the OS. This is normally a RegEx pub version: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// The architecture of the OS pub arch: Option, } @@ -192,8 +195,10 @@ pub struct OsRule { #[derive(Serialize, Deserialize, Debug)] /// A rule which depends on the toggled features of the launcher pub struct FeatureRule { + #[serde(skip_serializing_if = "Option::is_none")] /// Whether the user is in demo mode pub is_demo_user: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// Whether the user is using the demo resolution pub has_demo_resolution: Option, } @@ -203,8 +208,10 @@ pub struct FeatureRule { pub struct Rule { /// The action the rule takes pub action: RuleAction, + #[serde(skip_serializing_if = "Option::is_none")] /// The OS rule pub os: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// The feature rule pub features: Option, } @@ -212,6 +219,7 @@ pub struct Rule { #[derive(Serialize, Deserialize, Debug)] /// Information delegating the extraction of the library pub struct LibraryExtract { + #[serde(skip_serializing_if = "Option::is_none")] /// Files/Folders to be excluded from the extraction of the library pub exclude: Option>, } @@ -219,14 +227,21 @@ pub struct LibraryExtract { #[derive(Serialize, Deserialize, Debug)] /// A library which the game relies on to run pub struct Library { + #[serde(skip_serializing_if = "Option::is_none")] /// The files the library has - pub downloads: LibraryDownloads, + pub downloads: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// Rules of the extraction of the file pub extract: Option, /// The maven name of the library. The format is `groupId:artifactId:version` pub name: String, + #[serde(skip_serializing_if = "Option::is_none")] + /// The URL to the repository where the library can be downloaded + pub url: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// Native files that the library relies on pub natives: Option>, + #[serde(skip_serializing_if = "Option::is_none")] /// Rules deciding whether the library should be downloaded or not pub rules: Option>, } @@ -270,6 +285,7 @@ pub enum ArgumentType { #[serde(rename_all = "camelCase")] /// Information about a version pub struct VersionInfo { + #[serde(skip_serializing_if = "Option::is_none")] /// Arguments passed to the game or JVM pub arguments: Option>>, /// Assets for the game @@ -284,6 +300,7 @@ pub struct VersionInfo { pub libraries: Vec, /// The classpath to the main class to launch the game pub main_class: String, + #[serde(skip_serializing_if = "Option::is_none")] /// (Legacy) Arguments passed to the game pub minecraft_arguments: Option, /// The minimum version of the Minecraft Launcher that can run this version of the game diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs new file mode 100644 index 00000000..890bd5be --- /dev/null +++ b/daedalus_client/src/fabric.rs @@ -0,0 +1,188 @@ +use crate::{format_url, upload_file_to_bucket, Error}; +use daedalus::fabric::PartialVersionInfo; +use daedalus::minecraft::Library; +use std::collections::HashMap; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::{Duration, Instant}; + +pub async fn retrieve_data() -> Result<(), Error> { + let mut list = daedalus::fabric::fetch_fabric_versions(None).await?; + + let loaders = RwLock::new(Vec::new()); + let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); + + if let Some(latest) = list.loader.get(0) { + { + let mut loaders = match loaders.write() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + + loaders.push(latest.version.clone()); + + if !latest.stable { + if let Some(stable) = list.loader.iter().find(|x| x.stable) { + loaders.push(stable.version.clone()); + } + } + + list.loader = list + .loader + .into_iter() + .filter(|x| loaders.contains(&x.version)) + .collect(); + } + + let mut versions = list + .game + .iter_mut() + .map(|game_version| { + let loaders = match loaders.read() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + + let visited_artifacts_mutex = Arc::clone(&visited_artifacts_mutex); + let game_version_mutex = Mutex::new(HashMap::new()); + + async move { + let versions = futures::future::try_join_all(loaders.clone().into_iter().map( + |loader| async { + let version = daedalus::fabric::fetch_fabric_version( + &*game_version.version, + &*loader, + ) + .await + .expect(&*format!("{}, {}", game_version.version, loader)); + + Ok::<(String, PartialVersionInfo), Error>((loader, version)) + }, + )) + .await?; + + futures::future::try_join_all(versions.into_iter().map( + |(loader, version)| async { + let libs = futures::future::try_join_all( + version.libraries.into_iter().map(|mut lib| async { + { + let mut visited_assets = + match visited_artifacts_mutex.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + + if visited_assets.contains(&lib.name) { + lib.url = Some(format_url("maven/")); + + return Ok(lib); + } else { + visited_assets.push(lib.name.clone()) + } + } + + let artifact_path = + daedalus::get_path_from_artifact(&*lib.name)?; + + let artifact = daedalus::download_file( + &*format!( + "{}{}", + lib.url.unwrap_or_else(|| { + "https://maven.fabricmc.net/".to_string() + }), + artifact_path + ), + None, + ) + .await?; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + ) + .await?; + + Ok::(lib) + }), + ) + .await?; + + let version_path = format!( + "fabric/v{}/versions/{}-{}.json", + daedalus::fabric::CURRENT_FORMAT_VERSION, + version.inherits_from, + loader + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&PartialVersionInfo { + arguments: version.arguments, + id: version.id, + main_class: version.main_class, + release_time: version.release_time, + time: version.time, + type_: version.type_, + inherits_from: version.inherits_from, + libraries: libs, + })?, + Some("application/json".to_string()), + ) + .await?; + + { + let mut game_version_map = match game_version_mutex.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + }; + game_version_map.insert(loader, format_url(&*version_path)); + } + + Ok::<(), Error>(()) + }, + )) + .await?; + + game_version.urls = Some( + match game_version_mutex.lock() { + Ok(guard) => guard, + Err(poisoned) => poisoned.into_inner(), + } + .clone(), + ); + + Ok::<(), Error>(()) + } + }) + .peekable(); + + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); + + let chunk: Vec<_> = versions.by_ref().take(10).collect(); + futures::future::try_join_all(chunk).await?; + + std::thread::sleep(Duration::from_secs(1)); + + chunk_index += 1; + + let elapsed = now.elapsed(); + println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } + } + + upload_file_to_bucket( + format!( + "fabric/v{}/manifest.json", + daedalus::fabric::CURRENT_FORMAT_VERSION, + ), + serde_json::to_vec(&list)?, + Some("application/json".to_string()), + ) + .await?; + + Ok(()) +} diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index bb10303a..7ac9a186 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -1,9 +1,10 @@ use log::{error, info, warn}; use rusoto_core::credential::StaticProvider; use rusoto_core::{HttpClient, Region, RusotoError}; -use rusoto_s3::{S3Client, PutObjectError}; +use rusoto_s3::{PutObjectError, S3Client}; use rusoto_s3::{PutObjectRequest, S3}; +mod fabric; mod minecraft; #[derive(thiserror::Error, Debug)] @@ -13,10 +14,7 @@ pub enum Error { #[error("Error while deserializing JSON")] SerdeError(#[from] serde_json::Error), #[error("Unable to fetch {item}")] - FetchError { - inner: reqwest::Error, - item: String, - }, + FetchError { inner: reqwest::Error, item: String }, #[error("Error while managing asynchronous tasks")] TaskError(#[from] tokio::task::JoinError), #[error("Error while uploading file to S3")] @@ -34,7 +32,7 @@ async fn main() { return; } - minecraft::retrieve_data().await.unwrap(); + fabric::retrieve_data().await.unwrap(); } fn check_env_vars() -> bool { @@ -96,7 +94,11 @@ lazy_static::lazy_static! { ); } -pub async fn upload_file_to_bucket(path: String, bytes: Vec, content_type: Option) -> Result<(), Error> { +pub async fn upload_file_to_bucket( + path: String, + bytes: Vec, + content_type: Option, +) -> Result<(), Error> { CLIENT .put_object(PutObjectRequest { bucket: dotenv::var("S3_BUCKET_NAME").unwrap(), @@ -109,7 +111,7 @@ pub async fn upload_file_to_bucket(path: String, bytes: Vec, content_type: O .await .map_err(|err| Error::S3Error { inner: err, - file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path) + file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path), })?; Ok(()) diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 1336e9ff..ddcf286e 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -6,14 +6,13 @@ use std::time::{Duration, Instant}; pub async fn retrieve_data() -> Result<(), Error> { let old_manifest = daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&*format!( - "minecraft/v{}/version_manifest.json", + "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION )))) .await .ok(); - let mut manifest = daedalus::minecraft::fetch_version_manifest(None) - .await?; + let mut manifest = daedalus::minecraft::fetch_version_manifest(None).await?; let cloned_manifest = Arc::new(Mutex::new(manifest.clone())); let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); @@ -44,12 +43,7 @@ pub async fn retrieve_data() -> Result<(), Error> { async move { let mut upload_futures = Vec::new(); - let now = Instant::now(); - let mut version_println = daedalus::minecraft::fetch_version_info(version) - .await - ?; - let elapsed = now.elapsed(); - println!("Version {} Elapsed: {:.2?}", version.id, elapsed); + let mut version_info = daedalus::minecraft::fetch_version_info(version).await?; let version_path = format!( "minecraft/v{}/versions/{}.json", @@ -59,9 +53,9 @@ pub async fn retrieve_data() -> Result<(), Error> { let assets_path = format!( "minecraft/v{}/assets/{}.json", daedalus::minecraft::CURRENT_FORMAT_VERSION, - version_println.asset_index.id + version_info.asset_index.id ); - let assets_index_url = version_println.asset_index.url.clone(); + let assets_index_url = version_info.asset_index.url.clone(); { let mut cloned_manifest = match cloned_manifest_mutex.lock() { @@ -76,10 +70,10 @@ pub async fn retrieve_data() -> Result<(), Error> { .unwrap(); cloned_manifest.versions[position].url = format_url(&version_path); cloned_manifest.versions[position].assets_index_sha1 = - Some(version_println.asset_index.sha1.clone()); + Some(version_info.asset_index.sha1.clone()); cloned_manifest.versions[position].assets_index_url = Some(format_url(&assets_path)); - version_println.asset_index.url = format_url(&assets_path); + version_info.asset_index.url = format_url(&assets_path); } let mut download_assets = false; @@ -90,9 +84,9 @@ pub async fn retrieve_data() -> Result<(), Error> { Err(poisoned) => poisoned.into_inner(), }; - if !visited_assets.contains(&version_println.asset_index.id) { + if !visited_assets.contains(&version_info.asset_index.id) { if let Some(assets_hash) = assets_hash { - if version_println.asset_index.sha1 != assets_hash { + if version_info.asset_index.sha1 != assets_hash { download_assets = true; } } else { @@ -101,26 +95,29 @@ pub async fn retrieve_data() -> Result<(), Error> { } if download_assets { - visited_assets.push(version_println.asset_index.id.clone()); + visited_assets.push(version_info.asset_index.id.clone()); } } if download_assets { let assets_index = - download_file(&assets_index_url, Some(&version_println.asset_index.sha1)) + download_file(&assets_index_url, Some(&version_info.asset_index.sha1)) .await?; { - upload_futures - .push(upload_file_to_bucket(assets_path, assets_index.to_vec(), Some("application/json".to_string()))); + upload_futures.push(upload_file_to_bucket( + assets_path, + assets_index.to_vec(), + Some("application/json".to_string()), + )); } } { upload_futures.push(upload_file_to_bucket( version_path, - serde_json::to_vec(&version_println)?, - Some("application/json".to_string()) + serde_json::to_vec(&version_info)?, + Some("application/json".to_string()), )); } @@ -154,14 +151,14 @@ pub async fn retrieve_data() -> Result<(), Error> { upload_file_to_bucket( format!( - "minecraft/v{}/version_manifest.json", + "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION ), serde_json::to_vec(&*match cloned_manifest.lock() { Ok(guard) => guard, Err(poisoned) => poisoned.into_inner(), })?, - Some("application/json".to_string()) + Some("application/json".to_string()), ) .await?; From 16af479b833b86b1c0169085a56be7d252a767a9 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 17 Oct 2021 23:22:23 -0700 Subject: [PATCH 03/76] Legacy Forge Support --- LICENSE | 0 README.md | 0 daedalus/Cargo.toml | 9 ++++++++- daedalus/README.md | 6 ++++++ daedalus/src/fabric.rs | 8 ++++++-- daedalus/src/forge.rs | 21 +++++++++++++++++++++ daedalus/src/lib.rs | 27 ++++++++++++++++++++++++++- daedalus/src/minecraft.rs | 21 ++++++++++++--------- daedalus_client/src/forge.rs | 0 9 files changed, 79 insertions(+), 13 deletions(-) create mode 100644 LICENSE create mode 100644 README.md create mode 100644 daedalus/README.md create mode 100644 daedalus/src/forge.rs create mode 100644 daedalus_client/src/forge.rs diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..e69de29b diff --git a/README.md b/README.md new file mode 100644 index 00000000..e69de29b diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index a52c207f..def9e0dc 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -3,6 +3,13 @@ name = "daedalus" version = "0.1.0" authors = ["Jai A "] edition = "2018" +license = "MIT" +description = "Utilities for querying and parsing Minecraft metadata" +repository = "https://github.com/modrinth/daedalus/" +include = ["Cargo.toml", "src/**/*.rs", "README.md", "LICENSE"] +keywords = ["minecraft", "launcher"] +categories = ["game-development", "api-bindings"] +readme = "README.md" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -14,4 +21,4 @@ chrono = { version = "0.4", features = ["serde"] } bytes = "1" thiserror = "1.0" tokio = { version = "1", features = ["full"] } -sha1 = { version = "0.6.0", features = ["std"]} \ No newline at end of file +sha1 = { version = "0.6.0", features = ["std"]} diff --git a/daedalus/README.md b/daedalus/README.md new file mode 100644 index 00000000..fb57d0f0 --- /dev/null +++ b/daedalus/README.md @@ -0,0 +1,6 @@ +# Daedalus + +Daedalus (the rust library) is a library providing model structs and methods for requesting and parsing things +from Minecraft and other mod loaders meta APIs. + +This is a work in progress! \ No newline at end of file diff --git a/daedalus/src/fabric.rs b/daedalus/src/fabric.rs index 052486ff..d63705a7 100644 --- a/daedalus/src/fabric.rs +++ b/daedalus/src/fabric.rs @@ -20,7 +20,7 @@ pub struct PartialVersionInfo { /// The latest time a file in this version was updated pub time: DateTime, /// The classpath to the main class to launch the game - pub main_class: String, + pub main_class: Option, /// Arguments passed to the game or JVM pub arguments: Option>>, /// Libraries that the version depends on @@ -51,7 +51,11 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> .into_iter() .chain(merge.libraries) .collect::>(), - main_class: partial.main_class, + main_class: if let Some(main_class) = partial.main_class { + main_class + } else { + merge.main_class + }, minecraft_arguments: merge.minecraft_arguments, minimum_launcher_version: merge.minimum_launcher_version, release_time: partial.release_time, diff --git a/daedalus/src/forge.rs b/daedalus/src/forge.rs new file mode 100644 index 00000000..4381f030 --- /dev/null +++ b/daedalus/src/forge.rs @@ -0,0 +1,21 @@ +use crate::{download_file, Error}; + +use std::collections::HashMap; + +/// The latest version of the format the model structs deserialize to +pub const CURRENT_FORMAT_VERSION: usize = 0; + +const DEFAULT_MAVEN_METADATA_URL: &str = + "https://files.minecraftforge.net/net/minecraftforge/forge/maven-metadata.json"; + +/// Fetches the forge maven metadata from the specified URL. If no URL is specified, the default is used. +/// Returns a hashmap specifying the versions of the forge mod loader +/// The hashmap key is a Minecraft version, and the value is the loader versions that work on +/// the specified Minecraft version +pub async fn fetch_maven_metadata( + url: Option<&str>, +) -> Result>, Error> { + Ok(serde_json::from_slice( + &download_file(url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), None).await?, + )?) +} diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index 92ff542f..4f314755 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -6,6 +6,8 @@ /// Models and methods for fetching metadata for the Fabric mod loader pub mod fabric; +/// Models and methods for fetching metadata for the Forge mod loader +pub mod forge; /// Models and methods for fetching metadata for Minecraft pub mod minecraft; @@ -56,18 +58,41 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { })?; Ok(format!( - "{}/{}/{}-{}.jar", + "{}/{}/{}/{}-{}.jar", package.replace(".", "/"), + name, version, name, version )) } +/// Downloads a file from specified mirrors +pub async fn download_file_mirrors( + base: &str, + mirrors: &[&str], + sha1: Option<&str>, +) -> Result { + if mirrors.is_empty() { + return Err(Error::ParseError("No mirrors provided!".to_string())); + } + + for (index, mirror) in mirrors.iter().enumerate() { + let result = download_file(&*format!("{}{}", mirror, base), sha1).await; + + if result.is_ok() || (result.is_err() && index == (mirrors.len() - 1)) { + return result; + } + } + + unreachable!() +} + /// Downloads a file with retry and checksum functionality pub async fn download_file(url: &str, sha1: Option<&str>) -> Result { let client = reqwest::Client::builder() .tcp_keepalive(Some(std::time::Duration::from_secs(10))) + .timeout(std::time::Duration::from_secs(30)) .build() .map_err(|err| Error::FetchError { inner: err, diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index e90e8f4d..d345f04c 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -69,7 +69,7 @@ pub struct LatestVersion { } #[derive(Serialize, Deserialize, Debug, Clone)] -/// Data of all game versions of Minecrafat +/// Data of all game versions of Minecraft pub struct VersionManifest { /// A struct containing the latest snapshot and release of the game pub latest: LatestVersion, @@ -154,7 +154,7 @@ pub struct LibraryDownloads { pub classifiers: Option>, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "snake_case")] /// The action a rule can follow pub enum RuleAction { @@ -164,7 +164,7 @@ pub enum RuleAction { Disallow, } -#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)] +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] #[serde(rename_all = "snake_case")] /// An enum representing the different types of operating systems pub enum Os { @@ -178,7 +178,7 @@ pub enum Os { Unknown, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// A rule which depends on what OS the user is on pub struct OsRule { #[serde(skip_serializing_if = "Option::is_none")] @@ -192,7 +192,7 @@ pub struct OsRule { pub arch: Option, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// A rule which depends on the toggled features of the launcher pub struct FeatureRule { #[serde(skip_serializing_if = "Option::is_none")] @@ -203,7 +203,7 @@ pub struct FeatureRule { pub has_demo_resolution: Option, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// A rule deciding whether a file is downloaded, an argument is used, etc. pub struct Rule { /// The action the rule takes @@ -244,9 +244,12 @@ pub struct Library { #[serde(skip_serializing_if = "Option::is_none")] /// Rules deciding whether the library should be downloaded or not pub rules: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + /// SHA1 Checksums for validating the library's integrity. Only present for forge libraries + pub checksums: Option>, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[serde(untagged)] /// A container for an argument or multiple arguments pub enum ArgumentValue { @@ -256,7 +259,7 @@ pub enum ArgumentValue { Many(Vec), } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[serde(untagged)] /// A command line argument passed to a program pub enum Argument { @@ -271,7 +274,7 @@ pub enum Argument { }, } -#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)] +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone, Copy)] #[serde(rename_all = "snake_case")] /// The type of argument pub enum ArgumentType { diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs new file mode 100644 index 00000000..e69de29b From 6528d3d7dac09b1f4e3ff42188d0790f826afe8b Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 17 Oct 2021 23:23:27 -0700 Subject: [PATCH 04/76] Legacy Forge Support (forgot to git add) --- LICENSE | 7 + README.md | 12 ++ daedalus_client/Cargo.toml | 5 + daedalus_client/src/fabric.rs | 214 ++++++++++++++----------------- daedalus_client/src/forge.rs | 210 ++++++++++++++++++++++++++++++ daedalus_client/src/main.rs | 32 ++++- daedalus_client/src/minecraft.rs | 30 ++--- 7 files changed, 376 insertions(+), 134 deletions(-) diff --git a/LICENSE b/LICENSE index e69de29b..6d21f198 100644 --- a/LICENSE +++ b/LICENSE @@ -0,0 +1,7 @@ +Copyright © 2021 Guavy LLC + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md index e69de29b..f0d5bd82 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,12 @@ +# Daedalus + +Daedalus is a powerful tool which queries and generates metadata for the Minecraft (and other games in the future!) game +and mod loaders for: +- Performance (Serving static files can be easily cached and is extremely quick) +- Ease for Launcher Devs (Metadata is served in an easy to query and use format) +- Reliability (Provides a versioning system which ensures no breakage with updates) + +Daedalus is currently a work in progress, but will support the original Minecraft data and reposting for all Forge and +Fabric artifacts. + +Once Daedalus is done, Modrinth will provide full documentation for how to query from it and use it for your own launcher! \ No newline at end of file diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 4f2fa019..33f1c063 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -12,10 +12,15 @@ tokio = { version = "1", features = ["full"] } futures = "0.3.17" dotenv = "0.15.0" log = "0.4.8" +serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" lazy_static = "1.4.0" thiserror = "1.0" reqwest = "0.11.4" +zip = "0.5.13" +semver = "1.0" +chrono = { version = "0.4", features = ["serde"] } +bytes = "1.1.0" rusoto_core = "0.47.0" rusoto_s3 = "0.47.0" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 890bd5be..809ec832 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -2,21 +2,19 @@ use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::fabric::PartialVersionInfo; use daedalus::minecraft::Library; use std::collections::HashMap; -use std::sync::{Arc, Mutex, RwLock}; +use std::sync::{Arc}; use std::time::{Duration, Instant}; +use futures::lock::Mutex; pub async fn retrieve_data() -> Result<(), Error> { let mut list = daedalus::fabric::fetch_fabric_versions(None).await?; - let loaders = RwLock::new(Vec::new()); - let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); - if let Some(latest) = list.loader.get(0) { + let loaders_mutex = Arc::new(Mutex::new(Vec::new())); + let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); + { - let mut loaders = match loaders.write() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; + let mut loaders = loaders_mutex.lock().await; loaders.push(latest.version.clone()); @@ -33,131 +31,115 @@ pub async fn retrieve_data() -> Result<(), Error> { .collect(); } - let mut versions = list - .game - .iter_mut() - .map(|game_version| { - let loaders = match loaders.read() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; + let mut version_futures = Vec::new(); - let visited_artifacts_mutex = Arc::clone(&visited_artifacts_mutex); - let game_version_mutex = Mutex::new(HashMap::new()); + for game_version in list.game.iter_mut() { + let visited_artifacts_mutex = Arc::clone(&visited_artifacts_mutex); + let game_version_mutex = Mutex::new(HashMap::new()); + let loaders_mutex = Arc::clone(&loaders_mutex); + version_futures.push(async move { + let versions = futures::future::try_join_all(loaders_mutex.lock().await.clone().into_iter().map( + |loader| async { + let version = daedalus::fabric::fetch_fabric_version( + &*game_version.version, + &*loader, + ) + .await + .expect(&*format!("{}, {}", game_version.version, loader)); - async move { - let versions = futures::future::try_join_all(loaders.clone().into_iter().map( - |loader| async { - let version = daedalus::fabric::fetch_fabric_version( - &*game_version.version, - &*loader, - ) - .await - .expect(&*format!("{}, {}", game_version.version, loader)); + Ok::<(String, PartialVersionInfo), Error>((loader, version)) + }, + )) + .await?; - Ok::<(String, PartialVersionInfo), Error>((loader, version)) - }, - )) - .await?; + futures::future::try_join_all(versions.into_iter().map( + |(loader, version)| async { + let libs = futures::future::try_join_all( + version.libraries.into_iter().map(|mut lib| async { + { + let mut visited_assets = visited_artifacts_mutex.lock().await; - futures::future::try_join_all(versions.into_iter().map( - |(loader, version)| async { - let libs = futures::future::try_join_all( - version.libraries.into_iter().map(|mut lib| async { - { - let mut visited_assets = - match visited_artifacts_mutex.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; + if visited_assets.contains(&lib.name) { + lib.url = Some(format_url("maven/")); - if visited_assets.contains(&lib.name) { - lib.url = Some(format_url("maven/")); - - return Ok(lib); - } else { - visited_assets.push(lib.name.clone()) - } + return Ok(lib); + } else { + visited_assets.push(lib.name.clone()) } + } - let artifact_path = - daedalus::get_path_from_artifact(&*lib.name)?; + let artifact_path = + daedalus::get_path_from_artifact(&*lib.name)?; - let artifact = daedalus::download_file( - &*format!( - "{}{}", - lib.url.unwrap_or_else(|| { - "https://maven.fabricmc.net/".to_string() - }), - artifact_path - ), - None, - ) - .await?; + let artifact = daedalus::download_file( + &*format!( + "{}{}", + lib.url.unwrap_or_else(|| { + "https://maven.fabricmc.net/".to_string() + }), + artifact_path + ), + None, + ) + .await?; - lib.url = Some(format_url("maven/")); + lib.url = Some(format_url("maven/")); - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - ) - .await?; + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + ) + .await?; - Ok::(lib) - }), - ) - .await?; + Ok::(lib) + }), + ) + .await?; - let version_path = format!( - "fabric/v{}/versions/{}-{}.json", - daedalus::fabric::CURRENT_FORMAT_VERSION, - version.inherits_from, - loader - ); + let version_path = format!( + "fabric/v{}/versions/{}-{}.json", + daedalus::fabric::CURRENT_FORMAT_VERSION, + version.inherits_from, + loader + ); - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&PartialVersionInfo { - arguments: version.arguments, - id: version.id, - main_class: version.main_class, - release_time: version.release_time, - time: version.time, - type_: version.type_, - inherits_from: version.inherits_from, - libraries: libs, - })?, - Some("application/json".to_string()), - ) - .await?; + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&PartialVersionInfo { + arguments: version.arguments, + id: version.id, + main_class: version.main_class, + release_time: version.release_time, + time: version.time, + type_: version.type_, + inherits_from: version.inherits_from, + libraries: libs, + })?, + Some("application/json".to_string()), + ) + .await?; - { - let mut game_version_map = match game_version_mutex.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; - game_version_map.insert(loader, format_url(&*version_path)); - } - - Ok::<(), Error>(()) - }, - )) - .await?; - - game_version.urls = Some( - match game_version_mutex.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), + { + let mut game_version_map = game_version_mutex.lock().await; + game_version_map.insert(loader, format_url(&*version_path)); } - .clone(), - ); - Ok::<(), Error>(()) - } - }) - .peekable(); + Ok::<(), Error>(()) + }, + )) + .await?; + game_version.urls = Some( + game_version_mutex.lock().await + .clone(), + ); + + Ok::<(), Error>(()) + }); + } + + let mut versions = version_futures.into_iter().peekable(); let mut chunk_index = 0; while versions.peek().is_some() { let now = Instant::now(); diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index e69de29b..61dd1fc1 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -0,0 +1,210 @@ +use crate::{format_url, upload_file_to_bucket, Error}; +use semver::{VersionReq, Version}; +use lazy_static::lazy_static; +use daedalus::download_file; +use std::io::Read; +use tokio::sync::{Mutex}; +use std::sync::{Arc}; +use daedalus::minecraft::{Library, VersionType, ArgumentType, Argument}; +use chrono::{DateTime, Utc}; +use serde::{Serialize, Deserialize}; +use daedalus::fabric::PartialVersionInfo; +use std::time::{Instant, Duration}; + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct ForgeInstallerProfileInstallDataV1 { + pub mirror_list: String, + pub target: String, + /// Path to the Forge universal library + pub file_path: String, + pub logo: String, + pub welcome: String, + pub version: String, + /// Maven coordinates of the Forge universal library + pub path: String, + pub profile_name: String, + pub minecraft: String, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct ForgeInstallerProfileManifestV1 { + pub id: String, + pub libraries: Vec, + pub main_class: Option, + pub minecraft_arguments: Option, + pub release_time: DateTime, + pub time: DateTime, + pub type_: VersionType, + pub assets: Option, + pub inherits_from: Option, + pub jar: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct ForgeInstallerProfileV1 { + pub install: ForgeInstallerProfileInstallDataV1, + pub version_info: ForgeInstallerProfileManifestV1, +} + +lazy_static! { + static ref FORGE_MANIFEST_V1_QUERY: VersionReq = VersionReq::parse(">=8.0.684, <23.5.2851").unwrap(); +} + +pub async fn retrieve_data() -> Result<(), Error> { + let maven_metadata = daedalus::forge::fetch_maven_metadata(None).await?; + + let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); + + let mut version_futures = Vec::new(); + + for (minecraft_version, loader_versions) in maven_metadata { + if let Some(loader_version_full) = loader_versions.into_iter().last() { + let loader_version = loader_version_full.split('-').into_iter().nth(1); + + if let Some(loader_version_raw) = loader_version { + // This is a dirty hack to get around Forge not complying with SemVer, but whatever + // Most of this is a hack anyways :( + // Works for all forge versions! + let split = loader_version_raw.split('.').collect::>(); + let loader_version =if split.len() >= 4 { + if split[0].parse::().unwrap() < 6 { + format!("{}.{}.{}", split[0], split[1], split[3]) + } else { + format!("{}.{}.{}", split[1], split[2], split[3]) + } + } else { + loader_version_raw.to_string() + }; + + if FORGE_MANIFEST_V1_QUERY.matches(&Version::parse(&*loader_version).unwrap()) { + version_futures.push(async { + let visited_assets = Arc::clone(&visited_assets_mutex); + async move { + println!("installer start {}", loader_version_full.clone()); + let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await.unwrap(); + + let reader = std::io::Cursor::new(&*bytes); + + if let Ok(mut archive) = zip::ZipArchive::new(reader) { + let install_profile = { + let mut install_profile = archive.by_name("install_profile.json").unwrap(); + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents).unwrap(); + + contents + }; + + let profile = serde_json::from_str::(&*install_profile).unwrap(); + + let forge_universal_bytes = { + let mut forge_universal_file = archive.by_name(&*profile.install.file_path).unwrap(); + let mut forge_universal = Vec::new(); + forge_universal_file.read_to_end(&mut forge_universal).unwrap(); + + bytes::Bytes::from(forge_universal) + }; + let forge_universal_path = profile.install.file_path.clone(); + + let now = Instant::now(); + let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { + if let Some(url) = lib.url { + { + let mut visited_assets = visited_assets.lock().await; + + if visited_assets.contains(&lib.name) { + lib.url = Some(format_url("maven/")); + + return Ok::(lib); + } else { + visited_assets.push(lib.name.clone()) + } + } + + let artifact_path = + daedalus::get_path_from_artifact(&*lib.name)?; + + let artifact = if lib.name == forge_universal_path { + forge_universal_bytes.clone() + } else { + let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; + + daedalus::download_file_mirrors( + &*artifact_path, + &mirrors, + None, + ) + .await? + }; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + ).await?; + } + + Ok::(lib) + })).await?; + + let elapsed = now.elapsed(); + println!("Elapsed lib DL: {:.2?}", elapsed); + + let new_profile = PartialVersionInfo { + id: profile.version_info.id, + inherits_from: profile.install.minecraft, + release_time: profile.version_info.release_time, + time: profile.version_info.time, + main_class: profile.version_info.main_class, + arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), + libraries: libs, + type_: profile.version_info.type_, + }; + + let version_path = format!( + "forge/v{}/versions/{}.json", + daedalus::forge::CURRENT_FORMAT_VERSION, + new_profile.id + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + ).await?; + } + + + Ok::<(), Error>(()) + }.await?; + + Ok::<(), Error>(()) + }); + } + } + } + } + + let mut versions = version_futures.into_iter().peekable(); + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); + + let chunk: Vec<_> = versions.by_ref().take(100).collect(); + futures::future::try_join_all(chunk).await?; + + std::thread::sleep(Duration::from_secs(1)); + + chunk_index += 1; + + let elapsed = now.elapsed(); + println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } + + Ok(()) +} \ No newline at end of file diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 7ac9a186..321ef10b 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -3,9 +3,11 @@ use rusoto_core::credential::StaticProvider; use rusoto_core::{HttpClient, Region, RusotoError}; use rusoto_s3::{PutObjectError, S3Client}; use rusoto_s3::{PutObjectRequest, S3}; +use std::time::Duration; mod fabric; mod minecraft; +mod forge; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -32,7 +34,35 @@ async fn main() { return; } - fabric::retrieve_data().await.unwrap(); + let mut timer = tokio::time::interval(Duration::from_secs(10 * 60)); + + loop { + timer.tick().await; + tokio::spawn( + async { + match fabric::retrieve_data().await { + Ok(..) => {} + Err(err) => error!("{:?}", err) + }; + } + ); + tokio::spawn( + async { + match minecraft::retrieve_data().await { + Ok(..) => {} + Err(err) => error!("{:?}", err) + }; + } + ); + tokio::spawn( + async { + match forge::retrieve_data().await { + Ok(..) => {} + Err(err) => error!("{:?}", err) + }; + } + ); + } } fn check_env_vars() -> bool { diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index ddcf286e..90b58806 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,7 +1,8 @@ use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::download_file; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc}; use std::time::{Duration, Instant}; +use futures::lock::Mutex; pub async fn retrieve_data() -> Result<(), Error> { let old_manifest = @@ -19,10 +20,13 @@ pub async fn retrieve_data() -> Result<(), Error> { let now = Instant::now(); - let mut versions = manifest + let mut version_futures = Vec::new(); + + for version in manifest .versions .iter_mut() - .map(|version| async { + { + version_futures.push(async { let old_version = if let Some(old_manifest) = &old_manifest { old_manifest.versions.iter().find(|x| x.id == version.id) } else { @@ -58,10 +62,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let assets_index_url = version_info.asset_index.url.clone(); { - let mut cloned_manifest = match cloned_manifest_mutex.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; + let mut cloned_manifest = cloned_manifest_mutex.lock().await; let position = cloned_manifest .versions @@ -79,10 +80,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let mut download_assets = false; { - let mut visited_assets = match visited_assets_mutex.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - }; + let mut visited_assets = visited_assets_mutex.lock().await; if !visited_assets.contains(&version_info.asset_index.id) { if let Some(assets_hash) = assets_hash { @@ -128,12 +126,13 @@ pub async fn retrieve_data() -> Result<(), Error> { Ok::<(), Error>(()) } - .await?; + .await?; Ok::<(), Error>(()) }) - .peekable(); + } + let mut versions = version_futures.into_iter().peekable(); let mut chunk_index = 0; while versions.peek().is_some() { let now = Instant::now(); @@ -154,10 +153,7 @@ pub async fn retrieve_data() -> Result<(), Error> { "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION ), - serde_json::to_vec(&*match cloned_manifest.lock() { - Ok(guard) => guard, - Err(poisoned) => poisoned.into_inner(), - })?, + serde_json::to_vec(&*cloned_manifest.lock().await)?, Some("application/json".to_string()), ) .await?; From 673658dfd24facd6745258b84e34e5ae004e6e3a Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 19 Oct 2021 23:08:44 -0700 Subject: [PATCH 05/76] Simplify mod loader manifests, start work on new forge profiles --- .idea/discord.xml | 7 + daedalus/src/fabric.rs | 136 -------------- daedalus/src/forge.rs | 21 --- daedalus/src/lib.rs | 6 +- daedalus/src/modded.rs | 115 ++++++++++++ daedalus_client/src/fabric.rs | 303 ++++++++++++++++++++----------- daedalus_client/src/forge.rs | 253 +++++++++++++++++--------- daedalus_client/src/main.rs | 46 +++-- daedalus_client/src/minecraft.rs | 11 +- 9 files changed, 521 insertions(+), 377 deletions(-) create mode 100644 .idea/discord.xml delete mode 100644 daedalus/src/fabric.rs delete mode 100644 daedalus/src/forge.rs create mode 100644 daedalus/src/modded.rs diff --git a/.idea/discord.xml b/.idea/discord.xml new file mode 100644 index 00000000..d8e95616 --- /dev/null +++ b/.idea/discord.xml @@ -0,0 +1,7 @@ + + + + + \ No newline at end of file diff --git a/daedalus/src/fabric.rs b/daedalus/src/fabric.rs deleted file mode 100644 index d63705a7..00000000 --- a/daedalus/src/fabric.rs +++ /dev/null @@ -1,136 +0,0 @@ -use crate::minecraft::{Argument, ArgumentType, Library, VersionInfo, VersionType}; -use crate::{download_file, Error}; -use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; - -/// The latest version of the format the model structs deserialize to -pub const CURRENT_FORMAT_VERSION: usize = 0; - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -/// A partial version returned by fabric meta -pub struct PartialVersionInfo { - /// The version ID of the version - pub id: String, - /// The version ID this partial version inherits from - pub inherits_from: String, - /// The time that the version was released - pub release_time: DateTime, - /// The latest time a file in this version was updated - pub time: DateTime, - /// The classpath to the main class to launch the game - pub main_class: Option, - /// Arguments passed to the game or JVM - pub arguments: Option>>, - /// Libraries that the version depends on - pub libraries: Vec, - #[serde(rename = "type")] - /// The type of version - pub type_: VersionType, -} - -/// Merges a partial version into a complete one -pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> VersionInfo { - VersionInfo { - arguments: if let Some(partial_args) = partial.arguments { - if let Some(merge_args) = merge.arguments { - Some(partial_args.into_iter().chain(merge_args).collect()) - } else { - Some(partial_args) - } - } else { - merge.arguments - }, - asset_index: merge.asset_index, - assets: merge.assets, - downloads: merge.downloads, - id: merge.id, - libraries: partial - .libraries - .into_iter() - .chain(merge.libraries) - .collect::>(), - main_class: if let Some(main_class) = partial.main_class { - main_class - } else { - merge.main_class - }, - minecraft_arguments: merge.minecraft_arguments, - minimum_launcher_version: merge.minimum_launcher_version, - release_time: partial.release_time, - time: partial.time, - type_: partial.type_, - } -} - -/// The default servers for fabric meta -pub const FABRIC_META_URL: &str = "https://meta.fabricmc.net/v2"; - -/// Fetches the manifest of a fabric loader version and game version -pub async fn fetch_fabric_version( - version_number: &str, - loader_version: &str, -) -> Result { - Ok(serde_json::from_slice( - &download_file( - &*format!( - "{}/versions/loader/{}/{}/profile/json", - FABRIC_META_URL, version_number, loader_version - ), - None, - ) - .await?, - )?) -} - -/// Fetches the manifest of a game version's URL -pub async fn fetch_fabric_game_version(url: &str) -> Result { - Ok(serde_json::from_slice(&download_file(url, None).await?)?) -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -/// Versions of fabric components -pub struct FabricVersions { - /// Versions of Minecraft that fabric supports - pub game: Vec, - /// Available versions of the fabric loader - pub loader: Vec, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -/// A version of Minecraft that fabric supports -pub struct FabricGameVersion { - /// The version number of the game - pub version: String, - /// Whether the Minecraft version is stable or not - pub stable: bool, - /// (Modrinth Provided) The URLs to download this version's profile with a loader - /// The key of the map is the loader version, and the value is the URL - pub urls: Option>, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -/// A version of the fabric loader -pub struct FabricLoaderVersion { - /// The separator to get the build number - pub separator: String, - /// The build number - pub build: u32, - /// The maven artifact - pub maven: String, - /// The version number of the fabric loader - pub version: String, - /// Whether the loader is stable or not - pub stable: bool, -} -/// Fetches the list of fabric versions -pub async fn fetch_fabric_versions(url: Option<&str>) -> Result { - Ok(serde_json::from_slice( - &download_file( - url.unwrap_or(&*format!("{}/versions", FABRIC_META_URL)), - None, - ) - .await?, - )?) -} diff --git a/daedalus/src/forge.rs b/daedalus/src/forge.rs deleted file mode 100644 index 4381f030..00000000 --- a/daedalus/src/forge.rs +++ /dev/null @@ -1,21 +0,0 @@ -use crate::{download_file, Error}; - -use std::collections::HashMap; - -/// The latest version of the format the model structs deserialize to -pub const CURRENT_FORMAT_VERSION: usize = 0; - -const DEFAULT_MAVEN_METADATA_URL: &str = - "https://files.minecraftforge.net/net/minecraftforge/forge/maven-metadata.json"; - -/// Fetches the forge maven metadata from the specified URL. If no URL is specified, the default is used. -/// Returns a hashmap specifying the versions of the forge mod loader -/// The hashmap key is a Minecraft version, and the value is the loader versions that work on -/// the specified Minecraft version -pub async fn fetch_maven_metadata( - url: Option<&str>, -) -> Result>, Error> { - Ok(serde_json::from_slice( - &download_file(url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), None).await?, - )?) -} diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index 4f314755..a4ac0059 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -4,12 +4,10 @@ #![warn(missing_docs, unused_import_braces, missing_debug_implementations)] -/// Models and methods for fetching metadata for the Fabric mod loader -pub mod fabric; -/// Models and methods for fetching metadata for the Forge mod loader -pub mod forge; /// Models and methods for fetching metadata for Minecraft pub mod minecraft; +/// Models and methods for fetching metadata for Minecraft mod loaders +pub mod modded; #[derive(thiserror::Error, Debug)] /// An error type representing possible errors when fetching metadata diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs new file mode 100644 index 00000000..cff051fa --- /dev/null +++ b/daedalus/src/modded.rs @@ -0,0 +1,115 @@ +use crate::{download_file, Error}; + +use crate::minecraft::{Argument, ArgumentType, Library, VersionInfo, VersionType}; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +/// The latest version of the format the fabric model structs deserialize to +pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0; +/// The latest version of the format the fabric model structs deserialize to +pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0; + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +/// A partial version returned by fabric meta +pub struct PartialVersionInfo { + /// The version ID of the version + pub id: String, + /// The version ID this partial version inherits from + pub inherits_from: String, + /// The time that the version was released + pub release_time: DateTime, + /// The latest time a file in this version was updated + pub time: DateTime, + /// The classpath to the main class to launch the game + pub main_class: Option, + /// Arguments passed to the game or JVM + pub arguments: Option>>, + /// Libraries that the version depends on + pub libraries: Vec, + #[serde(rename = "type")] + /// The type of version + pub type_: VersionType, +} + +/// Fetches the version manifest of a game version's URL +pub async fn fetch_partial_version(url: &str) -> Result { + Ok(serde_json::from_slice(&download_file(url, None).await?)?) +} + +/// Merges a partial version into a complete one +pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> VersionInfo { + VersionInfo { + arguments: if let Some(partial_args) = partial.arguments { + if let Some(merge_args) = merge.arguments { + Some(partial_args.into_iter().chain(merge_args).collect()) + } else { + Some(partial_args) + } + } else { + merge.arguments + }, + asset_index: merge.asset_index, + assets: merge.assets, + downloads: merge.downloads, + id: merge.id, + libraries: partial + .libraries + .into_iter() + .chain(merge.libraries) + .collect::>(), + main_class: if let Some(main_class) = partial.main_class { + main_class + } else { + merge.main_class + }, + minecraft_arguments: merge.minecraft_arguments, + minimum_launcher_version: merge.minimum_launcher_version, + release_time: partial.release_time, + time: partial.time, + type_: partial.type_, + } +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +/// A manifest containing information about a mod loader's versions +pub struct Manifest { + /// The game versions the mod loader supports + pub game_versions: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] +#[serde(rename_all = "camelCase")] +/// The version type of the loader +pub enum LoaderType { + /// The latest type is for experimental loader versions that may not be ready for normal use + Latest, + /// The stable type is for the most stable but recent loader version. For the forge mod loader, + /// this is never used + Stable, +} + +#[derive(Serialize, Deserialize, Debug)] +/// A game version of Minecraft +pub struct Version { + /// The minecraft version ID + pub id: String, + /// A map that contains loader versions for the game version + pub loaders: HashMap, +} + +#[derive(Serialize, Deserialize, Debug)] +/// A version of a Minecraft mod loader +pub struct LoaderVersion { + /// The version ID of the loader + pub id: String, + /// The URL of the version's manifest + pub url: String, +} + +/// Fetches the manifest of a mod loader +pub async fn fetch_manifest(url: &str) -> Result { + Ok(serde_json::from_slice(&download_file(url, None).await?)?) +} diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 809ec832..63f637f4 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -1,33 +1,47 @@ use crate::{format_url, upload_file_to_bucket, Error}; -use daedalus::fabric::PartialVersionInfo; +use daedalus::download_file; use daedalus::minecraft::Library; -use std::collections::HashMap; -use std::sync::{Arc}; -use std::time::{Duration, Instant}; +use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Version}; use futures::lock::Mutex; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::Arc; +use std::time::{Duration, Instant}; pub async fn retrieve_data() -> Result<(), Error> { - let mut list = daedalus::fabric::fetch_fabric_versions(None).await?; + let mut list = fetch_fabric_versions(None).await?; + let old_manifest = daedalus::modded::fetch_manifest(&*format!( + "fabric/v{}/manifest.json", + daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, + )) + .await + .ok(); + + let versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { + old_manifest.game_versions + } else { + Vec::new() + })); if let Some(latest) = list.loader.get(0) { - let loaders_mutex = Arc::new(Mutex::new(Vec::new())); + let loaders_mutex = Arc::new(Mutex::new(HashMap::new())); let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); { let mut loaders = loaders_mutex.lock().await; - loaders.push(latest.version.clone()); + loaders.insert(LoaderType::Latest, latest.version.clone()); if !latest.stable { if let Some(stable) = list.loader.iter().find(|x| x.stable) { - loaders.push(stable.version.clone()); + loaders.insert(LoaderType::Stable, stable.version.clone()); } } list.loader = list .loader .into_iter() - .filter(|x| loaders.contains(&x.version)) + .filter(|x| loaders.values().any(|val| val == &x.version)) .collect(); } @@ -35,105 +49,127 @@ pub async fn retrieve_data() -> Result<(), Error> { for game_version in list.game.iter_mut() { let visited_artifacts_mutex = Arc::clone(&visited_artifacts_mutex); - let game_version_mutex = Mutex::new(HashMap::new()); let loaders_mutex = Arc::clone(&loaders_mutex); + + let versions_mutex = Arc::clone(&versions); version_futures.push(async move { - let versions = futures::future::try_join_all(loaders_mutex.lock().await.clone().into_iter().map( - |loader| async { - let version = daedalus::fabric::fetch_fabric_version( - &*game_version.version, - &*loader, - ) - .await - .expect(&*format!("{}, {}", game_version.version, loader)); + let loader_version_mutex = Mutex::new(HashMap::new()); - Ok::<(String, PartialVersionInfo), Error>((loader, version)) - }, - )) - .await?; - - futures::future::try_join_all(versions.into_iter().map( - |(loader, version)| async { - let libs = futures::future::try_join_all( - version.libraries.into_iter().map(|mut lib| async { + let versions = + futures::future::try_join_all( + loaders_mutex.lock().await.clone().into_iter().map( + |(type_, loader)| async { { - let mut visited_assets = visited_artifacts_mutex.lock().await; - - if visited_assets.contains(&lib.name) { - lib.url = Some(format_url("maven/")); - - return Ok(lib); - } else { - visited_assets.push(lib.name.clone()) + if versions_mutex.lock().await.iter().any(|x| { + x.id == game_version.version + && x.loaders + .get(&type_) + .map(|x| x.id == loader) + .unwrap_or(false) + }) { + return Ok(None); } } - let artifact_path = - daedalus::get_path_from_artifact(&*lib.name)?; + let version = + fetch_fabric_version(&*game_version.version, &*loader).await?; - let artifact = daedalus::download_file( - &*format!( - "{}{}", - lib.url.unwrap_or_else(|| { - "https://maven.fabricmc.net/".to_string() - }), - artifact_path - ), - None, - ) - .await?; + Ok::, Error>(Some( + (type_, loader, version), + )) + }, + ), + ) + .await? + .into_iter() + .flatten(); - lib.url = Some(format_url("maven/")); + futures::future::try_join_all(versions.map(|(type_, loader, version)| async { + let libs = futures::future::try_join_all(version.libraries.into_iter().map( + |mut lib| async { + { + let mut visited_assets = visited_artifacts_mutex.lock().await; - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - ) - .await?; + if visited_assets.contains(&lib.name) { + lib.url = Some(format_url("maven/")); - Ok::(lib) - }), - ) - .await?; + return Ok(lib); + } else { + visited_assets.push(lib.name.clone()) + } + } - let version_path = format!( - "fabric/v{}/versions/{}-{}.json", - daedalus::fabric::CURRENT_FORMAT_VERSION, - version.inherits_from, - loader + let artifact_path = daedalus::get_path_from_artifact(&*lib.name)?; + + let artifact = daedalus::download_file( + &*format!( + "{}{}", + lib.url.unwrap_or_else(|| { + "https://maven.fabricmc.net/".to_string() + }), + artifact_path + ), + None, + ) + .await?; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + ) + .await?; + + Ok::(lib) + }, + )) + .await?; + + let version_path = format!( + "fabric/v{}/versions/{}-{}.json", + daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, + version.inherits_from, + loader + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&PartialVersionInfo { + arguments: version.arguments, + id: version.id, + main_class: version.main_class, + release_time: version.release_time, + time: version.time, + type_: version.type_, + inherits_from: version.inherits_from, + libraries: libs, + })?, + Some("application/json".to_string()), + ) + .await?; + + { + let mut loader_version_map = loader_version_mutex.lock().await; + loader_version_map.insert( + type_, + LoaderVersion { + id: loader, + url: format_url(&*version_path), + }, ); + } - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&PartialVersionInfo { - arguments: version.arguments, - id: version.id, - main_class: version.main_class, - release_time: version.release_time, - time: version.time, - type_: version.type_, - inherits_from: version.inherits_from, - libraries: libs, - })?, - Some("application/json".to_string()), - ) - .await?; - - { - let mut game_version_map = game_version_mutex.lock().await; - game_version_map.insert(loader, format_url(&*version_path)); - } - - Ok::<(), Error>(()) - }, - )) + Ok::<(), Error>(()) + })) .await?; - game_version.urls = Some( - game_version_mutex.lock().await - .clone(), - ); + let mut versions = versions_mutex.lock().await; + versions.push(Version { + id: game_version.version.clone(), + loaders: loader_version_mutex.into_inner(), + }); Ok::<(), Error>(()) }); @@ -156,15 +192,80 @@ pub async fn retrieve_data() -> Result<(), Error> { } } - upload_file_to_bucket( - format!( - "fabric/v{}/manifest.json", - daedalus::fabric::CURRENT_FORMAT_VERSION, - ), - serde_json::to_vec(&list)?, - Some("application/json".to_string()), - ) - .await?; + if let Ok(versions) = Arc::try_unwrap(versions) { + upload_file_to_bucket( + format!( + "fabric/v{}/manifest.json", + daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, + ), + serde_json::to_vec(&Manifest { + game_versions: versions.into_inner(), + })?, + Some("application/json".to_string()), + ) + .await?; + } Ok(()) } + +const FABRIC_META_URL: &str = "https://meta.fabricmc.net/v2"; + +async fn fetch_fabric_version( + version_number: &str, + loader_version: &str, +) -> Result { + Ok(serde_json::from_slice( + &download_file( + &*format!( + "{}/versions/loader/{}/{}/profile/json", + FABRIC_META_URL, version_number, loader_version + ), + None, + ) + .await?, + )?) +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// Versions of fabric components +struct FabricVersions { + /// Versions of Minecraft that fabric supports + pub game: Vec, + /// Available versions of the fabric loader + pub loader: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// A version of Minecraft that fabric supports +struct FabricGameVersion { + /// The version number of the game + pub version: String, + /// Whether the Minecraft version is stable or not + pub stable: bool, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// A version of the fabric loader +struct FabricLoaderVersion { + /// The separator to get the build number + pub separator: String, + /// The build number + pub build: u32, + /// The maven artifact + pub maven: String, + /// The version number of the fabric loader + pub version: String, + /// Whether the loader is stable or not + pub stable: bool, +} +/// Fetches the list of fabric versions +async fn fetch_fabric_versions(url: Option<&str>) -> Result { + Ok(serde_json::from_slice( + &download_file( + url.unwrap_or(&*format!("{}/versions", FABRIC_META_URL)), + None, + ) + .await?, + )?) +} diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 61dd1fc1..f06d23b0 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -1,60 +1,39 @@ use crate::{format_url, upload_file_to_bucket, Error}; -use semver::{VersionReq, Version}; -use lazy_static::lazy_static; -use daedalus::download_file; -use std::io::Read; -use tokio::sync::{Mutex}; -use std::sync::{Arc}; -use daedalus::minecraft::{Library, VersionType, ArgumentType, Argument}; use chrono::{DateTime, Utc}; -use serde::{Serialize, Deserialize}; -use daedalus::fabric::PartialVersionInfo; -use std::time::{Instant, Duration}; - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -struct ForgeInstallerProfileInstallDataV1 { - pub mirror_list: String, - pub target: String, - /// Path to the Forge universal library - pub file_path: String, - pub logo: String, - pub welcome: String, - pub version: String, - /// Maven coordinates of the Forge universal library - pub path: String, - pub profile_name: String, - pub minecraft: String, -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -struct ForgeInstallerProfileManifestV1 { - pub id: String, - pub libraries: Vec, - pub main_class: Option, - pub minecraft_arguments: Option, - pub release_time: DateTime, - pub time: DateTime, - pub type_: VersionType, - pub assets: Option, - pub inherits_from: Option, - pub jar: Option, -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -struct ForgeInstallerProfileV1 { - pub install: ForgeInstallerProfileInstallDataV1, - pub version_info: ForgeInstallerProfileManifestV1, -} +use daedalus::download_file; +use daedalus::minecraft::{Argument, ArgumentType, Library, VersionType}; +use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo}; +use lazy_static::lazy_static; +use semver::{Version, VersionReq}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::io::Read; +use std::sync::Arc; +use std::time::{Duration, Instant}; +use tokio::sync::Mutex; lazy_static! { - static ref FORGE_MANIFEST_V1_QUERY: VersionReq = VersionReq::parse(">=8.0.684, <23.5.2851").unwrap(); + static ref FORGE_MANIFEST_V1_QUERY: VersionReq = + VersionReq::parse(">=8.0.684, <23.5.2851").unwrap(); + static ref FORGE_MANIFEST_V2_QUERY: VersionReq = + VersionReq::parse(">=23.5.2851, <37.0.0").unwrap(); + static ref FORGE_MANIFEST_V3_QUERY: VersionReq = VersionReq::parse(">=37.0.0").unwrap(); } pub async fn retrieve_data() -> Result<(), Error> { - let maven_metadata = daedalus::forge::fetch_maven_metadata(None).await?; + let maven_metadata = fetch_maven_metadata(None).await?; + let old_manifest = daedalus::modded::fetch_manifest(&*format!( + "forge/v{}/manifest.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + )) + .await + .ok(); + + let versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { + old_manifest.game_versions + } else { + Vec::new() + })); let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); @@ -69,8 +48,8 @@ pub async fn retrieve_data() -> Result<(), Error> { // Most of this is a hack anyways :( // Works for all forge versions! let split = loader_version_raw.split('.').collect::>(); - let loader_version =if split.len() >= 4 { - if split[0].parse::().unwrap() < 6 { + let loader_version = if split.len() >= 4 { + if split[0].parse::().unwrap_or(0) < 6 { format!("{}.{}.{}", split[0], split[1], split[3]) } else { format!("{}.{}.{}", split[1], split[2], split[3]) @@ -79,31 +58,46 @@ pub async fn retrieve_data() -> Result<(), Error> { loader_version_raw.to_string() }; - if FORGE_MANIFEST_V1_QUERY.matches(&Version::parse(&*loader_version).unwrap()) { - version_futures.push(async { - let visited_assets = Arc::clone(&visited_assets_mutex); - async move { - println!("installer start {}", loader_version_full.clone()); - let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await.unwrap(); + let version = Version::parse(&*loader_version)?; - let reader = std::io::Cursor::new(&*bytes); + version_futures.push(async { + let versions_mutex = Arc::clone(&versions); + let visited_assets = Arc::clone(&visited_assets_mutex); + async move { + { + if versions_mutex.lock().await.iter().any(|x| { + x.id == minecraft_version + && x.loaders + .get(&LoaderType::Latest) + .map(|x| x.id == loader_version_full) + .unwrap_or(false) + }) { + return Ok(()); + } + } - if let Ok(mut archive) = zip::ZipArchive::new(reader) { + println!("installer start {}", loader_version_full.clone()); + let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; + + let reader = std::io::Cursor::new(&*bytes); + + if let Ok(mut archive) = zip::ZipArchive::new(reader) { + if FORGE_MANIFEST_V1_QUERY.matches(&version) { let install_profile = { - let mut install_profile = archive.by_name("install_profile.json").unwrap(); + let mut install_profile = archive.by_name("install_profile.json")?; let mut contents = String::new(); - install_profile.read_to_string(&mut contents).unwrap(); + install_profile.read_to_string(&mut contents)?; contents }; - let profile = serde_json::from_str::(&*install_profile).unwrap(); + let profile = serde_json::from_str::(&*install_profile)?; let forge_universal_bytes = { - let mut forge_universal_file = archive.by_name(&*profile.install.file_path).unwrap(); + let mut forge_universal_file = archive.by_name(&*profile.install.file_path)?; let mut forge_universal = Vec::new(); - forge_universal_file.read_to_end(&mut forge_universal).unwrap(); + forge_universal_file.read_to_end(&mut forge_universal)?; bytes::Bytes::from(forge_universal) }; @@ -168,7 +162,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let version_path = format!( "forge/v{}/versions/{}.json", - daedalus::forge::CURRENT_FORMAT_VERSION, + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, new_profile.id ); @@ -177,34 +171,127 @@ pub async fn retrieve_data() -> Result<(), Error> { serde_json::to_vec(&new_profile)?, Some("application/json".to_string()), ).await?; + + let mut map = HashMap::new(); + map.insert(LoaderType::Latest, LoaderVersion { + id: loader_version_full, + url: format_url(&*version_path) + }); + versions_mutex.lock().await.push(daedalus::modded::Version { + id: minecraft_version, + loaders: map + }) + } else if FORGE_MANIFEST_V2_QUERY.matches(&version) { + let install_profile = { + let mut install_profile = archive.by_name("install_profile.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + contents + }; + } else if FORGE_MANIFEST_V3_QUERY.matches(&version) { + } - - - Ok::<(), Error>(()) - }.await?; + } Ok::<(), Error>(()) - }); - } + }.await?; + + Ok::<(), Error>(()) + }); } } } - let mut versions = version_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions.peek().is_some() { - let now = Instant::now(); + { + let mut versions_peek = version_futures.into_iter().peekable(); + let mut chunk_index = 0; + while versions_peek.peek().is_some() { + let now = Instant::now(); - let chunk: Vec<_> = versions.by_ref().take(100).collect(); - futures::future::try_join_all(chunk).await?; + let chunk: Vec<_> = versions_peek.by_ref().take(100).collect(); + futures::future::try_join_all(chunk).await?; - std::thread::sleep(Duration::from_secs(1)); + std::thread::sleep(Duration::from_secs(1)); - chunk_index += 1; + chunk_index += 1; - let elapsed = now.elapsed(); - println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + let elapsed = now.elapsed(); + println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } + } + + if let Ok(versions) = Arc::try_unwrap(versions) { + upload_file_to_bucket( + format!( + "forge/v{}/manifest.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + ), + serde_json::to_vec(&Manifest { + game_versions: versions.into_inner(), + })?, + Some("application/json".to_string()), + ) + .await?; } Ok(()) -} \ No newline at end of file +} + +const DEFAULT_MAVEN_METADATA_URL: &str = + "https://files.minecraftforge.net/net/minecraftforge/forge/maven-metadata.json"; + +/// Fetches the forge maven metadata from the specified URL. If no URL is specified, the default is used. +/// Returns a hashmap specifying the versions of the forge mod loader +/// The hashmap key is a Minecraft version, and the value is the loader versions that work on +/// the specified Minecraft version +pub async fn fetch_maven_metadata( + url: Option<&str>, +) -> Result>, Error> { + Ok(serde_json::from_slice( + &download_file(url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), None).await?, + )?) +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct ForgeInstallerProfileInstallDataV1 { + pub mirror_list: String, + pub target: String, + /// Path to the Forge universal library + pub file_path: String, + pub logo: String, + pub welcome: String, + pub version: String, + /// Maven coordinates of the Forge universal library + pub path: String, + pub profile_name: String, + pub minecraft: String, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct ForgeInstallerProfileManifestV1 { + pub id: String, + pub libraries: Vec, + pub main_class: Option, + pub minecraft_arguments: Option, + pub release_time: DateTime, + pub time: DateTime, + pub type_: VersionType, + pub assets: Option, + pub inherits_from: Option, + pub jar: Option, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct ForgeInstallerProfileV1 { + pub install: ForgeInstallerProfileInstallDataV1, + pub version_info: ForgeInstallerProfileManifestV1, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct ForgeInstallerProfileV2 {} diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 321ef10b..14f4fb13 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -6,8 +6,8 @@ use rusoto_s3::{PutObjectRequest, S3}; use std::time::Duration; mod fabric; -mod minecraft; mod forge; +mod minecraft; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -24,6 +24,12 @@ pub enum Error { inner: RusotoError, file: String, }, + #[error("Error while parsing version as semver: {0}")] + SemVerError(#[from] semver::Error), + #[error("Error while reading zip file: {0}")] + ZipError(#[from] zip::result::ZipError), + #[error("Error while reading zip file: {0}")] + IoError(#[from] std::io::Error), } #[tokio::main] @@ -38,30 +44,20 @@ async fn main() { loop { timer.tick().await; - tokio::spawn( - async { - match fabric::retrieve_data().await { - Ok(..) => {} - Err(err) => error!("{:?}", err) - }; - } - ); - tokio::spawn( - async { - match minecraft::retrieve_data().await { - Ok(..) => {} - Err(err) => error!("{:?}", err) - }; - } - ); - tokio::spawn( - async { - match forge::retrieve_data().await { - Ok(..) => {} - Err(err) => error!("{:?}", err) - }; - } - ); + tokio::spawn(async { + match fabric::retrieve_data().await { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; + match minecraft::retrieve_data().await { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; + match forge::retrieve_data().await { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; + }); } } diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 90b58806..07907c4f 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,8 +1,8 @@ use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::download_file; -use std::sync::{Arc}; -use std::time::{Duration, Instant}; use futures::lock::Mutex; +use std::sync::Arc; +use std::time::{Duration, Instant}; pub async fn retrieve_data() -> Result<(), Error> { let old_manifest = @@ -22,10 +22,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let mut version_futures = Vec::new(); - for version in manifest - .versions - .iter_mut() - { + for version in manifest.versions.iter_mut() { version_futures.push(async { let old_version = if let Some(old_manifest) = &old_manifest { old_manifest.versions.iter().find(|x| x.id == version.id) @@ -126,7 +123,7 @@ pub async fn retrieve_data() -> Result<(), Error> { Ok::<(), Error>(()) } - .await?; + .await?; Ok::<(), Error>(()) }) From d8332a27e5c8fb4ccad70695b03534c8312010a5 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 24 Oct 2021 14:25:24 -0700 Subject: [PATCH 06/76] Finish newer forge versions --- .env | 2 + daedalus/src/lib.rs | 2 +- daedalus/src/modded.rs | 35 +++- daedalus_client/Cargo.toml | 1 + daedalus_client/src/fabric.rs | 4 +- daedalus_client/src/forge.rs | 327 +++++++++++++++++++++---------- daedalus_client/src/main.rs | 2 + daedalus_client/src/minecraft.rs | 2 +- 8 files changed, 266 insertions(+), 109 deletions(-) diff --git a/.env b/.env index 4e9b803b..56db8578 100644 --- a/.env +++ b/.env @@ -1,3 +1,5 @@ +RUST_LOG=info,error + BASE_URL=https://modrinth-cdn-staging.nyc3.digitaloceanspaces.com BASE_FOLDER=gamedata diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index a4ac0059..5e0f5e42 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -90,7 +90,7 @@ pub async fn download_file_mirrors( pub async fn download_file(url: &str, sha1: Option<&str>) -> Result { let client = reqwest::Client::builder() .tcp_keepalive(Some(std::time::Duration::from_secs(10))) - .timeout(std::time::Duration::from_secs(30)) + .timeout(std::time::Duration::from_secs(15)) .build() .map_err(|err| Error::FetchError { inner: err, diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index cff051fa..c631b619 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -10,6 +10,15 @@ pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0; /// The latest version of the format the fabric model structs deserialize to pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0; +/// A data variable entry that depends on the side of the installation +#[derive(Serialize, Deserialize, Debug)] +pub struct SidedDataEntry { + /// The value on the client + pub client: String, + /// The value on the server + pub server: String, +} + #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// A partial version returned by fabric meta @@ -31,6 +40,26 @@ pub struct PartialVersionInfo { #[serde(rename = "type")] /// The type of version pub type_: VersionType, + /// (Forge-only) + pub data: Option>, + /// (Forge-only) The list of processors to run after downloading the files + pub processors: Option>, +} + +/// A processor to be ran after downloading the files +#[derive(Serialize, Deserialize, Debug)] +pub struct Processor { + /// Maven coordinates for the JAR library of this processor. + pub jar: String, + /// Maven coordinates for all the libraries that must be included in classpath when running this processor. + pub classpath: Vec, + /// Arguments for this processor. + pub args: Vec, + /// Represents a map of outputs. Keys and values can be data values + pub outputs: Option>, + /// Which sides this processor shall be ran on. + /// Valid values: client, server, extract + pub sides: Option>, } /// Fetches the version manifest of a game version's URL @@ -72,7 +101,7 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> } } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] /// A manifest containing information about a mod loader's versions pub struct Manifest { @@ -91,7 +120,7 @@ pub enum LoaderType { Stable, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// A game version of Minecraft pub struct Version { /// The minecraft version ID @@ -100,7 +129,7 @@ pub struct Version { pub loaders: HashMap, } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// A version of a Minecraft mod loader pub struct LoaderVersion { /// The version ID of the loader diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 33f1c063..f36f6f3b 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -12,6 +12,7 @@ tokio = { version = "1", features = ["full"] } futures = "0.3.17" dotenv = "0.15.0" log = "0.4.8" +env_logger="0.9.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" lazy_static = "1.4.0" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 63f637f4..5ac71f16 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -145,6 +145,8 @@ pub async fn retrieve_data() -> Result<(), Error> { type_: version.type_, inherits_from: version.inherits_from, libraries: libs, + processors: None, + data: None })?, Some("application/json".to_string()), ) @@ -183,7 +185,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let chunk: Vec<_> = versions.by_ref().take(10).collect(); futures::future::try_join_all(chunk).await?; - std::thread::sleep(Duration::from_secs(1)); + tokio::time::sleep(Duration::from_secs(1)).await; chunk_index += 1; diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index f06d23b0..af31f015 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -2,7 +2,7 @@ use crate::{format_url, upload_file_to_bucket, Error}; use chrono::{DateTime, Utc}; use daedalus::download_file; use daedalus::minecraft::{Argument, ArgumentType, Library, VersionType}; -use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo}; +use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry}; use lazy_static::lazy_static; use semver::{Version, VersionReq}; use serde::{Deserialize, Serialize}; @@ -15,17 +15,21 @@ use tokio::sync::Mutex; lazy_static! { static ref FORGE_MANIFEST_V1_QUERY: VersionReq = VersionReq::parse(">=8.0.684, <23.5.2851").unwrap(); - static ref FORGE_MANIFEST_V2_QUERY: VersionReq = - VersionReq::parse(">=23.5.2851, <37.0.0").unwrap(); + + static ref FORGE_MANIFEST_V2_QUERY_P1: VersionReq = + VersionReq::parse(">=23.5.2851, <31.2.52").unwrap(); + static ref FORGE_MANIFEST_V2_QUERY_P2: VersionReq = + VersionReq::parse(">=32.0.1, <37.0.0").unwrap(); + static ref FORGE_MANIFEST_V3_QUERY: VersionReq = VersionReq::parse(">=37.0.0").unwrap(); } pub async fn retrieve_data() -> Result<(), Error> { let maven_metadata = fetch_maven_metadata(None).await?; - let old_manifest = daedalus::modded::fetch_manifest(&*format!( + let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!( "forge/v{}/manifest.json", daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - )) + ))) .await .ok(); @@ -40,7 +44,9 @@ pub async fn retrieve_data() -> Result<(), Error> { let mut version_futures = Vec::new(); for (minecraft_version, loader_versions) in maven_metadata { - if let Some(loader_version_full) = loader_versions.into_iter().last() { + let mut loaders = Vec::new(); + + for loader_version_full in loader_versions { let loader_version = loader_version_full.split('-').into_iter().nth(1); if let Some(loader_version_raw) = loader_version { @@ -60,57 +66,173 @@ pub async fn retrieve_data() -> Result<(), Error> { let version = Version::parse(&*loader_version)?; - version_futures.push(async { - let versions_mutex = Arc::clone(&versions); - let visited_assets = Arc::clone(&visited_assets_mutex); - async move { - { - if versions_mutex.lock().await.iter().any(|x| { - x.id == minecraft_version - && x.loaders - .get(&LoaderType::Latest) - .map(|x| x.id == loader_version_full) - .unwrap_or(false) - }) { - return Ok(()); - } + if FORGE_MANIFEST_V1_QUERY.matches(&version) || FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { + loaders.push((loader_version_full, version)) + } + } + } + + if let Some((loader_version_full, version)) = loaders.into_iter().last() { + version_futures.push(async { + let versions_mutex = Arc::clone(&versions); + let visited_assets = Arc::clone(&visited_assets_mutex); + async move { + { + if versions_mutex.lock().await.iter().any(|x| { + x.id == minecraft_version + && x.loaders + .get(&LoaderType::Latest) + .map(|x| x.id == loader_version_full) + .unwrap_or(false) + }) { + return Ok(()); } + } - println!("installer start {}", loader_version_full.clone()); - let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; + println!("installer start {}", loader_version_full.clone()); + let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; - let reader = std::io::Cursor::new(&*bytes); + let reader = std::io::Cursor::new(&*bytes); - if let Ok(mut archive) = zip::ZipArchive::new(reader) { - if FORGE_MANIFEST_V1_QUERY.matches(&version) { - let install_profile = { - let mut install_profile = archive.by_name("install_profile.json")?; + if let Ok(mut archive) = zip::ZipArchive::new(reader) { + if FORGE_MANIFEST_V1_QUERY.matches(&version) { + let profile = { + let mut install_profile = archive.by_name("install_profile.json")?; - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; - contents - }; + serde_json::from_str::(&*contents)? + }; - let profile = serde_json::from_str::(&*install_profile)?; + let forge_universal_bytes = { + let mut forge_universal_file = archive.by_name(&*profile.install.file_path)?; + let mut forge_universal = Vec::new(); + forge_universal_file.read_to_end(&mut forge_universal)?; - let forge_universal_bytes = { - let mut forge_universal_file = archive.by_name(&*profile.install.file_path)?; + bytes::Bytes::from(forge_universal) + }; + let forge_universal_path = profile.install.file_path.clone(); + + let now = Instant::now(); + let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { + if let Some(url) = lib.url { + { + let mut visited_assets = visited_assets.lock().await; + + if visited_assets.contains(&lib.name) { + lib.url = Some(format_url("maven/")); + + return Ok::(lib); + } else { + visited_assets.push(lib.name.clone()) + } + } + + let artifact_path = + daedalus::get_path_from_artifact(&*lib.name)?; + + let artifact = if lib.name == forge_universal_path { + forge_universal_bytes.clone() + } else { + let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; + + daedalus::download_file_mirrors( + &*artifact_path, + &mirrors, + None, + ) + .await? + }; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + ).await?; + } + + Ok::(lib) + })).await?; + + let elapsed = now.elapsed(); + println!("Elapsed lib DL: {:.2?}", elapsed); + + let new_profile = PartialVersionInfo { + id: profile.version_info.id, + inherits_from: profile.install.minecraft, + release_time: profile.version_info.release_time, + time: profile.version_info.time, + main_class: profile.version_info.main_class, + arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), + libraries: libs, + type_: profile.version_info.type_, + data: None, + processors: None + }; + + let version_path = format!( + "forge/v{}/versions/{}.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + new_profile.id + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + ).await?; + + let mut map = HashMap::new(); + map.insert(LoaderType::Latest, LoaderVersion { + id: loader_version_full, + url: format_url(&*version_path) + }); + versions_mutex.lock().await.push(daedalus::modded::Version { + id: minecraft_version, + loaders: map + }) + } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { + let profile = { + let mut install_profile = archive.by_name("install_profile.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + serde_json::from_str::(&*contents)? + }; + + let version_info = { + let mut install_profile = archive.by_name("version.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + serde_json::from_str::(&*contents)? + }; + + let forge_universal_bytes = { + if let Some(path) = &profile.path { + let mut forge_universal_file = archive.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*path)?))?; let mut forge_universal = Vec::new(); forge_universal_file.read_to_end(&mut forge_universal)?; - bytes::Bytes::from(forge_universal) - }; - let forge_universal_path = profile.install.file_path.clone(); + Some(bytes::Bytes::from(forge_universal)) + } else { + None + } + }; - let now = Instant::now(); - let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { - if let Some(url) = lib.url { + let now = Instant::now(); + let libs = futures::future::try_join_all(profile.libraries.into_iter().chain(version_info.libraries).map(|mut lib| async { + if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { { let mut visited_assets = visited_assets.lock().await; if visited_assets.contains(&lib.name) { - lib.url = Some(format_url("maven/")); + artifact.url = format_url(&*format!("maven/{}", artifact.path)); return Ok::(lib); } else { @@ -121,86 +243,74 @@ pub async fn retrieve_data() -> Result<(), Error> { let artifact_path = daedalus::get_path_from_artifact(&*lib.name)?; - let artifact = if lib.name == forge_universal_path { - forge_universal_bytes.clone() + let artifact_bytes = if &*artifact.url == "" { + forge_universal_bytes.clone().unwrap_or_default() } else { - let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; - - daedalus::download_file_mirrors( - &*artifact_path, - &mirrors, - None, + daedalus::download_file( + &*artifact.url, + Some(&*artifact.sha1), ) .await? }; - lib.url = Some(format_url("maven/")); + artifact.url = format_url(&*format!("maven/{}", artifact.path)); upload_file_to_bucket( format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), + artifact_bytes.to_vec(), Some("application/java-archive".to_string()), ).await?; } + } - Ok::(lib) - })).await?; + Ok::(lib) + })).await?; - let elapsed = now.elapsed(); - println!("Elapsed lib DL: {:.2?}", elapsed); + let elapsed = now.elapsed(); + println!("Elapsed lib DL: {:.2?}", elapsed); - let new_profile = PartialVersionInfo { - id: profile.version_info.id, - inherits_from: profile.install.minecraft, - release_time: profile.version_info.release_time, - time: profile.version_info.time, - main_class: profile.version_info.main_class, - arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), - libraries: libs, - type_: profile.version_info.type_, - }; + let new_profile = PartialVersionInfo { + id: version_info.id, + inherits_from: version_info.inherits_from, + release_time: version_info.release_time, + time: version_info.time, + main_class: version_info.main_class, + arguments: version_info.arguments, + libraries: libs, + type_: version_info.type_, + data: Some(profile.data), + processors: Some(profile.processors), + }; - let version_path = format!( - "forge/v{}/versions/{}.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - new_profile.id - ); + let version_path = format!( + "forge/v{}/versions/{}.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + new_profile.id + ); - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&new_profile)?, - Some("application/json".to_string()), - ).await?; + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + ).await?; - let mut map = HashMap::new(); - map.insert(LoaderType::Latest, LoaderVersion { - id: loader_version_full, - url: format_url(&*version_path) - }); - versions_mutex.lock().await.push(daedalus::modded::Version { - id: minecraft_version, - loaders: map - }) - } else if FORGE_MANIFEST_V2_QUERY.matches(&version) { - let install_profile = { - let mut install_profile = archive.by_name("install_profile.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - contents - }; - } else if FORGE_MANIFEST_V3_QUERY.matches(&version) { - - } + let mut map = HashMap::new(); + map.insert(LoaderType::Latest, LoaderVersion { + id: loader_version_full, + url: format_url(&*version_path) + }); + versions_mutex.lock().await.push(daedalus::modded::Version { + id: minecraft_version, + loaders: map + }) } - - Ok::<(), Error>(()) - }.await?; + } Ok::<(), Error>(()) - }); - } + }.await?; + + Ok::<(), Error>(()) + }); } } @@ -208,12 +318,13 @@ pub async fn retrieve_data() -> Result<(), Error> { let mut versions_peek = version_futures.into_iter().peekable(); let mut chunk_index = 0; while versions_peek.peek().is_some() { + println!("Chunk {} Start", chunk_index); let now = Instant::now(); let chunk: Vec<_> = versions_peek.by_ref().take(100).collect(); futures::future::try_join_all(chunk).await?; - std::thread::sleep(Duration::from_secs(1)); + tokio::time::sleep(Duration::from_secs(1)).await; chunk_index += 1; @@ -294,4 +405,14 @@ struct ForgeInstallerProfileV1 { #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] -struct ForgeInstallerProfileV2 {} +struct ForgeInstallerProfileV2 { + pub spec: i32, + pub profile: String, + pub version: String, + pub json: String, + pub path: Option, + pub minecraft: String, + pub data: HashMap, + pub libraries: Vec, + pub processors: Vec, +} \ No newline at end of file diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 14f4fb13..7d051142 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -34,6 +34,8 @@ pub enum Error { #[tokio::main] async fn main() { + env_logger::init(); + if check_env_vars() { error!("Some environment variables are missing!"); diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 07907c4f..c6062dbb 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -137,7 +137,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let chunk: Vec<_> = versions.by_ref().take(100).collect(); futures::future::try_join_all(chunk).await?; - std::thread::sleep(Duration::from_secs(1)); + tokio::time::sleep(Duration::from_secs(1)).await; chunk_index += 1; From 5218543c5864b22f6f002cbf84bed0eaba39d37b Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 24 Oct 2021 16:10:38 -0700 Subject: [PATCH 07/76] Add GitHub Actions --- .github/workflows/clippy.yml | 0 .github/workflows/docker.yml | 0 .github/workflows/publish.yml | 0 .github/workflows/rust.yml | 0 Dockerfile | 0 5 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 .github/workflows/clippy.yml create mode 100644 .github/workflows/docker.yml create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/rust.yml create mode 100644 Dockerfile diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml new file mode 100644 index 00000000..e69de29b diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..e69de29b diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..e69de29b diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml new file mode 100644 index 00000000..e69de29b diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..e69de29b From 4294081abbba4e5630d59d6b7cbd02a5fc649c24 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 24 Oct 2021 16:13:04 -0700 Subject: [PATCH 08/76] Add GitHub actions again --- .github/workflows/clippy.yml | 19 +++++++++++++++++++ .github/workflows/docker.yml | 33 +++++++++++++++++++++++++++++++++ .github/workflows/publish.yml | 30 ++++++++++++++++++++++++++++++ .github/workflows/rust.yml | 29 +++++++++++++++++++++++++++++ Dockerfile | 7 +++++++ 5 files changed, 118 insertions(+) diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml index e69de29b..36b9ad18 100644 --- a/.github/workflows/clippy.yml +++ b/.github/workflows/clippy.yml @@ -0,0 +1,19 @@ +name: Code quality + +on: + push: + tags: + - 'v*' +env: + CARGO_TERM_COLOR: always +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - run: cargo login ${CRATES_IO_TOKEN} + working-directory: ./daedalus + env: + CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} + - run: cargo publish + working-directory: ./daedalus diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e69de29b..0b7a22b1 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -0,0 +1,33 @@ +name: docker-build + +on: + push: + branches: + - '**' + tags: + - 'v*' + pull_request: + +jobs: + docker: + runs-on: ubuntu-latest + steps: + - + name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + - + name: Login to GitHub Images + uses: docker/login-action@v1 + with: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - + name: Build and push + id: docker_build + uses: docker/build-push-action@v2 + with: + push: ${{ github.event_name != 'pull_request' }} + tags: ghcr.io/modrinth/daedalus:latest diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e69de29b..d22210e6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -0,0 +1,30 @@ +name: Publish to crates.io + +on: + push: + branches: [ master ] + pull_request: +env: + CARGO_TERM_COLOR: always +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + components: rustfmt, clippy + - name: Cache build artifacts + id: cache-build + uses: actions/cache@v2 + with: + path: target/** + key: ${{ runner.os }}-build-cache + - name: Annotate commit with clippy warnings + uses: actions-rs/clippy-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + args: --all-features diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index e69de29b..a55aaf6e 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -0,0 +1,29 @@ +name: Rust building + +on: + push: + branches: [ master ] + pull_request: +env: + CARGO_TERM_COLOR: always +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Get build cache + id: cache-build + uses: actions/cache@v2 + with: + path: target/** + key: ${{ runner.os }}-build-cache + - name: Install toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + env: + SQLX_OFFLINE: true + - uses: actions-rs/cargo@v1 + name: Build program + with: + command: build diff --git a/Dockerfile b/Dockerfile index e69de29b..c98c954f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -0,0 +1,7 @@ +FROM rust:1.56 + +COPY ./ ./ + +RUN cargo build --release + +CMD ["./target/release/daedalus_client"] \ No newline at end of file From ecdfd65f50d534beb3eb377e590e28be35eb93a1 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 24 Oct 2021 16:16:23 -0700 Subject: [PATCH 09/76] Fix incorrect file names --- .github/workflows/clippy.yml | 19 ------------------- .github/workflows/lint.yml | 30 ++++++++++++++++++++++++++++++ .github/workflows/publish.yml | 33 +++++++++++---------------------- 3 files changed, 41 insertions(+), 41 deletions(-) delete mode 100644 .github/workflows/clippy.yml create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/clippy.yml b/.github/workflows/clippy.yml deleted file mode 100644 index 36b9ad18..00000000 --- a/.github/workflows/clippy.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Code quality - -on: - push: - tags: - - 'v*' -env: - CARGO_TERM_COLOR: always -jobs: - publish: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - run: cargo login ${CRATES_IO_TOKEN} - working-directory: ./daedalus - env: - CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} - - run: cargo publish - working-directory: ./daedalus diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000..d22210e6 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,30 @@ +name: Publish to crates.io + +on: + push: + branches: [ master ] + pull_request: +env: + CARGO_TERM_COLOR: always +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install toolchain + uses: actions-rs/toolchain@v1 + with: + toolchain: stable + override: true + components: rustfmt, clippy + - name: Cache build artifacts + id: cache-build + uses: actions/cache@v2 + with: + path: target/** + key: ${{ runner.os }}-build-cache + - name: Annotate commit with clippy warnings + uses: actions-rs/clippy-check@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + args: --all-features diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index d22210e6..36b9ad18 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,30 +1,19 @@ -name: Publish to crates.io +name: Code quality on: push: - branches: [ master ] - pull_request: + tags: + - 'v*' env: CARGO_TERM_COLOR: always jobs: - lint: + publish: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Install toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - components: rustfmt, clippy - - name: Cache build artifacts - id: cache-build - uses: actions/cache@v2 - with: - path: target/** - key: ${{ runner.os }}-build-cache - - name: Annotate commit with clippy warnings - uses: actions-rs/clippy-check@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - args: --all-features + - uses: actions/checkout@v1 + - run: cargo login ${CRATES_IO_TOKEN} + working-directory: ./daedalus + env: + CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} + - run: cargo publish + working-directory: ./daedalus From e36a191240187bab3c17574efb5ba20b8fb1d879 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 24 Oct 2021 16:16:49 -0700 Subject: [PATCH 10/76] Fix incorrect file names (again) --- .github/workflows/clippy.yml~ | 0 .github/workflows/docker.yml | 1 - .github/workflows/publish.yml | 2 +- 3 files changed, 1 insertion(+), 2 deletions(-) create mode 100644 .github/workflows/clippy.yml~ diff --git a/.github/workflows/clippy.yml~ b/.github/workflows/clippy.yml~ new file mode 100644 index 00000000..e69de29b diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 0b7a22b1..e10d8794 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -7,7 +7,6 @@ on: tags: - 'v*' pull_request: - jobs: docker: runs-on: ubuntu-latest diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 36b9ad18..9e224e2a 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,4 +1,4 @@ -name: Code quality +name: Publish to crates.io on: push: From 3c5edb6171d70b8ce47bb31920db7e7664bf12b3 Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Sun, 24 Oct 2021 16:18:17 -0700 Subject: [PATCH 11/76] Delete clippy.yml~ --- .github/workflows/clippy.yml~ | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .github/workflows/clippy.yml~ diff --git a/.github/workflows/clippy.yml~ b/.github/workflows/clippy.yml~ deleted file mode 100644 index e69de29b..00000000 From e8057a5c8a414bffa814df683e63e12c32ebde22 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 24 Oct 2021 16:21:15 -0700 Subject: [PATCH 12/76] Fix incorrect docker registry --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index e10d8794..4f4245c7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -21,6 +21,7 @@ jobs: name: Login to GitHub Images uses: docker/login-action@v1 with: + registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - From fb16f25b070ebaede31941aaaf1845bd517482c9 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 2 Nov 2021 19:59:10 -0700 Subject: [PATCH 13/76] Fixes in forge universal lib + other things --- .env | 3 +- .github/workflows/docker.yml | 15 ++++---- daedalus/src/modded.rs | 2 +- daedalus_client/src/fabric.rs | 41 ++++++++++++++------- daedalus_client/src/forge.rs | 42 ++++++++++++++------- daedalus_client/src/main.rs | 63 ++++++++++++++++++++++++++++---- daedalus_client/src/minecraft.rs | 40 ++++++++++++-------- 7 files changed, 148 insertions(+), 58 deletions(-) diff --git a/.env b/.env index 56db8578..dc87710f 100644 --- a/.env +++ b/.env @@ -10,4 +10,5 @@ S3_REGION=none S3_BUCKET_NAME=none DO_INTEGRATION=false -DO_ACCESS_KEY=none \ No newline at end of file +DO_ACCESS_KEY=none +DO_ENDPOINT_ID=none \ No newline at end of file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 4f4245c7..8465c522 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -11,12 +11,12 @@ jobs: docker: runs-on: ubuntu-latest steps: - - - name: Set up QEMU - uses: docker/setup-qemu-action@v1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + - name: Checkout + uses: actions/checkout@v2 + - name: Fetch docker metadata + uses: docker/metadata-action@v3 + with: + images: ghcr.io/modrinth/daedalus - name: Login to GitHub Images uses: docker/login-action@v1 @@ -30,4 +30,5 @@ jobs: uses: docker/build-push-action@v2 with: push: ${{ github.event_name != 'pull_request' }} - tags: ghcr.io/modrinth/daedalus:latest + tags: ${{ steps.docker_meta.outputs.tags }} + labels: ${{ steps.docker_meta.outputs.labels }} diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index c631b619..f9d9bc84 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -82,7 +82,7 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> asset_index: merge.asset_index, assets: merge.assets, downloads: merge.downloads, - id: merge.id, + id: partial.id, libraries: partial .libraries .into_iter() diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 5ac71f16..d9a2a164 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -2,13 +2,13 @@ use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::download_file; use daedalus::minecraft::Library; use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Version}; -use futures::lock::Mutex; +use tokio::sync::Mutex; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; -pub async fn retrieve_data() -> Result<(), Error> { +pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { let mut list = fetch_fabric_versions(None).await?; let old_manifest = daedalus::modded::fetch_manifest(&*format!( "fabric/v{}/manifest.json", @@ -23,6 +23,8 @@ pub async fn retrieve_data() -> Result<(), Error> { Vec::new() })); + let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); + if let Some(latest) = list.loader.get(0) { let loaders_mutex = Arc::new(Mutex::new(HashMap::new())); let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); @@ -50,6 +52,7 @@ pub async fn retrieve_data() -> Result<(), Error> { for game_version in list.game.iter_mut() { let visited_artifacts_mutex = Arc::clone(&visited_artifacts_mutex); let loaders_mutex = Arc::clone(&loaders_mutex); + let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); let versions_mutex = Arc::clone(&versions); version_futures.push(async move { @@ -119,8 +122,9 @@ pub async fn retrieve_data() -> Result<(), Error> { format!("{}/{}", "maven", artifact_path), artifact.to_vec(), Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref(), ) - .await?; + .await?; Ok::(lib) }, @@ -131,9 +135,11 @@ pub async fn retrieve_data() -> Result<(), Error> { "fabric/v{}/versions/{}-{}.json", daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, version.inherits_from, - loader + &loader ); + let inherits_from = version.inherits_from.clone(); + upload_file_to_bucket( version_path.clone(), serde_json::to_vec(&PartialVersionInfo { @@ -146,21 +152,25 @@ pub async fn retrieve_data() -> Result<(), Error> { inherits_from: version.inherits_from, libraries: libs, processors: None, - data: None + data: None, })?, Some("application/json".to_string()), + uploaded_files_mutex.as_ref(), ) .await?; { let mut loader_version_map = loader_version_mutex.lock().await; - loader_version_map.insert( - type_, - LoaderVersion { - id: loader, - url: format_url(&*version_path), - }, - ); + async move { + loader_version_map.insert( + type_, + LoaderVersion { + id: format!("{}-{}", inherits_from, loader), + url: format_url(&*version_path), + }, + ); + } + .await; } Ok::<(), Error>(()) @@ -190,7 +200,7 @@ pub async fn retrieve_data() -> Result<(), Error> { chunk_index += 1; let elapsed = now.elapsed(); - println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); } } @@ -204,10 +214,15 @@ pub async fn retrieve_data() -> Result<(), Error> { game_versions: versions.into_inner(), })?, Some("application/json".to_string()), + uploaded_files_mutex.as_ref(), ) .await?; } + if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { + uploaded_files.extend(uploaded_files_mutex.into_inner()); + } + Ok(()) } diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index af31f015..9ba8dfc9 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -2,7 +2,9 @@ use crate::{format_url, upload_file_to_bucket, Error}; use chrono::{DateTime, Utc}; use daedalus::download_file; use daedalus::minecraft::{Argument, ArgumentType, Library, VersionType}; -use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry}; +use daedalus::modded::{ + LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry, +}; use lazy_static::lazy_static; use semver::{Version, VersionReq}; use serde::{Deserialize, Serialize}; @@ -11,20 +13,19 @@ use std::io::Read; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::Mutex; +use log::info; lazy_static! { static ref FORGE_MANIFEST_V1_QUERY: VersionReq = VersionReq::parse(">=8.0.684, <23.5.2851").unwrap(); - static ref FORGE_MANIFEST_V2_QUERY_P1: VersionReq = VersionReq::parse(">=23.5.2851, <31.2.52").unwrap(); static ref FORGE_MANIFEST_V2_QUERY_P2: VersionReq = VersionReq::parse(">=32.0.1, <37.0.0").unwrap(); - static ref FORGE_MANIFEST_V3_QUERY: VersionReq = VersionReq::parse(">=37.0.0").unwrap(); } -pub async fn retrieve_data() -> Result<(), Error> { +pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { let maven_metadata = fetch_maven_metadata(None).await?; let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!( "forge/v{}/manifest.json", @@ -40,6 +41,7 @@ pub async fn retrieve_data() -> Result<(), Error> { })); let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); + let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); let mut version_futures = Vec::new(); @@ -66,7 +68,11 @@ pub async fn retrieve_data() -> Result<(), Error> { let version = Version::parse(&*loader_version)?; - if FORGE_MANIFEST_V1_QUERY.matches(&version) || FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { + if FORGE_MANIFEST_V1_QUERY.matches(&version) + || FORGE_MANIFEST_V2_QUERY_P1.matches(&version) + || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) + || FORGE_MANIFEST_V3_QUERY.matches(&version) + { loaders.push((loader_version_full, version)) } } @@ -76,6 +82,7 @@ pub async fn retrieve_data() -> Result<(), Error> { version_futures.push(async { let versions_mutex = Arc::clone(&versions); let visited_assets = Arc::clone(&visited_assets_mutex); + let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); async move { { if versions_mutex.lock().await.iter().any(|x| { @@ -89,7 +96,7 @@ pub async fn retrieve_data() -> Result<(), Error> { } } - println!("installer start {}", loader_version_full.clone()); + info!("Forge - Installer Start {}", loader_version_full.clone()); let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; let reader = std::io::Cursor::new(&*bytes); @@ -112,7 +119,7 @@ pub async fn retrieve_data() -> Result<(), Error> { bytes::Bytes::from(forge_universal) }; - let forge_universal_path = profile.install.file_path.clone(); + let forge_universal_path = profile.install.path.clone(); let now = Instant::now(); let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { @@ -151,6 +158,7 @@ pub async fn retrieve_data() -> Result<(), Error> { format!("{}/{}", "maven", artifact_path), artifact.to_vec(), Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref(), ).await?; } @@ -158,7 +166,7 @@ pub async fn retrieve_data() -> Result<(), Error> { })).await?; let elapsed = now.elapsed(); - println!("Elapsed lib DL: {:.2?}", elapsed); + info!("Elapsed lib DL: {:.2?}", elapsed); let new_profile = PartialVersionInfo { id: profile.version_info.id, @@ -183,6 +191,7 @@ pub async fn retrieve_data() -> Result<(), Error> { version_path.clone(), serde_json::to_vec(&new_profile)?, Some("application/json".to_string()), + uploaded_files_mutex.as_ref() ).await?; let mut map = HashMap::new(); @@ -243,7 +252,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let artifact_path = daedalus::get_path_from_artifact(&*lib.name)?; - let artifact_bytes = if &*artifact.url == "" { + let artifact_bytes = if artifact.url.is_empty() { forge_universal_bytes.clone().unwrap_or_default() } else { daedalus::download_file( @@ -259,6 +268,7 @@ pub async fn retrieve_data() -> Result<(), Error> { format!("{}/{}", "maven", artifact_path), artifact_bytes.to_vec(), Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref() ).await?; } } @@ -267,7 +277,7 @@ pub async fn retrieve_data() -> Result<(), Error> { })).await?; let elapsed = now.elapsed(); - println!("Elapsed lib DL: {:.2?}", elapsed); + info!("Elapsed lib DL: {:.2?}", elapsed); let new_profile = PartialVersionInfo { id: version_info.id, @@ -292,6 +302,7 @@ pub async fn retrieve_data() -> Result<(), Error> { version_path.clone(), serde_json::to_vec(&new_profile)?, Some("application/json".to_string()), + uploaded_files_mutex.as_ref() ).await?; let mut map = HashMap::new(); @@ -318,7 +329,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let mut versions_peek = version_futures.into_iter().peekable(); let mut chunk_index = 0; while versions_peek.peek().is_some() { - println!("Chunk {} Start", chunk_index); + info!("Chunk {} Start", chunk_index); let now = Instant::now(); let chunk: Vec<_> = versions_peek.by_ref().take(100).collect(); @@ -329,7 +340,7 @@ pub async fn retrieve_data() -> Result<(), Error> { chunk_index += 1; let elapsed = now.elapsed(); - println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); } } @@ -343,10 +354,15 @@ pub async fn retrieve_data() -> Result<(), Error> { game_versions: versions.into_inner(), })?, Some("application/json".to_string()), + uploaded_files_mutex.as_ref() ) .await?; } + if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { + uploaded_files.extend(uploaded_files_mutex.into_inner()); + } + Ok(()) } @@ -415,4 +431,4 @@ struct ForgeInstallerProfileV2 { pub data: HashMap, pub libraries: Vec, pub processors: Vec, -} \ No newline at end of file +} diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 7d051142..b5419fe2 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -47,15 +47,22 @@ async fn main() { loop { timer.tick().await; tokio::spawn(async { - match fabric::retrieve_data().await { + let mut uploaded_files = Vec::new(); + + match fabric::retrieve_data(&mut uploaded_files).await { Ok(..) => {} Err(err) => error!("{:?}", err), }; - match minecraft::retrieve_data().await { + match minecraft::retrieve_data(&mut uploaded_files).await { Ok(..) => {} Err(err) => error!("{:?}", err), }; - match forge::retrieve_data().await { + match forge::retrieve_data(&mut uploaded_files).await { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; + + match purge_digitalocean_cache(uploaded_files).await { Ok(..) => {} Err(err) => error!("{:?}", err), }; @@ -97,10 +104,12 @@ fn check_env_vars() -> bool { let do_integration = dotenv::var("DO_INTEGRATION") .ok() .map(|x| x.parse::().ok()) - .flatten(); + .flatten() + .unwrap_or(false); - if do_integration.unwrap_or(false) { - failed |= check_var::("DO_ACCESS_KEY"); + if do_integration { + failed |= check_var::("DO_ACCESS_KEY"); + failed |= check_var::("DO_ENDPOINT_ID"); } failed @@ -126,11 +135,14 @@ pub async fn upload_file_to_bucket( path: String, bytes: Vec, content_type: Option, + uploaded_files: &tokio::sync::Mutex>, ) -> Result<(), Error> { + let key = format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path); + CLIENT .put_object(PutObjectRequest { bucket: dotenv::var("S3_BUCKET_NAME").unwrap(), - key: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path), + key: key.clone(), body: Some(bytes.into()), acl: Some("public-read".to_string()), content_type, @@ -142,6 +154,11 @@ pub async fn upload_file_to_bucket( file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path), })?; + { + let mut uploaded_files = uploaded_files.lock().await; + uploaded_files.push(key); + } + Ok(()) } @@ -153,3 +170,35 @@ pub fn format_url(path: &str) -> String { path ) } + +#[derive(serde::Serialize)] +struct PurgeCacheRequest { + pub files: Vec, +} + +pub async fn purge_digitalocean_cache(files: Vec) -> Result<(), Error> { + if !dotenv::var("DO_INTEGRATION") + .ok() + .map(|x| x.parse::().ok()) + .flatten() + .unwrap_or(false) { + + return Ok(()) + } + + let client = reqwest::Client::new(); + + client + .delete(&format!( + "https://api.digitalocean.com/v2/cdn/endpoints/{}/cache", + &*dotenv::var("DO_ENDPOINT_ID").unwrap() + )) + .header("Authorization", &*format!("Bearer {}", &*dotenv::var("DO_ACCESS_KEY").unwrap())) + .json(&PurgeCacheRequest { files }) + .send().await.map_err(|err| Error::FetchError { + inner: err, + item: "purging digital ocean cache".to_string() + })?; + + Ok(()) +} diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index c6062dbb..3245238c 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,10 +1,10 @@ use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::download_file; -use futures::lock::Mutex; +use tokio::sync::Mutex; use std::sync::Arc; use std::time::{Duration, Instant}; -pub async fn retrieve_data() -> Result<(), Error> { +pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { let old_manifest = daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&*format!( "minecraft/v{}/manifest.json", @@ -17,6 +17,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let cloned_manifest = Arc::new(Mutex::new(manifest.clone())); let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); + let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); let now = Instant::now(); @@ -38,6 +39,7 @@ pub async fn retrieve_data() -> Result<(), Error> { let visited_assets_mutex = Arc::clone(&visited_assets_mutex); let cloned_manifest_mutex = Arc::clone(&cloned_manifest); + let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); let assets_hash = old_version.map(|x| x.assets_index_sha1.clone()).flatten(); @@ -104,6 +106,7 @@ pub async fn retrieve_data() -> Result<(), Error> { assets_path, assets_index.to_vec(), Some("application/json".to_string()), + uploaded_files_mutex.as_ref() )); } } @@ -113,13 +116,11 @@ pub async fn retrieve_data() -> Result<(), Error> { version_path, serde_json::to_vec(&version_info)?, Some("application/json".to_string()), + uploaded_files_mutex.as_ref() )); } - let now = Instant::now(); futures::future::try_join_all(upload_futures).await?; - let elapsed = now.elapsed(); - println!("Spaces Upload {} Elapsed: {:.2?}", version.id, elapsed); Ok::<(), Error>(()) } @@ -129,20 +130,22 @@ pub async fn retrieve_data() -> Result<(), Error> { }) } - let mut versions = version_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions.peek().is_some() { - let now = Instant::now(); + { + let mut versions = version_futures.into_iter().peekable(); + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); - let chunk: Vec<_> = versions.by_ref().take(100).collect(); - futures::future::try_join_all(chunk).await?; + let chunk: Vec<_> = versions.by_ref().take(100).collect(); + futures::future::try_join_all(chunk).await?; - tokio::time::sleep(Duration::from_secs(1)).await; + tokio::time::sleep(Duration::from_secs(1)).await; - chunk_index += 1; + chunk_index += 1; - let elapsed = now.elapsed(); - println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + let elapsed = now.elapsed(); + info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } } upload_file_to_bucket( @@ -152,11 +155,16 @@ pub async fn retrieve_data() -> Result<(), Error> { ), serde_json::to_vec(&*cloned_manifest.lock().await)?, Some("application/json".to_string()), + uploaded_files_mutex.as_ref() ) .await?; + if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { + uploaded_files.extend(uploaded_files_mutex.into_inner()); + } + let elapsed = now.elapsed(); - println!("Elapsed: {:.2?}", elapsed); + info!("Elapsed: {:.2?}", elapsed); Ok(()) } From 8704eff632dd9cd8b5bed822c9b60c9a0943a5d2 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 2 Nov 2021 20:17:12 -0700 Subject: [PATCH 14/76] Fix docker action --- .github/workflows/docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 8465c522..1aa837a1 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -29,6 +29,8 @@ jobs: id: docker_build uses: docker/build-push-action@v2 with: + context: . + file: ./Dockerfile push: ${{ github.event_name != 'pull_request' }} tags: ${{ steps.docker_meta.outputs.tags }} labels: ${{ steps.docker_meta.outputs.labels }} From 061b88f5b5c9ddd4ba150d1b105949635c76e28e Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 2 Nov 2021 20:19:43 -0700 Subject: [PATCH 15/76] Fix docker action again --- .github/workflows/docker.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1aa837a1..fb101422 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -14,6 +14,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Fetch docker metadata + id: docker_meta uses: docker/metadata-action@v3 with: images: ghcr.io/modrinth/daedalus From bec54b42837050252bb0dac192aaad86a1ce18f0 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 2 Nov 2021 20:20:55 -0700 Subject: [PATCH 16/76] Fix action title --- .github/workflows/lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index d22210e6..8bf19521 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,4 +1,4 @@ -name: Publish to crates.io +name: Rust lint on: push: From d596bdb454995f7e628e18dbc84b8973e8b50701 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 2 Nov 2021 20:24:51 -0700 Subject: [PATCH 17/76] Fix import errors --- daedalus_client/src/fabric.rs | 1 + daedalus_client/src/main.rs | 2 +- daedalus_client/src/minecraft.rs | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index d9a2a164..5bf84c04 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -7,6 +7,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; +use log::info; pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { let mut list = fetch_fabric_versions(None).await?; diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index b5419fe2..f599b118 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -1,4 +1,4 @@ -use log::{error, info, warn}; +use log::{error, warn}; use rusoto_core::credential::StaticProvider; use rusoto_core::{HttpClient, Region, RusotoError}; use rusoto_s3::{PutObjectError, S3Client}; diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 3245238c..69484d80 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -3,6 +3,7 @@ use daedalus::download_file; use tokio::sync::Mutex; use std::sync::Arc; use std::time::{Duration, Instant}; +use log::info; pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { let old_manifest = From c744dc8cc369aae5173a92189b09145031bee175 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 2 Nov 2021 20:25:55 -0700 Subject: [PATCH 18/76] Bump library version --- daedalus/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index def9e0dc..a248cf8f 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.0" +version = "0.1.1" authors = ["Jai A "] edition = "2018" license = "MIT" From 240269eb25e3f320bf3aa7c2c0c10d6b872dc142 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 2 Nov 2021 21:44:31 -0700 Subject: [PATCH 19/76] Fix lib not parsing maven file extensions --- daedalus/src/lib.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index 5e0f5e42..b24d304a 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -51,17 +51,22 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { let name = name_items.get(1).ok_or_else(|| { Error::ParseError(format!("Unable to find name for library {}", &artifact)) })?; - let version = name_items.get(2).ok_or_else(|| { + let version_ext = name_items.get(2).ok_or_else(|| { + Error::ParseError(format!("Unable to find version for library {}", &artifact)) + })?.split('@').collect::>(); + let version = version_ext.get(0).ok_or_else(|| { Error::ParseError(format!("Unable to find version for library {}", &artifact)) })?; + let ext = version_ext.get(1); Ok(format!( - "{}/{}/{}/{}-{}.jar", + "{}/{}/{}/{}-{}.{}", package.replace(".", "/"), name, version, name, - version + version, + ext.unwrap_or(&"jar") )) } From 793e542312f6a71c177632fcf2d869f50e6b327f Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 2 Nov 2021 21:44:59 -0700 Subject: [PATCH 20/76] Bump version --- daedalus/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index a248cf8f..336e30cb 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.1" +version = "0.1.2" authors = ["Jai A "] edition = "2018" license = "MIT" From 2a7dbda133be3ad14eff311c857bea59fe4c4709 Mon Sep 17 00:00:00 2001 From: Jai A Date: Wed, 3 Nov 2021 17:49:58 -0700 Subject: [PATCH 21/76] Bump version + Fix partial version and full version argument joining --- daedalus/Cargo.toml | 2 +- daedalus/src/modded.rs | 23 +++++++++++++++++++++-- daedalus_client/src/fabric.rs | 1 + daedalus_client/src/forge.rs | 2 ++ 4 files changed, 25 insertions(+), 3 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 336e30cb..d7252669 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.2" +version = "0.1.3" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index f9d9bc84..471701c8 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -33,6 +33,8 @@ pub struct PartialVersionInfo { pub time: DateTime, /// The classpath to the main class to launch the game pub main_class: Option, + /// (Legacy) Arguments passed to the game + pub minecraft_arguments: Option, /// Arguments passed to the game or JVM pub arguments: Option>>, /// Libraries that the version depends on @@ -72,7 +74,24 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> VersionInfo { arguments: if let Some(partial_args) = partial.arguments { if let Some(merge_args) = merge.arguments { - Some(partial_args.into_iter().chain(merge_args).collect()) + let mut new_map = HashMap::new(); + + fn add_keys(new_map: &mut HashMap>, args: HashMap>) { + for (type_, arguments) in args { + for arg in arguments { + if let Some(vec) = new_map.get_mut(&type_) { + vec.push(arg); + } else { + new_map.insert(type_, vec![arg]); + } + } + } + } + + add_keys(&mut new_map, merge_args); + add_keys(&mut new_map, partial_args); + + Some(new_map) } else { Some(partial_args) } @@ -93,7 +112,7 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> } else { merge.main_class }, - minecraft_arguments: merge.minecraft_arguments, + minecraft_arguments: partial.minecraft_arguments, minimum_launcher_version: merge.minimum_launcher_version, release_time: partial.release_time, time: partial.time, diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 5bf84c04..2734bc26 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -152,6 +152,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error type_: version.type_, inherits_from: version.inherits_from, libraries: libs, + minecraft_arguments: version.minecraft_arguments, processors: None, data: None, })?, diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 9ba8dfc9..ff20be0c 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -174,6 +174,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error release_time: profile.version_info.release_time, time: profile.version_info.time, main_class: profile.version_info.main_class, + minecraft_arguments: profile.version_info.minecraft_arguments.clone(), arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), libraries: libs, type_: profile.version_info.type_, @@ -285,6 +286,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error release_time: version_info.release_time, time: version_info.time, main_class: version_info.main_class, + minecraft_arguments: version_info.minecraft_arguments, arguments: version_info.arguments, libraries: libs, type_: version_info.type_, From e91f8f693b0e103ce4ebc317c61fa6f78551f386 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 7 Nov 2021 18:42:33 -0700 Subject: [PATCH 22/76] Add local libs to modrinth maven and other fixes --- daedalus/Cargo.toml | 2 +- daedalus/src/lib.rs | 66 +++++++--- daedalus/src/modded.rs | 5 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/fabric.rs | 6 +- daedalus_client/src/forge.rs | 209 +++++++++++++++++++++++-------- daedalus_client/src/main.rs | 67 ++++++---- daedalus_client/src/minecraft.rs | 10 +- 8 files changed, 261 insertions(+), 106 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index d7252669..4f674a30 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.3" +version = "0.1.4" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index b24d304a..77657323 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -51,23 +51,57 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { let name = name_items.get(1).ok_or_else(|| { Error::ParseError(format!("Unable to find name for library {}", &artifact)) })?; - let version_ext = name_items.get(2).ok_or_else(|| { - Error::ParseError(format!("Unable to find version for library {}", &artifact)) - })?.split('@').collect::>(); - let version = version_ext.get(0).ok_or_else(|| { - Error::ParseError(format!("Unable to find version for library {}", &artifact)) - })?; - let ext = version_ext.get(1); - Ok(format!( - "{}/{}/{}/{}-{}.{}", - package.replace(".", "/"), - name, - version, - name, - version, - ext.unwrap_or(&"jar") - )) + if name_items.len() == 3 { + let version_ext = name_items + .get(2) + .ok_or_else(|| { + Error::ParseError(format!("Unable to find version for library {}", &artifact)) + })? + .split('@') + .collect::>(); + let version = version_ext.get(0).ok_or_else(|| { + Error::ParseError(format!("Unable to find version for library {}", &artifact)) + })?; + let ext = version_ext.get(1); + + Ok(format!( + "{}/{}/{}/{}-{}.{}", + package.replace(".", "/"), + name, + version, + name, + version, + ext.unwrap_or(&"jar") + )) + } else { + let version = name_items.get(2).ok_or_else(|| { + Error::ParseError(format!("Unable to find version for library {}", &artifact)) + })?; + + let data_ext = name_items + .get(3) + .ok_or_else(|| { + Error::ParseError(format!("Unable to find data for library {}", &artifact)) + })? + .split('@') + .collect::>(); + let data = data_ext.get(0).ok_or_else(|| { + Error::ParseError(format!("Unable to find data for library {}", &artifact)) + })?; + let ext = data_ext.get(1); + + Ok(format!( + "{}/{}/{}/{}-{}-{}.{}", + package.replace(".", "/"), + name, + version, + name, + version, + data, + ext.unwrap_or(&"jar") + )) + } } /// Downloads a file from specified mirrors diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 471701c8..6cc42098 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -76,7 +76,10 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> if let Some(merge_args) = merge.arguments { let mut new_map = HashMap::new(); - fn add_keys(new_map: &mut HashMap>, args: HashMap>) { + fn add_keys( + new_map: &mut HashMap>, + args: HashMap>, + ) { for (type_, arguments) in args { for arg in arguments { if let Some(vec) = new_map.get_mut(&type_) { diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index f36f6f3b..f30de5b5 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.0" +version = "0.1.4" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 2734bc26..0024a6ab 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -2,12 +2,12 @@ use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::download_file; use daedalus::minecraft::Library; use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Version}; -use tokio::sync::Mutex; +use log::info; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; -use log::info; +use tokio::sync::Mutex; pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { let mut list = fetch_fabric_versions(None).await?; @@ -125,7 +125,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error Some("application/java-archive".to_string()), uploaded_files_mutex.as_ref(), ) - .await?; + .await?; Ok::(lib) }, diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index ff20be0c..48dae3ab 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -6,6 +6,7 @@ use daedalus::modded::{ LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry, }; use lazy_static::lazy_static; +use log::info; use semver::{Version, VersionReq}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -13,7 +14,6 @@ use std::io::Read; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::Mutex; -use log::info; lazy_static! { static ref FORGE_MANIFEST_V1_QUERY: VersionReq = @@ -99,26 +99,30 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error info!("Forge - Installer Start {}", loader_version_full.clone()); let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; - let reader = std::io::Cursor::new(&*bytes); + let reader = std::io::Cursor::new(bytes); - if let Ok(mut archive) = zip::ZipArchive::new(reader) { + if let Ok(archive) = zip::ZipArchive::new(reader) { if FORGE_MANIFEST_V1_QUERY.matches(&version) { - let profile = { - let mut install_profile = archive.by_name("install_profile.json")?; + let mut archive_clone = archive.clone(); + let profile = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("install_profile.json")?; let mut contents = String::new(); install_profile.read_to_string(&mut contents)?; - serde_json::from_str::(&*contents)? - }; + Ok::(serde_json::from_str::(&*contents)?) + }).await??; - let forge_universal_bytes = { - let mut forge_universal_file = archive.by_name(&*profile.install.file_path)?; + let mut archive_clone = archive.clone(); + let file_path = profile.install.file_path.clone(); + let forge_universal_bytes = tokio::task::spawn_blocking(move || { + let mut forge_universal_file = archive_clone.by_name(&*file_path)?; let mut forge_universal = Vec::new(); forge_universal_file.read_to_end(&mut forge_universal)?; - bytes::Bytes::from(forge_universal) - }; + + Ok::(bytes::Bytes::from(forge_universal)) + }).await??; let forge_universal_path = profile.install.path.clone(); let now = Instant::now(); @@ -205,73 +209,170 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error loaders: map }) } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { - let profile = { - let mut install_profile = archive.by_name("install_profile.json")?; + let mut archive_clone = archive.clone(); + let mut profile = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("install_profile.json")?; let mut contents = String::new(); install_profile.read_to_string(&mut contents)?; - serde_json::from_str::(&*contents)? - }; + Ok::(serde_json::from_str::(&*contents)?) + }).await??; - let version_info = { - let mut install_profile = archive.by_name("version.json")?; + let mut archive_clone = archive.clone(); + let version_info = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("version.json")?; let mut contents = String::new(); install_profile.read_to_string(&mut contents)?; - serde_json::from_str::(&*contents)? - }; - let forge_universal_bytes = { - if let Some(path) = &profile.path { - let mut forge_universal_file = archive.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*path)?))?; - let mut forge_universal = Vec::new(); - forge_universal_file.read_to_end(&mut forge_universal)?; + Ok::(serde_json::from_str::(&*contents)?) + }).await??; - Some(bytes::Bytes::from(forge_universal)) - } else { - None + + let mut libs : Vec = profile.libraries.into_iter().chain(version_info.libraries).collect(); + + let mut local_libs : HashMap = HashMap::new(); + + for lib in &libs { + if lib.downloads.as_ref().map(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).flatten().unwrap_or(false) { + let mut archive_clone = archive.clone(); + let lib_name_clone = lib.name.clone(); + + let lib_bytes = tokio::task::spawn_blocking(move || { + let mut lib_file = archive_clone.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*lib_name_clone)?))?; + let mut lib_bytes = Vec::new(); + lib_file.read_to_end(&mut lib_bytes)?; + + Ok::(bytes::Bytes::from(lib_bytes)) + }).await??; + + local_libs.insert(lib.name.clone(), lib_bytes); } - }; + } - let now = Instant::now(); - let libs = futures::future::try_join_all(profile.libraries.into_iter().chain(version_info.libraries).map(|mut lib| async { - if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - { - let mut visited_assets = visited_assets.lock().await; + let path = profile.path.clone(); + let version = profile.version.clone(); - if visited_assets.contains(&lib.name) { - artifact.url = format_url(&*format!("maven/{}", artifact.path)); + for entry in profile.data.values_mut() { + if entry.client.starts_with('/') || entry.server.starts_with('/') { + macro_rules! read_data { + ($value:expr) => { + let mut archive_clone = archive.clone(); + let value_clone = $value.clone(); + let lib_bytes = tokio::task::spawn_blocking(move || { + let mut lib_file = archive_clone.by_name(&value_clone[1..value_clone.len()])?; + let mut lib_bytes = Vec::new(); + lib_file.read_to_end(&mut lib_bytes)?; - return Ok::(lib); - } else { - visited_assets.push(lib.name.clone()) + Ok::(bytes::Bytes::from(lib_bytes)) + }).await??; + + let split = $value.split('/').last(); + + if let Some(last) = split { + let mut file = last.split('.'); + + if let Some(file_name) = file.next() { + if let Some(ext) = file.next() { + let path = format!("{}:{}@{}", path.as_deref().unwrap_or(&*format!("net.minecraftforge:forge:{}", version)), file_name, ext); + $value = format!("[{}]", &path); + local_libs.insert(path.clone(), bytes::Bytes::from(lib_bytes)); + + libs.push(Library { + downloads: None, + extract: None, + name: path, + url: Some("".to_string()), + natives: None, + rules: None, + checksums: None + }); + } + } } } + } - let artifact_path = - daedalus::get_path_from_artifact(&*lib.name)?; + if entry.client.starts_with('/') { + read_data!(entry.client); + } - let artifact_bytes = if artifact.url.is_empty() { - forge_universal_bytes.clone().unwrap_or_default() + // Do we really need to support server installs? Keeping this here + // just in case + // + // if entry.server.starts_with('/') { + // read_data!(entry.server); + // } + } + } + + let now = Instant::now(); + let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { + let artifact_path = + daedalus::get_path_from_artifact(&*lib.name)?; + + { + let mut visited_assets = visited_assets.lock().await; + + if visited_assets.contains(&lib.name) { + if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { + artifact.url = format_url(&*format!("maven/{}", artifact_path)); + } + } else if lib.url.is_some() { + lib.url = Some(format_url(&*format!("maven/{}", artifact_path))); + } + + return Ok::(lib); + } else { + visited_assets.push(lib.name.clone()) + } + } + + let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { + let res = if artifact.url.is_empty() { + local_libs.get(&lib.name).cloned() } else { - daedalus::download_file( + Some(daedalus::download_file( &*artifact.url, Some(&*artifact.sha1), ) - .await? + .await?) }; - artifact.url = format_url(&*format!("maven/{}", artifact.path)); + if res.is_some() { + artifact.url = format_url(&*format!("maven/{}", artifact_path)); + } - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact_bytes.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref() - ).await?; + res + } else { None } + } else if let Some(ref mut url) = lib.url { + let res = if url.is_empty() { + local_libs.get(&lib.name).cloned() + } else { + Some(daedalus::download_file( + url, + None, + ) + .await?) + }; + + if res.is_some() { + lib.url = Some(format_url(&*format!("maven/{}", artifact_path))); } + + res + } else { None }; + + if let Some(bytes) = artifact_bytes { + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + bytes.to_vec(), + Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref() + ).await?; } Ok::(lib) @@ -334,7 +435,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error info!("Chunk {} Start", chunk_index); let now = Instant::now(); - let chunk: Vec<_> = versions_peek.by_ref().take(100).collect(); + let chunk: Vec<_> = versions_peek.by_ref().take(10).collect(); futures::future::try_join_all(chunk).await?; tokio::time::sleep(Duration::from_secs(1)).await; @@ -356,7 +457,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error game_versions: versions.into_inner(), })?, Some("application/json".to_string()), - uploaded_files_mutex.as_ref() + uploaded_files_mutex.as_ref(), ) .await?; } diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index f599b118..123e2b4f 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -62,7 +62,7 @@ async fn main() { Err(err) => error!("{:?}", err), }; - match purge_digitalocean_cache(uploaded_files).await { + match purge_digitalocean_cache(uploaded_files).await { Ok(..) => {} Err(err) => error!("{:?}", err), }; @@ -139,27 +139,39 @@ pub async fn upload_file_to_bucket( ) -> Result<(), Error> { let key = format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path); - CLIENT - .put_object(PutObjectRequest { - bucket: dotenv::var("S3_BUCKET_NAME").unwrap(), - key: key.clone(), - body: Some(bytes.into()), - acl: Some("public-read".to_string()), - content_type, - ..Default::default() - }) - .await - .map_err(|err| Error::S3Error { - inner: err, - file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path), - })?; + for attempt in 1..=4 { + let result = CLIENT + .put_object(PutObjectRequest { + bucket: dotenv::var("S3_BUCKET_NAME").unwrap(), + key: key.clone(), + body: Some(bytes.clone().into()), + acl: Some("public-read".to_string()), + content_type: content_type.clone(), + ..Default::default() + }) + .await + .map_err(|err| Error::S3Error { + inner: err, + file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path), + }); - { - let mut uploaded_files = uploaded_files.lock().await; - uploaded_files.push(key); + match result { + Ok(_) => { + { + let mut uploaded_files = uploaded_files.lock().await; + uploaded_files.push(key); + } + + return Ok(()); + } + Err(_) if attempt <= 3 => continue, + Err(_) => { + result?; + } + } } - Ok(()) + unreachable!() } pub fn format_url(path: &str) -> String { @@ -181,9 +193,9 @@ pub async fn purge_digitalocean_cache(files: Vec) -> Result<(), Error> { .ok() .map(|x| x.parse::().ok()) .flatten() - .unwrap_or(false) { - - return Ok(()) + .unwrap_or(false) + { + return Ok(()); } let client = reqwest::Client::new(); @@ -193,11 +205,16 @@ pub async fn purge_digitalocean_cache(files: Vec) -> Result<(), Error> { "https://api.digitalocean.com/v2/cdn/endpoints/{}/cache", &*dotenv::var("DO_ENDPOINT_ID").unwrap() )) - .header("Authorization", &*format!("Bearer {}", &*dotenv::var("DO_ACCESS_KEY").unwrap())) + .header( + "Authorization", + &*format!("Bearer {}", &*dotenv::var("DO_ACCESS_KEY").unwrap()), + ) .json(&PurgeCacheRequest { files }) - .send().await.map_err(|err| Error::FetchError { + .send() + .await + .map_err(|err| Error::FetchError { inner: err, - item: "purging digital ocean cache".to_string() + item: "purging digital ocean cache".to_string(), })?; Ok(()) diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 69484d80..babb6a18 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,9 +1,9 @@ use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::download_file; -use tokio::sync::Mutex; +use log::info; use std::sync::Arc; use std::time::{Duration, Instant}; -use log::info; +use tokio::sync::Mutex; pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { let old_manifest = @@ -107,7 +107,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error assets_path, assets_index.to_vec(), Some("application/json".to_string()), - uploaded_files_mutex.as_ref() + uploaded_files_mutex.as_ref(), )); } } @@ -117,7 +117,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error version_path, serde_json::to_vec(&version_info)?, Some("application/json".to_string()), - uploaded_files_mutex.as_ref() + uploaded_files_mutex.as_ref(), )); } @@ -156,7 +156,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error ), serde_json::to_vec(&*cloned_manifest.lock().await)?, Some("application/json".to_string()), - uploaded_files_mutex.as_ref() + uploaded_files_mutex.as_ref(), ) .await?; From 0990ac4fc16677b86669f5bb03ed473891aee35f Mon Sep 17 00:00:00 2001 From: Jai A Date: Mon, 8 Nov 2021 20:17:20 -0700 Subject: [PATCH 23/76] Add forge data to main version info --- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 11 +++++++++++ daedalus/src/modded.rs | 9 +++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 4f674a30..ef541991 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.4" +version = "0.1.5" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index d345f04c..02d14424 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -1,3 +1,4 @@ +use crate::modded::{Processor, SidedDataEntry}; use crate::{download_file, Error}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; @@ -51,9 +52,11 @@ pub struct Version { pub sha1: String, /// Whether the version supports the latest player safety features pub compliance_level: u32, + #[serde(skip_serializing_if = "Option::is_none")] /// (Modrinth Provided) The link to the assets index for this version /// This is only available when using the Modrinth mirror pub assets_index_url: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// (Modrinth Provided) The SHA1 hash of the assets index for this version /// This is only available when using the Modrinth mirror pub assets_index_sha1: Option, @@ -147,8 +150,10 @@ pub struct LibraryDownload { #[derive(Serialize, Deserialize, Debug)] /// A list of files that should be downloaded for libraries pub struct LibraryDownloads { + #[serde(skip_serializing_if = "Option::is_none")] /// The primary library artifact pub artifact: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// Conditional files that may be needed to be downloaded alongside the library /// The HashMap key specifies a classifier as additional information for downloading files pub classifiers: Option>, @@ -315,6 +320,12 @@ pub struct VersionInfo { #[serde(rename = "type")] /// The type of version pub type_: VersionType, + #[serde(skip_serializing_if = "Option::is_none")] + /// (Forge-only) + pub data: Option>, + #[serde(skip_serializing_if = "Option::is_none")] + /// (Forge-only) The list of processors to run after downloading the files + pub processors: Option>, } /// Fetches detailed information about a version from the manifest diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 6cc42098..41289f34 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -31,10 +31,13 @@ pub struct PartialVersionInfo { pub release_time: DateTime, /// The latest time a file in this version was updated pub time: DateTime, + #[serde(skip_serializing_if = "Option::is_none")] /// The classpath to the main class to launch the game pub main_class: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// (Legacy) Arguments passed to the game pub minecraft_arguments: Option, + #[serde(skip_serializing_if = "Option::is_none")] /// Arguments passed to the game or JVM pub arguments: Option>>, /// Libraries that the version depends on @@ -42,8 +45,10 @@ pub struct PartialVersionInfo { #[serde(rename = "type")] /// The type of version pub type_: VersionType, + #[serde(skip_serializing_if = "Option::is_none")] /// (Forge-only) pub data: Option>, + #[serde(skip_serializing_if = "Option::is_none")] /// (Forge-only) The list of processors to run after downloading the files pub processors: Option>, } @@ -57,8 +62,10 @@ pub struct Processor { pub classpath: Vec, /// Arguments for this processor. pub args: Vec, + #[serde(skip_serializing_if = "Option::is_none")] /// Represents a map of outputs. Keys and values can be data values pub outputs: Option>, + #[serde(skip_serializing_if = "Option::is_none")] /// Which sides this processor shall be ran on. /// Valid values: client, server, extract pub sides: Option>, @@ -120,6 +127,8 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> release_time: partial.release_time, time: partial.time, type_: partial.type_, + data: partial.data, + processors: partial.processors, } } From f6c611bbba1b93f96466babb2d2f811ededed069 Mon Sep 17 00:00:00 2001 From: Jai A Date: Wed, 10 Nov 2021 17:34:53 -0700 Subject: [PATCH 24/76] Add classpath variable for libraries --- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 7 +++++++ daedalus_client/src/forge.rs | 16 +++++++++++++--- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index ef541991..dc5496ae 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.5" +version = "0.1.6" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 02d14424..0e9a863b 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -252,6 +252,13 @@ pub struct Library { #[serde(skip_serializing_if = "Option::is_none")] /// SHA1 Checksums for validating the library's integrity. Only present for forge libraries pub checksums: Option>, + #[serde(default = "default_include_in_classpath")] + /// Whether the library should be included in the classpath at the game's launch + pub include_in_classpath: bool, +} + +fn default_include_in_classpath() -> bool { + true } #[derive(Serialize, Deserialize, Debug, Clone)] diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 48dae3ab..f9854b84 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -230,7 +230,16 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error }).await??; - let mut libs : Vec = profile.libraries.into_iter().chain(version_info.libraries).collect(); + let mut libs : Vec = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library { + downloads: x.downloads, + extract: x.extract, + name: x.name, + url: x.url, + natives: x.natives, + rules: x.rules, + checksums: x.checksums, + include_in_classpath: false + })).collect(); let mut local_libs : HashMap = HashMap::new(); @@ -286,7 +295,8 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error url: Some("".to_string()), natives: None, rules: None, - checksums: None + checksums: None, + include_in_classpath: false, }); } } @@ -321,7 +331,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error artifact.url = format_url(&*format!("maven/{}", artifact_path)); } } else if lib.url.is_some() { - lib.url = Some(format_url(&*format!("maven/{}", artifact_path))); + lib.url = Some(format_url("maven/")); } return Ok::(lib); From d7e04687762b74ca542cf16b8fed20ae6eb1f203 Mon Sep 17 00:00:00 2001 From: Jai A Date: Wed, 10 Nov 2021 18:40:12 -0700 Subject: [PATCH 25/76] Fix library URL being set incorrectly --- daedalus/Cargo.toml | 2 +- daedalus_client/src/forge.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index dc5496ae..f5e583ba 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.6" +version = "0.1.7" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index f9854b84..baa81673 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -370,7 +370,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error }; if res.is_some() { - lib.url = Some(format_url(&*format!("maven/{}", artifact_path))); + lib.url = Some(format_url("maven/")); } res From 5a6c06c8a3dea11e5f10a491a40687b8ddcccba1 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sat, 18 Dec 2021 22:55:03 -0700 Subject: [PATCH 26/76] Host all loaders for forge, fix stable markers, add java version to daedalus --- .idea/runConfigurations.xml | 10 --- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 12 ++++ daedalus/src/modded.rs | 16 ++--- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/fabric.rs | 90 ++++++++++++++++-------- daedalus_client/src/forge.rs | 116 ++++++++++++++++++++----------- daedalus_client/src/main.rs | 31 ++++++--- daedalus_client/src/minecraft.rs | 7 +- 9 files changed, 180 insertions(+), 106 deletions(-) delete mode 100644 .idea/runConfigurations.xml diff --git a/.idea/runConfigurations.xml b/.idea/runConfigurations.xml deleted file mode 100644 index 797acea5..00000000 --- a/.idea/runConfigurations.xml +++ /dev/null @@ -1,10 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index f5e583ba..38aee979 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.7" +version = "0.1.8" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 0e9a863b..250262cd 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -229,6 +229,15 @@ pub struct LibraryExtract { pub exclude: Option>, } +#[derive(Serialize, Deserialize, Debug)] +/// Information about the java version the game needs +pub struct JavaVersion { + /// The component needed for the Java installation + component: String, + /// The major Java version number + major_version: u32, +} + #[derive(Serialize, Deserialize, Debug)] /// A library which the game relies on to run pub struct Library { @@ -311,6 +320,9 @@ pub struct VersionInfo { pub downloads: HashMap, /// The version ID of the version pub id: String, + + /// The Java version this version supports + pub java_version: JavaVersion, /// Libraries that the version depends on pub libraries: Vec, /// The classpath to the main class to launch the game diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 41289f34..ea3e07c9 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -112,6 +112,7 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> assets: merge.assets, downloads: merge.downloads, id: partial.id, + java_version: merge.java_version, libraries: partial .libraries .into_iter() @@ -140,24 +141,13 @@ pub struct Manifest { pub game_versions: Vec, } -#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] -#[serde(rename_all = "camelCase")] -/// The version type of the loader -pub enum LoaderType { - /// The latest type is for experimental loader versions that may not be ready for normal use - Latest, - /// The stable type is for the most stable but recent loader version. For the forge mod loader, - /// this is never used - Stable, -} - #[derive(Serialize, Deserialize, Debug, Clone)] /// A game version of Minecraft pub struct Version { /// The minecraft version ID pub id: String, /// A map that contains loader versions for the game version - pub loaders: HashMap, + pub loaders: Vec, } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -167,6 +157,8 @@ pub struct LoaderVersion { pub id: String, /// The URL of the version's manifest pub url: String, + /// Whether the loader is stable or not + pub stable: bool, } /// Fetches the manifest of a mod loader diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index f30de5b5..77ae6fd4 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.4" +version = "0.1.8" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 0024a6ab..514252c9 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -1,15 +1,17 @@ use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::download_file; -use daedalus::minecraft::Library; -use daedalus::modded::{LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Version}; +use daedalus::minecraft::{Library, VersionManifest}; +use daedalus::modded::{LoaderVersion, Manifest, PartialVersionInfo, Version}; use log::info; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; use std::sync::Arc; use std::time::{Duration, Instant}; -use tokio::sync::Mutex; +use tokio::sync::{Mutex, RwLock}; -pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { +pub async fn retrieve_data( + minecraft_versions: &VersionManifest, + uploaded_files: &mut Vec, +) -> Result<(), Error> { let mut list = fetch_fabric_versions(None).await?; let old_manifest = daedalus::modded::fetch_manifest(&*format!( "fabric/v{}/manifest.json", @@ -27,24 +29,28 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); if let Some(latest) = list.loader.get(0) { - let loaders_mutex = Arc::new(Mutex::new(HashMap::new())); + let loaders_mutex = Arc::new(RwLock::new(Vec::new())); let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); { - let mut loaders = loaders_mutex.lock().await; + let mut loaders = loaders_mutex.write().await; - loaders.insert(LoaderType::Latest, latest.version.clone()); + // for loader in &list.loader { + // loaders.push((Box::new(loader.stable), loader.version.clone())) + // } + + loaders.push((Box::new(latest.stable), latest.version.clone())); if !latest.stable { if let Some(stable) = list.loader.iter().find(|x| x.stable) { - loaders.insert(LoaderType::Stable, stable.version.clone()); + loaders.push((Box::new(stable.stable), stable.version.clone())); } } list.loader = list .loader .into_iter() - .filter(|x| loaders.values().any(|val| val == &x.version)) + .filter(|x| loaders.iter().any(|val| val.1 == x.version)) .collect(); } @@ -57,19 +63,16 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error let versions_mutex = Arc::clone(&versions); version_futures.push(async move { - let loader_version_mutex = Mutex::new(HashMap::new()); + let loader_version_mutex = Mutex::new(Vec::new()); let versions = futures::future::try_join_all( - loaders_mutex.lock().await.clone().into_iter().map( - |(type_, loader)| async { + loaders_mutex.read().await.clone().into_iter().map( + |(stable, loader)| async { { if versions_mutex.lock().await.iter().any(|x| { x.id == game_version.version - && x.loaders - .get(&type_) - .map(|x| x.id == loader) - .unwrap_or(false) + && x.loaders.iter().any(|x| x.id == loader) }) { return Ok(None); } @@ -78,8 +81,8 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error let version = fetch_fabric_version(&*game_version.version, &*loader).await?; - Ok::, Error>(Some( - (type_, loader, version), + Ok::, String, PartialVersionInfo)>, Error>(Some( + (stable, loader, version), )) }, ), @@ -88,7 +91,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error .into_iter() .flatten(); - futures::future::try_join_all(versions.map(|(type_, loader, version)| async { + futures::future::try_join_all(versions.map(|(stable, loader, version)| async { let libs = futures::future::try_join_all(version.libraries.into_iter().map( |mut lib| async { { @@ -164,13 +167,11 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error { let mut loader_version_map = loader_version_mutex.lock().await; async move { - loader_version_map.insert( - type_, - LoaderVersion { - id: format!("{}-{}", inherits_from, loader), - url: format_url(&*version_path), - }, - ); + loader_version_map.push(LoaderVersion { + id: format!("{}-{}", inherits_from, loader), + url: format_url(&*version_path), + stable: *stable, + }); } .await; } @@ -207,13 +208,46 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error } if let Ok(versions) = Arc::try_unwrap(versions) { + let mut versions = versions.into_inner(); + + versions.sort_by(|x, y| { + minecraft_versions + .versions + .iter() + .position(|z| x.id == z.id) + .unwrap_or_default() + .cmp( + &minecraft_versions + .versions + .iter() + .position(|z| y.id == z.id) + .unwrap_or_default(), + ) + }); + + for version in &mut versions { + version.loaders.sort_by(|x, y| { + list.loader + .iter() + .position(|z| x.id == z.version) + .unwrap_or_default() + .cmp( + &list + .loader + .iter() + .position(|z| y.id == z.version) + .unwrap_or_default(), + ) + }) + } + upload_file_to_bucket( format!( "fabric/v{}/manifest.json", daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, ), serde_json::to_vec(&Manifest { - game_versions: versions.into_inner(), + game_versions: versions, })?, Some("application/json".to_string()), uploaded_files_mutex.as_ref(), diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index baa81673..f8f8e188 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -1,10 +1,8 @@ use crate::{format_url, upload_file_to_bucket, Error}; use chrono::{DateTime, Utc}; use daedalus::download_file; -use daedalus::minecraft::{Argument, ArgumentType, Library, VersionType}; -use daedalus::modded::{ - LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry, -}; +use daedalus::minecraft::{Argument, ArgumentType, Library, VersionManifest, VersionType}; +use daedalus::modded::{LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry}; use lazy_static::lazy_static; use log::info; use semver::{Version, VersionReq}; @@ -25,7 +23,10 @@ lazy_static! { static ref FORGE_MANIFEST_V3_QUERY: VersionReq = VersionReq::parse(">=37.0.0").unwrap(); } -pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { +pub async fn retrieve_data( + minecraft_versions: &VersionManifest, + uploaded_files: &mut Vec, +) -> Result<(), Error> { let maven_metadata = fetch_maven_metadata(None).await?; let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!( "forge/v{}/manifest.json", @@ -34,18 +35,20 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error .await .ok(); - let versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { + let old_versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { old_manifest.game_versions } else { Vec::new() })); + let versions = Arc::new(Mutex::new(Vec::new())); + let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); let mut version_futures = Vec::new(); - for (minecraft_version, loader_versions) in maven_metadata { + for (minecraft_version, loader_versions) in maven_metadata.clone() { let mut loaders = Vec::new(); for loader_version_full in loader_versions { @@ -77,22 +80,21 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error } } } - - if let Some((loader_version_full, version)) = loaders.into_iter().last() { - version_futures.push(async { - let versions_mutex = Arc::clone(&versions); + version_futures.push(async { + let loaders_versions = futures::future::try_join_all(loaders.into_iter().map(|(loader_version_full, version)| async { + let versions_mutex = Arc::clone(&old_versions); let visited_assets = Arc::clone(&visited_assets_mutex); let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); + let minecraft_version = minecraft_version.clone(); + async move { { - if versions_mutex.lock().await.iter().any(|x| { - x.id == minecraft_version - && x.loaders - .get(&LoaderType::Latest) - .map(|x| x.id == loader_version_full) - .unwrap_or(false) - }) { - return Ok(()); + let versions = versions_mutex.lock().await; + let version = versions.iter().find(|x| + x.id == minecraft_version).map(|x| x.loaders.iter().find(|x| x.id == loader_version_full)).flatten(); + + if let Some(version) = version { + return Ok::, Error>(Some(version.clone())); } } @@ -199,15 +201,11 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error uploaded_files_mutex.as_ref() ).await?; - let mut map = HashMap::new(); - map.insert(LoaderType::Latest, LoaderVersion { + return Ok(Some(LoaderVersion { id: loader_version_full, - url: format_url(&*version_path) - }); - versions_mutex.lock().await.push(daedalus::modded::Version { - id: minecraft_version, - loaders: map - }) + url: format_url(&*version_path), + stable: false + })); } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { let mut archive_clone = archive.clone(); let mut profile = tokio::task::spawn_blocking(move || { @@ -342,7 +340,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { if let Some(ref mut artifact) = downloads.artifact { - let res = if artifact.url.is_empty() { + let res = if artifact.url.is_empty() { local_libs.get(&lib.name).cloned() } else { Some(daedalus::download_file( @@ -418,24 +416,25 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error uploaded_files_mutex.as_ref() ).await?; - let mut map = HashMap::new(); - map.insert(LoaderType::Latest, LoaderVersion { + return Ok(Some(LoaderVersion { id: loader_version_full, - url: format_url(&*version_path) - }); - versions_mutex.lock().await.push(daedalus::modded::Version { - id: minecraft_version, - loaders: map - }) + url: format_url(&*version_path), + stable: false + })); } } - Ok::<(), Error>(()) - }.await?; + Ok(None) + }.await + })).await?.into_iter().flatten().collect(); - Ok::<(), Error>(()) + versions.lock().await.push(daedalus::modded::Version { + id: minecraft_version, + loaders: loaders_versions }); - } + + Ok::<(), Error>(()) + }); } { @@ -458,13 +457,48 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error } if let Ok(versions) = Arc::try_unwrap(versions) { + let mut versions = versions.into_inner(); + + versions.sort_by(|x, y| { + minecraft_versions + .versions + .iter() + .position(|z| x.id == z.id) + .unwrap_or_default() + .cmp( + &minecraft_versions + .versions + .iter() + .position(|z| y.id == z.id) + .unwrap_or_default(), + ) + }); + + for version in &mut versions { + let loader_versions = maven_metadata.get(&version.id); + if let Some(loader_versions) = loader_versions { + version.loaders.sort_by(|x, y| { + loader_versions + .iter() + .position(|z| &y.id == z) + .unwrap_or_default() + .cmp( + &loader_versions + .iter() + .position(|z| &x.id == z) + .unwrap_or_default(), + ) + }) + } + } + upload_file_to_bucket( format!( "forge/v{}/manifest.json", daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, ), serde_json::to_vec(&Manifest { - game_versions: versions.into_inner(), + game_versions: versions, })?, Some("application/json".to_string()), uploaded_files_mutex.as_ref(), diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 123e2b4f..5729cd15 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -30,6 +30,8 @@ pub enum Error { ZipError(#[from] zip::result::ZipError), #[error("Error while reading zip file: {0}")] IoError(#[from] std::io::Error), + #[error("Error while obtaining strong reference to Arc")] + ArcError, } #[tokio::main] @@ -49,19 +51,26 @@ async fn main() { tokio::spawn(async { let mut uploaded_files = Vec::new(); - match fabric::retrieve_data(&mut uploaded_files).await { - Ok(..) => {} - Err(err) => error!("{:?}", err), - }; - match minecraft::retrieve_data(&mut uploaded_files).await { - Ok(..) => {} - Err(err) => error!("{:?}", err), - }; - match forge::retrieve_data(&mut uploaded_files).await { - Ok(..) => {} - Err(err) => error!("{:?}", err), + let versions = match minecraft::retrieve_data(&mut uploaded_files).await { + Ok(res) => Some(res), + Err(err) => { + error!("{:?}", err); + + None + } }; + if let Some(manifest) = versions { + match fabric::retrieve_data(&manifest, &mut uploaded_files).await { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; + match forge::retrieve_data(&manifest, &mut uploaded_files).await { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; + } + match purge_digitalocean_cache(uploaded_files).await { Ok(..) => {} Err(err) => error!("{:?}", err), diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index babb6a18..d3f7c72f 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,11 +1,12 @@ use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::download_file; +use daedalus::minecraft::VersionManifest; use log::info; use std::sync::Arc; use std::time::{Duration, Instant}; use tokio::sync::Mutex; -pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error> { +pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result { let old_manifest = daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&*format!( "minecraft/v{}/manifest.json", @@ -167,5 +168,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result<(), Error let elapsed = now.elapsed(); info!("Elapsed: {:.2?}", elapsed); - Ok(()) + Ok(Arc::try_unwrap(cloned_manifest) + .map_err(|_| Error::ArcError)? + .into_inner()) } From 2a588d1e9a8d31079bd059a535dbd12172bae6ec Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 19 Dec 2021 15:09:36 -0700 Subject: [PATCH 27/76] Make java stuff public, fix forge erroring out due to ratelimiting --- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 4 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/forge.rs | 541 ++++++++++++++++++----------------- daedalus_client/src/main.rs | 39 ++- 5 files changed, 305 insertions(+), 283 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 38aee979..c4f01bb4 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.8" +version = "0.1.9" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 250262cd..ce15217a 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -233,9 +233,9 @@ pub struct LibraryExtract { /// Information about the java version the game needs pub struct JavaVersion { /// The component needed for the Java installation - component: String, + pub component: String, /// The major Java version number - major_version: u32, + pub major_version: u32, } #[derive(Serialize, Deserialize, Debug)] diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 77ae6fd4..9d4cf1e4 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.8" +version = "0.1.9" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index f8f8e188..99429ac1 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -27,6 +27,7 @@ pub async fn retrieve_data( minecraft_versions: &VersionManifest, uploaded_files: &mut Vec, ) -> Result<(), Error> { + println!("forg"); let maven_metadata = fetch_maven_metadata(None).await?; let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!( "forge/v{}/manifest.json", @@ -81,189 +82,192 @@ pub async fn retrieve_data( } } version_futures.push(async { - let loaders_versions = futures::future::try_join_all(loaders.into_iter().map(|(loader_version_full, version)| async { - let versions_mutex = Arc::clone(&old_versions); - let visited_assets = Arc::clone(&visited_assets_mutex); - let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); - let minecraft_version = minecraft_version.clone(); + let mut loaders_versions = Vec::new(); - async move { - { - let versions = versions_mutex.lock().await; - let version = versions.iter().find(|x| - x.id == minecraft_version).map(|x| x.loaders.iter().find(|x| x.id == loader_version_full)).flatten(); + { + let mut loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async { + let versions_mutex = Arc::clone(&old_versions); + let visited_assets = Arc::clone(&visited_assets_mutex); + let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); + let minecraft_version = minecraft_version.clone(); - if let Some(version) = version { - return Ok::, Error>(Some(version.clone())); + async move { + { + let versions = versions_mutex.lock().await; + let version = versions.iter().find(|x| + x.id == minecraft_version).map(|x| x.loaders.iter().find(|x| x.id == loader_version_full)).flatten(); + + if let Some(version) = version { + return Ok::, Error>(Some(version.clone())); + } } - } - info!("Forge - Installer Start {}", loader_version_full.clone()); - let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; + info!("Forge - Installer Start {}", loader_version_full.clone()); + let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; - let reader = std::io::Cursor::new(bytes); + let reader = std::io::Cursor::new(bytes); - if let Ok(archive) = zip::ZipArchive::new(reader) { - if FORGE_MANIFEST_V1_QUERY.matches(&version) { - let mut archive_clone = archive.clone(); - let profile = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("install_profile.json")?; + if let Ok(archive) = zip::ZipArchive::new(reader) { + if FORGE_MANIFEST_V1_QUERY.matches(&version) { + let mut archive_clone = archive.clone(); + let profile = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("install_profile.json")?; - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; - Ok::(serde_json::from_str::(&*contents)?) - }).await??; + Ok::(serde_json::from_str::(&*contents)?) + }).await??; - let mut archive_clone = archive.clone(); - let file_path = profile.install.file_path.clone(); - let forge_universal_bytes = tokio::task::spawn_blocking(move || { - let mut forge_universal_file = archive_clone.by_name(&*file_path)?; - let mut forge_universal = Vec::new(); - forge_universal_file.read_to_end(&mut forge_universal)?; + let mut archive_clone = archive.clone(); + let file_path = profile.install.file_path.clone(); + let forge_universal_bytes = tokio::task::spawn_blocking(move || { + let mut forge_universal_file = archive_clone.by_name(&*file_path)?; + let mut forge_universal = Vec::new(); + forge_universal_file.read_to_end(&mut forge_universal)?; - Ok::(bytes::Bytes::from(forge_universal)) - }).await??; - let forge_universal_path = profile.install.path.clone(); + Ok::(bytes::Bytes::from(forge_universal)) + }).await??; + let forge_universal_path = profile.install.path.clone(); - let now = Instant::now(); - let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { - if let Some(url) = lib.url { - { - let mut visited_assets = visited_assets.lock().await; + let now = Instant::now(); + let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { + if let Some(url) = lib.url { + { + let mut visited_assets = visited_assets.lock().await; - if visited_assets.contains(&lib.name) { - lib.url = Some(format_url("maven/")); + if visited_assets.contains(&lib.name) { + lib.url = Some(format_url("maven/")); - return Ok::(lib); - } else { - visited_assets.push(lib.name.clone()) + return Ok::(lib); + } else { + visited_assets.push(lib.name.clone()) + } } + + let artifact_path = + daedalus::get_path_from_artifact(&*lib.name)?; + + let artifact = if lib.name == forge_universal_path { + forge_universal_bytes.clone() + } else { + let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; + + daedalus::download_file_mirrors( + &*artifact_path, + &mirrors, + None, + ) + .await? + }; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref(), + ).await?; } - let artifact_path = - daedalus::get_path_from_artifact(&*lib.name)?; + Ok::(lib) + })).await?; - let artifact = if lib.name == forge_universal_path { - forge_universal_bytes.clone() - } else { - let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; + let elapsed = now.elapsed(); + info!("Elapsed lib DL: {:.2?}", elapsed); - daedalus::download_file_mirrors( - &*artifact_path, - &mirrors, - None, - ) - .await? - }; + let new_profile = PartialVersionInfo { + id: profile.version_info.id, + inherits_from: profile.install.minecraft, + release_time: profile.version_info.release_time, + time: profile.version_info.time, + main_class: profile.version_info.main_class, + minecraft_arguments: profile.version_info.minecraft_arguments.clone(), + arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), + libraries: libs, + type_: profile.version_info.type_, + data: None, + processors: None + }; - lib.url = Some(format_url("maven/")); + let version_path = format!( + "forge/v{}/versions/{}.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + new_profile.id + ); - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref(), - ).await?; + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + uploaded_files_mutex.as_ref() + ).await?; + + return Ok(Some(LoaderVersion { + id: loader_version_full, + url: format_url(&*version_path), + stable: false + })); + } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { + let mut archive_clone = archive.clone(); + let mut profile = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("install_profile.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + Ok::(serde_json::from_str::(&*contents)?) + }).await??; + + let mut archive_clone = archive.clone(); + let version_info = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("version.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + Ok::(serde_json::from_str::(&*contents)?) + }).await??; + + + let mut libs : Vec = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library { + downloads: x.downloads, + extract: x.extract, + name: x.name, + url: x.url, + natives: x.natives, + rules: x.rules, + checksums: x.checksums, + include_in_classpath: false + })).collect(); + + let mut local_libs : HashMap = HashMap::new(); + + for lib in &libs { + if lib.downloads.as_ref().map(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).flatten().unwrap_or(false) { + let mut archive_clone = archive.clone(); + let lib_name_clone = lib.name.clone(); + + let lib_bytes = tokio::task::spawn_blocking(move || { + let mut lib_file = archive_clone.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*lib_name_clone)?))?; + let mut lib_bytes = Vec::new(); + lib_file.read_to_end(&mut lib_bytes)?; + + Ok::(bytes::Bytes::from(lib_bytes)) + }).await??; + + local_libs.insert(lib.name.clone(), lib_bytes); + } } - Ok::(lib) - })).await?; + let path = profile.path.clone(); + let version = profile.version.clone(); - let elapsed = now.elapsed(); - info!("Elapsed lib DL: {:.2?}", elapsed); - - let new_profile = PartialVersionInfo { - id: profile.version_info.id, - inherits_from: profile.install.minecraft, - release_time: profile.version_info.release_time, - time: profile.version_info.time, - main_class: profile.version_info.main_class, - minecraft_arguments: profile.version_info.minecraft_arguments.clone(), - arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), - libraries: libs, - type_: profile.version_info.type_, - data: None, - processors: None - }; - - let version_path = format!( - "forge/v{}/versions/{}.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - new_profile.id - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&new_profile)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref() - ).await?; - - return Ok(Some(LoaderVersion { - id: loader_version_full, - url: format_url(&*version_path), - stable: false - })); - } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { - let mut archive_clone = archive.clone(); - let mut profile = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("install_profile.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&*contents)?) - }).await??; - - let mut archive_clone = archive.clone(); - let version_info = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("version.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&*contents)?) - }).await??; - - - let mut libs : Vec = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library { - downloads: x.downloads, - extract: x.extract, - name: x.name, - url: x.url, - natives: x.natives, - rules: x.rules, - checksums: x.checksums, - include_in_classpath: false - })).collect(); - - let mut local_libs : HashMap = HashMap::new(); - - for lib in &libs { - if lib.downloads.as_ref().map(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).flatten().unwrap_or(false) { - let mut archive_clone = archive.clone(); - let lib_name_clone = lib.name.clone(); - - let lib_bytes = tokio::task::spawn_blocking(move || { - let mut lib_file = archive_clone.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*lib_name_clone)?))?; - let mut lib_bytes = Vec::new(); - lib_file.read_to_end(&mut lib_bytes)?; - - Ok::(bytes::Bytes::from(lib_bytes)) - }).await??; - - local_libs.insert(lib.name.clone(), lib_bytes); - } - } - - let path = profile.path.clone(); - let version = profile.version.clone(); - - for entry in profile.data.values_mut() { - if entry.client.starts_with('/') || entry.server.starts_with('/') { - macro_rules! read_data { + for entry in profile.data.values_mut() { + if entry.client.starts_with('/') || entry.server.starts_with('/') { + macro_rules! read_data { ($value:expr) => { let mut archive_clone = archive.clone(); let value_clone = $value.clone(); @@ -302,131 +306,150 @@ pub async fn retrieve_data( } } - if entry.client.starts_with('/') { - read_data!(entry.client); - } - - // Do we really need to support server installs? Keeping this here - // just in case - // - // if entry.server.starts_with('/') { - // read_data!(entry.server); - // } - } - } - - let now = Instant::now(); - let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { - let artifact_path = - daedalus::get_path_from_artifact(&*lib.name)?; - - { - let mut visited_assets = visited_assets.lock().await; - - if visited_assets.contains(&lib.name) { - if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - artifact.url = format_url(&*format!("maven/{}", artifact_path)); - } - } else if lib.url.is_some() { - lib.url = Some(format_url("maven/")); + if entry.client.starts_with('/') { + read_data!(entry.client); } - return Ok::(lib); - } else { - visited_assets.push(lib.name.clone()) + // Do we really need to support server installs? Keeping this here + // just in case + // + // if entry.server.starts_with('/') { + // read_data!(entry.server); + // } } } - let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - let res = if artifact.url.is_empty() { + let now = Instant::now(); + let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { + let artifact_path = + daedalus::get_path_from_artifact(&*lib.name)?; + + { + let mut visited_assets = visited_assets.lock().await; + + if visited_assets.contains(&lib.name) { + if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { + artifact.url = format_url(&*format!("maven/{}", artifact_path)); + } + } else if lib.url.is_some() { + lib.url = Some(format_url("maven/")); + } + + return Ok::(lib); + } else { + visited_assets.push(lib.name.clone()) + } + } + + let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { + let res = if artifact.url.is_empty() { + local_libs.get(&lib.name).cloned() + } else { + Some(daedalus::download_file( + &*artifact.url, + Some(&*artifact.sha1), + ) + .await?) + }; + + if res.is_some() { + artifact.url = format_url(&*format!("maven/{}", artifact_path)); + } + + res + } else { None } + } else if let Some(ref mut url) = lib.url { + let res = if url.is_empty() { local_libs.get(&lib.name).cloned() } else { Some(daedalus::download_file( - &*artifact.url, - Some(&*artifact.sha1), + url, + None, ) .await?) }; if res.is_some() { - artifact.url = format_url(&*format!("maven/{}", artifact_path)); + lib.url = Some(format_url("maven/")); } res - } else { None } - } else if let Some(ref mut url) = lib.url { - let res = if url.is_empty() { - local_libs.get(&lib.name).cloned() - } else { - Some(daedalus::download_file( - url, - None, - ) - .await?) - }; + } else { None }; - if res.is_some() { - lib.url = Some(format_url("maven/")); + if let Some(bytes) = artifact_bytes { + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + bytes.to_vec(), + Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref() + ).await?; } - res - } else { None }; + Ok::(lib) + })).await?; - if let Some(bytes) = artifact_bytes { - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - bytes.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref() - ).await?; - } + let elapsed = now.elapsed(); + info!("Elapsed lib DL: {:.2?}", elapsed); - Ok::(lib) - })).await?; + let new_profile = PartialVersionInfo { + id: version_info.id, + inherits_from: version_info.inherits_from, + release_time: version_info.release_time, + time: version_info.time, + main_class: version_info.main_class, + minecraft_arguments: version_info.minecraft_arguments, + arguments: version_info.arguments, + libraries: libs, + type_: version_info.type_, + data: Some(profile.data), + processors: Some(profile.processors), + }; - let elapsed = now.elapsed(); - info!("Elapsed lib DL: {:.2?}", elapsed); + let version_path = format!( + "forge/v{}/versions/{}.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + new_profile.id + ); - let new_profile = PartialVersionInfo { - id: version_info.id, - inherits_from: version_info.inherits_from, - release_time: version_info.release_time, - time: version_info.time, - main_class: version_info.main_class, - minecraft_arguments: version_info.minecraft_arguments, - arguments: version_info.arguments, - libraries: libs, - type_: version_info.type_, - data: Some(profile.data), - processors: Some(profile.processors), - }; + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + uploaded_files_mutex.as_ref() + ).await?; - let version_path = format!( - "forge/v{}/versions/{}.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - new_profile.id - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&new_profile)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref() - ).await?; - - return Ok(Some(LoaderVersion { - id: loader_version_full, - url: format_url(&*version_path), - stable: false - })); + return Ok(Some(LoaderVersion { + id: loader_version_full, + url: format_url(&*version_path), + stable: false + })); + } } - } - Ok(None) - }.await - })).await?.into_iter().flatten().collect(); + Ok(None) + }.await + }).into_iter().peekable()/*.into_iter().flatten().collect()*/; + + let mut chunk_index = 0; + while loaders_futures.peek().is_some() { + info!("Loader Chunk {} Start", chunk_index); + let now = Instant::now(); + + let chunk: Vec<_> = loaders_futures.by_ref().take(10).collect(); + + let res = futures::future::try_join_all(chunk).await?; + loaders_versions.extend(res.into_iter().flatten()); + + tokio::time::sleep(Duration::from_secs(1)).await; + + chunk_index += 1; + + let elapsed = now.elapsed(); + info!("Loader Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } + } versions.lock().await.push(daedalus::modded::Version { id: minecraft_version, @@ -444,7 +467,7 @@ pub async fn retrieve_data( info!("Chunk {} Start", chunk_index); let now = Instant::now(); - let chunk: Vec<_> = versions_peek.by_ref().take(10).collect(); + let chunk: Vec<_> = versions_peek.by_ref().take(1).collect(); futures::future::try_join_all(chunk).await?; tokio::time::sleep(Duration::from_secs(1)).await; diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 5729cd15..ee0586e0 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -48,34 +48,33 @@ async fn main() { loop { timer.tick().await; - tokio::spawn(async { - let mut uploaded_files = Vec::new(); - let versions = match minecraft::retrieve_data(&mut uploaded_files).await { - Ok(res) => Some(res), - Err(err) => { - error!("{:?}", err); + let mut uploaded_files = Vec::new(); - None - } - }; + let versions = match minecraft::retrieve_data(&mut uploaded_files).await { + Ok(res) => Some(res), + Err(err) => { + error!("{:?}", err); - if let Some(manifest) = versions { - match fabric::retrieve_data(&manifest, &mut uploaded_files).await { - Ok(..) => {} - Err(err) => error!("{:?}", err), - }; - match forge::retrieve_data(&manifest, &mut uploaded_files).await { - Ok(..) => {} - Err(err) => error!("{:?}", err), - }; + None } + }; - match purge_digitalocean_cache(uploaded_files).await { + if let Some(manifest) = versions { + // match fabric::retrieve_data(&manifest, &mut uploaded_files).await { + // Ok(..) => {} + // Err(err) => error!("{:?}", err), + // }; + match forge::retrieve_data(&manifest, &mut uploaded_files).await { Ok(..) => {} Err(err) => error!("{:?}", err), }; - }); + } + + match purge_digitalocean_cache(uploaded_files).await { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; } } From c9598b674c803fbac031f88138f0eb34ecaedeaf Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 19 Dec 2021 18:52:00 -0700 Subject: [PATCH 28/76] Fix forge erroring out due to wonky version --- daedalus/Cargo.toml | 2 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/forge.rs | 10 ++++++++++ daedalus_client/src/main.rs | 8 ++++---- 4 files changed, 16 insertions(+), 6 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index c4f01bb4..b6b09a1a 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.9" +version = "0.1.10" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 9d4cf1e4..e196417d 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.9" +version = "0.1.10" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 99429ac1..a2fba717 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -92,6 +92,16 @@ pub async fn retrieve_data( let minecraft_version = minecraft_version.clone(); async move { + /// These forge versions are not worth supporting! + const WHITELIST : [&str; 1] = [ + // Not supported due to `data` field being `[]` even though the type is a map + "1.12.2-14.23.5.2851" + ]; + + if !WHITELIST.contains(&&*loader_version_full) { + return Ok(None); + } + { let versions = versions_mutex.lock().await; let version = versions.iter().find(|x| diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index ee0586e0..4a4f6766 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -61,10 +61,10 @@ async fn main() { }; if let Some(manifest) = versions { - // match fabric::retrieve_data(&manifest, &mut uploaded_files).await { - // Ok(..) => {} - // Err(err) => error!("{:?}", err), - // }; + match fabric::retrieve_data(&manifest, &mut uploaded_files).await { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; match forge::retrieve_data(&manifest, &mut uploaded_files).await { Ok(..) => {} Err(err) => error!("{:?}", err), From 80173634a02a86f1c4ecb2240e598e3541b51e5d Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 19 Dec 2021 19:04:42 -0700 Subject: [PATCH 29/76] Fix java version specifier being in snake case --- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 1 + daedalus_client/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index b6b09a1a..fedac092 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.10" +version = "0.1.11" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index ce15217a..73ee8097 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -230,6 +230,7 @@ pub struct LibraryExtract { } #[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] /// Information about the java version the game needs pub struct JavaVersion { /// The component needed for the Java installation diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index e196417d..60943cb6 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.10" +version = "0.1.11" authors = ["Jai A "] edition = "2018" From 1d86aac33800d84e2f227f3d17f8d6c046d452e3 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 19 Dec 2021 19:56:56 -0700 Subject: [PATCH 30/76] Fix java version not being optional --- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 2 +- daedalus_client/Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index fedac092..3190f9e0 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.11" +version = "0.1.12" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 73ee8097..339cbfef 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -323,7 +323,7 @@ pub struct VersionInfo { pub id: String, /// The Java version this version supports - pub java_version: JavaVersion, + pub java_version: Option, /// Libraries that the version depends on pub libraries: Vec, /// The classpath to the main class to launch the game diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 60943cb6..c7f1d2b6 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.11" +version = "0.1.12" authors = ["Jai A "] edition = "2018" From 09aef18999b1904c6e0639435b188c7d130a92f4 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 19 Dec 2021 20:51:34 -0700 Subject: [PATCH 31/76] Fix incorrect condition for forge and incorrect fabric loader version ordering --- daedalus/Cargo.toml | 2 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/fabric.rs | 14 ++++++++++++-- daedalus_client/src/forge.rs | 3 +-- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 3190f9e0..3daee5cf 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.12" +version = "0.1.13" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index c7f1d2b6..cf925cfc 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.12" +version = "0.1.13" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 514252c9..fdc3cf46 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -229,13 +229,23 @@ pub async fn retrieve_data( version.loaders.sort_by(|x, y| { list.loader .iter() - .position(|z| x.id == z.version) + .position(|z| { + x.id.split('-') + .next() + .unwrap_or_default() + == &*z.version + }) .unwrap_or_default() .cmp( &list .loader .iter() - .position(|z| y.id == z.version) + .position(|z| { + y.id.split('-') + .next() + .unwrap_or_default() + == z.version + }) .unwrap_or_default(), ) }) diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index a2fba717..c0e841c5 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -27,7 +27,6 @@ pub async fn retrieve_data( minecraft_versions: &VersionManifest, uploaded_files: &mut Vec, ) -> Result<(), Error> { - println!("forg"); let maven_metadata = fetch_maven_metadata(None).await?; let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!( "forge/v{}/manifest.json", @@ -98,7 +97,7 @@ pub async fn retrieve_data( "1.12.2-14.23.5.2851" ]; - if !WHITELIST.contains(&&*loader_version_full) { + if WHITELIST.contains(&&*loader_version_full) { return Ok(None); } From 00adf3631a5e3111798349f0dea5d416c56f122d Mon Sep 17 00:00:00 2001 From: Jai A Date: Mon, 20 Dec 2021 09:11:40 -0700 Subject: [PATCH 32/76] Remove replacing assets index URL in manifest to preserve file integrity --- daedalus/Cargo.toml | 2 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/minecraft.rs | 1 - 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 3daee5cf..448ead6e 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.13" +version = "0.1.14" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index cf925cfc..bf447332 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.13" +version = "0.1.14" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index d3f7c72f..58fae665 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -75,7 +75,6 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result Date: Sun, 26 Jun 2022 16:23:41 -0700 Subject: [PATCH 33/76] Add Bincode feature for efficient binary storage --- daedalus/Cargo.toml | 1 + daedalus/src/minecraft.rs | 48 +++++++++++++++++++++++++++++++++++---- daedalus/src/modded.rs | 24 +++++++++++++++++--- rustfmt.toml | 2 ++ 4 files changed, 68 insertions(+), 7 deletions(-) create mode 100644 rustfmt.toml diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 448ead6e..abd0f5af 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -22,3 +22,4 @@ bytes = "1" thiserror = "1.0" tokio = { version = "1", features = ["full"] } sha1 = { version = "0.6.0", features = ["std"]} +bincode = {version = "2.0.0-rc", features = ["serde"], optional = true} diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 339cbfef..c86c39a2 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -4,9 +4,13 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +#[cfg(feature = "bincode")] +use bincode::{Decode, Encode}; + /// The latest version of the format the model structs deserialize to pub const CURRENT_FORMAT_VERSION: usize = 0; +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "snake_case")] /// The version type @@ -33,6 +37,7 @@ impl VersionType { } } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] /// A game version of Minecraft @@ -45,8 +50,10 @@ pub struct Version { /// A link to additional information about the version pub url: String, /// The latest time a file in this version was updated + #[bincode(with_serde)] pub time: DateTime, /// The time this version was released + #[bincode(with_serde)] pub release_time: DateTime, /// The SHA1 hash of the additional information about the version pub sha1: String, @@ -62,6 +69,7 @@ pub struct Version { pub assets_index_sha1: Option, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// The latest snapshot and release of the game pub struct LatestVersion { @@ -71,6 +79,7 @@ pub struct LatestVersion { pub snapshot: String, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// Data of all game versions of Minecraft pub struct VersionManifest { @@ -85,12 +94,15 @@ pub const VERSION_MANIFEST_URL: &str = "https://launchermeta.mojang.com/mc/game/version_manifest_v2.json"; /// Fetches a version manifest from the specified URL. If no URL is specified, the default is used. -pub async fn fetch_version_manifest(url: Option<&str>) -> Result { +pub async fn fetch_version_manifest( + url: Option<&str>, +) -> Result { Ok(serde_json::from_slice( &download_file(url.unwrap_or(VERSION_MANIFEST_URL), None).await?, )?) } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// Information about the assets of the game @@ -107,6 +119,7 @@ pub struct AssetIndex { pub url: String, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)] #[serde(rename_all = "snake_case")] /// The type of download @@ -123,6 +136,7 @@ pub enum DownloadType { WindowsServer, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// Download information of a file pub struct Download { @@ -134,6 +148,7 @@ pub struct Download { pub url: String, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// Download information of a library pub struct LibraryDownload { @@ -147,6 +162,7 @@ pub struct LibraryDownload { pub url: String, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// A list of files that should be downloaded for libraries pub struct LibraryDownloads { @@ -159,6 +175,7 @@ pub struct LibraryDownloads { pub classifiers: Option>, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "snake_case")] /// The action a rule can follow @@ -169,6 +186,7 @@ pub enum RuleAction { Disallow, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] #[serde(rename_all = "snake_case")] /// An enum representing the different types of operating systems @@ -183,6 +201,7 @@ pub enum Os { Unknown, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A rule which depends on what OS the user is on pub struct OsRule { @@ -197,6 +216,7 @@ pub struct OsRule { pub arch: Option, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A rule which depends on the toggled features of the launcher pub struct FeatureRule { @@ -208,6 +228,7 @@ pub struct FeatureRule { pub has_demo_resolution: Option, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A rule deciding whether a file is downloaded, an argument is used, etc. pub struct Rule { @@ -221,6 +242,7 @@ pub struct Rule { pub features: Option, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// Information delegating the extraction of the library pub struct LibraryExtract { @@ -229,6 +251,7 @@ pub struct LibraryExtract { pub exclude: Option>, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// Information about the java version the game needs @@ -239,6 +262,7 @@ pub struct JavaVersion { pub major_version: u32, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// A library which the game relies on to run pub struct Library { @@ -271,6 +295,7 @@ fn default_include_in_classpath() -> bool { true } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(untagged)] /// A container for an argument or multiple arguments @@ -281,6 +306,7 @@ pub enum ArgumentValue { Many(Vec), } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(untagged)] /// A command line argument passed to a program @@ -296,6 +322,7 @@ pub enum Argument { }, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone, Copy)] #[serde(rename_all = "snake_case")] /// The type of argument @@ -306,6 +333,7 @@ pub enum ArgumentType { Jvm, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// Information about a version @@ -334,8 +362,10 @@ pub struct VersionInfo { /// The minimum version of the Minecraft Launcher that can run this version of the game pub minimum_launcher_version: u32, /// The time that the version was released + #[bincode(with_serde)] pub release_time: DateTime, /// The latest time a file in this version was updated + #[bincode(with_serde)] pub time: DateTime, #[serde(rename = "type")] /// The type of version @@ -349,12 +379,15 @@ pub struct VersionInfo { } /// Fetches detailed information about a version from the manifest -pub async fn fetch_version_info(version: &Version) -> Result { +pub async fn fetch_version_info( + version: &Version, +) -> Result { Ok(serde_json::from_slice( &download_file(&version.url, Some(&version.sha1)).await?, )?) } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// An asset of the game pub struct Asset { @@ -364,6 +397,7 @@ pub struct Asset { pub size: u32, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// An index containing all assets the game needs pub struct AssetsIndex { @@ -372,8 +406,14 @@ pub struct AssetsIndex { } /// Fetches the assets index from the version info -pub async fn fetch_assets_index(version: &VersionInfo) -> Result { +pub async fn fetch_assets_index( + version: &VersionInfo, +) -> Result { Ok(serde_json::from_slice( - &download_file(&version.asset_index.url, Some(&version.asset_index.sha1)).await?, + &download_file( + &version.asset_index.url, + Some(&version.asset_index.sha1), + ) + .await?, )?) } diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index ea3e07c9..4e51604e 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -1,16 +1,22 @@ use crate::{download_file, Error}; -use crate::minecraft::{Argument, ArgumentType, Library, VersionInfo, VersionType}; +use crate::minecraft::{ + Argument, ArgumentType, Library, VersionInfo, VersionType, +}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; +#[cfg(feature = "bincode")] +use bincode::{Decode, Encode}; + /// The latest version of the format the fabric model structs deserialize to pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0; /// The latest version of the format the fabric model structs deserialize to pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0; /// A data variable entry that depends on the side of the installation +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] pub struct SidedDataEntry { /// The value on the client @@ -19,6 +25,7 @@ pub struct SidedDataEntry { pub server: String, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// A partial version returned by fabric meta @@ -28,8 +35,10 @@ pub struct PartialVersionInfo { /// The version ID this partial version inherits from pub inherits_from: String, /// The time that the version was released + #[bincode(with_serde)] pub release_time: DateTime, /// The latest time a file in this version was updated + #[bincode(with_serde)] pub time: DateTime, #[serde(skip_serializing_if = "Option::is_none")] /// The classpath to the main class to launch the game @@ -54,6 +63,7 @@ pub struct PartialVersionInfo { } /// A processor to be ran after downloading the files +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] pub struct Processor { /// Maven coordinates for the JAR library of this processor. @@ -72,12 +82,17 @@ pub struct Processor { } /// Fetches the version manifest of a game version's URL -pub async fn fetch_partial_version(url: &str) -> Result { +pub async fn fetch_partial_version( + url: &str, +) -> Result { Ok(serde_json::from_slice(&download_file(url, None).await?)?) } /// Merges a partial version into a complete one -pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> VersionInfo { +pub fn merge_partial_version( + partial: PartialVersionInfo, + merge: VersionInfo, +) -> VersionInfo { VersionInfo { arguments: if let Some(partial_args) = partial.arguments { if let Some(merge_args) = merge.arguments { @@ -133,6 +148,7 @@ pub fn merge_partial_version(partial: PartialVersionInfo, merge: VersionInfo) -> } } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] /// A manifest containing information about a mod loader's versions @@ -141,6 +157,7 @@ pub struct Manifest { pub game_versions: Vec, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A game version of Minecraft pub struct Version { @@ -150,6 +167,7 @@ pub struct Version { pub loaders: Vec, } +#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A version of a Minecraft mod loader pub struct LoaderVersion { diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 00000000..f5a8b867 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,2 @@ +edition = "2018" +max_width = 80 \ No newline at end of file From 18153e0fcc4d0dd0460b098f503717075b5dd538 Mon Sep 17 00:00:00 2001 From: Danielle Hutzley Date: Sun, 26 Jun 2022 16:23:57 -0700 Subject: [PATCH 34/76] Bump version --- daedalus/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index abd0f5af..694da126 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.14" +version = "0.1.15" authors = ["Jai A "] edition = "2018" license = "MIT" From 93817ba92f90ac5bb126869aba4a3dfbfd390656 Mon Sep 17 00:00:00 2001 From: Danielle Hutzley Date: Sun, 26 Jun 2022 16:41:22 -0700 Subject: [PATCH 35/76] Actually compile without Bincode --- daedalus/src/minecraft.rs | 8 ++++---- daedalus/src/modded.rs | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index c86c39a2..478f6904 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -50,10 +50,10 @@ pub struct Version { /// A link to additional information about the version pub url: String, /// The latest time a file in this version was updated - #[bincode(with_serde)] + #[cfg_attr(feature = "bincode", bincode(with_serde))] pub time: DateTime, /// The time this version was released - #[bincode(with_serde)] + #[cfg_attr(feature = "bincode", bincode(with_serde))] pub release_time: DateTime, /// The SHA1 hash of the additional information about the version pub sha1: String, @@ -362,10 +362,10 @@ pub struct VersionInfo { /// The minimum version of the Minecraft Launcher that can run this version of the game pub minimum_launcher_version: u32, /// The time that the version was released - #[bincode(with_serde)] + #[cfg_attr(feature = "bincode", bincode(with_serde))] pub release_time: DateTime, /// The latest time a file in this version was updated - #[bincode(with_serde)] + #[cfg_attr(feature = "bincode", bincode(with_serde))] pub time: DateTime, #[serde(rename = "type")] /// The type of version diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 4e51604e..2e62c981 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -35,10 +35,10 @@ pub struct PartialVersionInfo { /// The version ID this partial version inherits from pub inherits_from: String, /// The time that the version was released - #[bincode(with_serde)] + #[cfg_attr(feature = "bincode", bincode(with_serde))] pub release_time: DateTime, /// The latest time a file in this version was updated - #[bincode(with_serde)] + #[cfg_attr(feature = "bincode", bincode(with_serde))] pub time: DateTime, #[serde(skip_serializing_if = "Option::is_none")] /// The classpath to the main class to launch the game From c2a1ed926e62c9713aeda3a148a247e0d43c49d8 Mon Sep 17 00:00:00 2001 From: Danielle Hutzley Date: Mon, 27 Jun 2022 13:54:09 -0700 Subject: [PATCH 36/76] Migrate to piston-meta --- daedalus/.#Cargo.toml | 1 + daedalus/src/minecraft.rs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) create mode 120000 daedalus/.#Cargo.toml diff --git a/daedalus/.#Cargo.toml b/daedalus/.#Cargo.toml new file mode 120000 index 00000000..655ff6ec --- /dev/null +++ b/daedalus/.#Cargo.toml @@ -0,0 +1 @@ +enderger@DannyPC.2785515:1655912895 \ No newline at end of file diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 478f6904..247274a5 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -91,7 +91,7 @@ pub struct VersionManifest { /// The URL to the version manifest pub const VERSION_MANIFEST_URL: &str = - "https://launchermeta.mojang.com/mc/game/version_manifest_v2.json"; + "https://piston-meta.mojang.com/mc/game/version_manifest_v2.json"; /// Fetches a version manifest from the specified URL. If no URL is specified, the default is used. pub async fn fetch_version_manifest( From 827c4e31ee7375b5c2e70ebf67dc45d7927eb0ae Mon Sep 17 00:00:00 2001 From: Danielle Hutzley Date: Mon, 27 Jun 2022 13:59:06 -0700 Subject: [PATCH 37/76] fixup! Migrate to piston-meta --- daedalus/.#Cargo.toml | 1 - daedalus/Cargo.toml | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 120000 daedalus/.#Cargo.toml diff --git a/daedalus/.#Cargo.toml b/daedalus/.#Cargo.toml deleted file mode 120000 index 655ff6ec..00000000 --- a/daedalus/.#Cargo.toml +++ /dev/null @@ -1 +0,0 @@ -enderger@DannyPC.2785515:1655912895 \ No newline at end of file diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 694da126..9b5929c5 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.15" +version = "0.1.16" authors = ["Jai A "] edition = "2018" license = "MIT" From 45dbf5393f62fb019ff6ff82b3495c18e856d0d7 Mon Sep 17 00:00:00 2001 From: Jai A Date: Thu, 22 Dec 2022 19:01:41 -0700 Subject: [PATCH 38/76] Bump versions + switch AWS/S3 library --- .env | 4 - daedalus/Cargo.toml | 6 +- daedalus/src/lib.rs | 12 +-- daedalus_client/Cargo.toml | 20 +++-- daedalus_client/src/fabric.rs | 18 ++--- daedalus_client/src/forge.rs | 36 ++++----- daedalus_client/src/main.rs | 135 ++++++++++--------------------- daedalus_client/src/minecraft.rs | 6 +- 8 files changed, 90 insertions(+), 147 deletions(-) diff --git a/.env b/.env index dc87710f..34bf98cd 100644 --- a/.env +++ b/.env @@ -8,7 +8,3 @@ S3_SECRET=none S3_URL=none S3_REGION=none S3_BUCKET_NAME=none - -DO_INTEGRATION=false -DO_ACCESS_KEY=none -DO_ENDPOINT_ID=none \ No newline at end of file diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 9b5929c5..3909d840 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.16" +version = "0.1.17" authors = ["Jai A "] edition = "2018" license = "MIT" @@ -21,5 +21,5 @@ chrono = { version = "0.4", features = ["serde"] } bytes = "1" thiserror = "1.0" tokio = { version = "1", features = ["full"] } -sha1 = { version = "0.6.0", features = ["std"]} -bincode = {version = "2.0.0-rc", features = ["serde"], optional = true} +sha1 = { version = "0.6.1", features = ["std"]} +bincode = {version = "2.0.0-rc.2", features = ["serde"], optional = true} diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index 77657323..678bf1c4 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -45,7 +45,7 @@ pub enum Error { pub fn get_path_from_artifact(artifact: &str) -> Result { let name_items = artifact.split(':').collect::>(); - let package = name_items.get(0).ok_or_else(|| { + let package = name_items.first().ok_or_else(|| { Error::ParseError(format!("Unable to find package for library {}", &artifact)) })?; let name = name_items.get(1).ok_or_else(|| { @@ -60,14 +60,14 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { })? .split('@') .collect::>(); - let version = version_ext.get(0).ok_or_else(|| { + let version = version_ext.first().ok_or_else(|| { Error::ParseError(format!("Unable to find version for library {}", &artifact)) })?; let ext = version_ext.get(1); Ok(format!( "{}/{}/{}/{}-{}.{}", - package.replace(".", "/"), + package.replace('.', "/"), name, version, name, @@ -86,14 +86,14 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { })? .split('@') .collect::>(); - let data = data_ext.get(0).ok_or_else(|| { + let data = data_ext.first().ok_or_else(|| { Error::ParseError(format!("Unable to find data for library {}", &artifact)) })?; let ext = data_ext.get(1); Ok(format!( "{}/{}/{}/{}-{}-{}.{}", - package.replace(".", "/"), + package.replace('.', "/"), name, version, name, @@ -115,7 +115,7 @@ pub async fn download_file_mirrors( } for (index, mirror) in mirrors.iter().enumerate() { - let result = download_file(&*format!("{}{}", mirror, base), sha1).await; + let result = download_file(&format!("{}{}", mirror, base), sha1).await; if result.is_ok() || (result.is_err() && index == (mirrors.len() - 1)) { return result; diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index bf447332..03558872 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.14" +version = "0.1.17" authors = ["Jai A "] edition = "2018" @@ -9,19 +9,17 @@ edition = "2018" [dependencies] daedalus = { path = "../daedalus" } tokio = { version = "1", features = ["full"] } -futures = "0.3.17" -dotenv = "0.15.0" -log = "0.4.8" -env_logger="0.9.0" +futures = "0.3.25" +dotenvy = "0.15.6" +log = "0.4.17" +env_logger= "0.10.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" lazy_static = "1.4.0" thiserror = "1.0" -reqwest = "0.11.4" -zip = "0.5.13" +reqwest = "0.11.13" +zip = "0.6.3" semver = "1.0" chrono = { version = "0.4", features = ["serde"] } -bytes = "1.1.0" - -rusoto_core = "0.47.0" -rusoto_s3 = "0.47.0" +bytes = "1.3.0" +rust-s3 = "0.32.3" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index fdc3cf46..3aef4fdc 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -13,7 +13,7 @@ pub async fn retrieve_data( uploaded_files: &mut Vec, ) -> Result<(), Error> { let mut list = fetch_fabric_versions(None).await?; - let old_manifest = daedalus::modded::fetch_manifest(&*format!( + let old_manifest = daedalus::modded::fetch_manifest(&format!( "fabric/v{}/manifest.json", daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, )) @@ -47,11 +47,7 @@ pub async fn retrieve_data( } } - list.loader = list - .loader - .into_iter() - .filter(|x| loaders.iter().any(|val| val.1 == x.version)) - .collect(); + list.loader.retain(|x| loaders.iter().any(|val| val.1 == x.version)) } let mut version_futures = Vec::new(); @@ -79,7 +75,7 @@ pub async fn retrieve_data( } let version = - fetch_fabric_version(&*game_version.version, &*loader).await?; + fetch_fabric_version(&game_version.version, &loader).await?; Ok::, String, PartialVersionInfo)>, Error>(Some( (stable, loader, version), @@ -106,10 +102,10 @@ pub async fn retrieve_data( } } - let artifact_path = daedalus::get_path_from_artifact(&*lib.name)?; + let artifact_path = daedalus::get_path_from_artifact(&lib.name)?; let artifact = daedalus::download_file( - &*format!( + &format!( "{}{}", lib.url.unwrap_or_else(|| { "https://maven.fabricmc.net/".to_string() @@ -169,7 +165,7 @@ pub async fn retrieve_data( async move { loader_version_map.push(LoaderVersion { id: format!("{}-{}", inherits_from, loader), - url: format_url(&*version_path), + url: format_url(&version_path), stable: *stable, }); } @@ -280,7 +276,7 @@ async fn fetch_fabric_version( ) -> Result { Ok(serde_json::from_slice( &download_file( - &*format!( + &format!( "{}/versions/loader/{}/{}/profile/json", FABRIC_META_URL, version_number, loader_version ), diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index c0e841c5..25da023a 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -28,7 +28,7 @@ pub async fn retrieve_data( uploaded_files: &mut Vec, ) -> Result<(), Error> { let maven_metadata = fetch_maven_metadata(None).await?; - let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!( + let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( "forge/v{}/manifest.json", daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, ))) @@ -69,7 +69,7 @@ pub async fn retrieve_data( loader_version_raw.to_string() }; - let version = Version::parse(&*loader_version)?; + let version = Version::parse(&loader_version)?; if FORGE_MANIFEST_V1_QUERY.matches(&version) || FORGE_MANIFEST_V2_QUERY_P1.matches(&version) @@ -104,7 +104,7 @@ pub async fn retrieve_data( { let versions = versions_mutex.lock().await; let version = versions.iter().find(|x| - x.id == minecraft_version).map(|x| x.loaders.iter().find(|x| x.id == loader_version_full)).flatten(); + x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full)); if let Some(version) = version { return Ok::, Error>(Some(version.clone())); @@ -112,7 +112,7 @@ pub async fn retrieve_data( } info!("Forge - Installer Start {}", loader_version_full.clone()); - let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; + let bytes = download_file(&format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; let reader = std::io::Cursor::new(bytes); @@ -125,13 +125,13 @@ pub async fn retrieve_data( let mut contents = String::new(); install_profile.read_to_string(&mut contents)?; - Ok::(serde_json::from_str::(&*contents)?) + Ok::(serde_json::from_str::(&contents)?) }).await??; let mut archive_clone = archive.clone(); let file_path = profile.install.file_path.clone(); let forge_universal_bytes = tokio::task::spawn_blocking(move || { - let mut forge_universal_file = archive_clone.by_name(&*file_path)?; + let mut forge_universal_file = archive_clone.by_name(&file_path)?; let mut forge_universal = Vec::new(); forge_universal_file.read_to_end(&mut forge_universal)?; @@ -156,7 +156,7 @@ pub async fn retrieve_data( } let artifact_path = - daedalus::get_path_from_artifact(&*lib.name)?; + daedalus::get_path_from_artifact(&lib.name)?; let artifact = if lib.name == forge_universal_path { forge_universal_bytes.clone() @@ -164,7 +164,7 @@ pub async fn retrieve_data( let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; daedalus::download_file_mirrors( - &*artifact_path, + &artifact_path, &mirrors, None, ) @@ -216,7 +216,7 @@ pub async fn retrieve_data( return Ok(Some(LoaderVersion { id: loader_version_full, - url: format_url(&*version_path), + url: format_url(&version_path), stable: false })); } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { @@ -227,7 +227,7 @@ pub async fn retrieve_data( let mut contents = String::new(); install_profile.read_to_string(&mut contents)?; - Ok::(serde_json::from_str::(&*contents)?) + Ok::(serde_json::from_str::(&contents)?) }).await??; let mut archive_clone = archive.clone(); @@ -237,7 +237,7 @@ pub async fn retrieve_data( let mut contents = String::new(); install_profile.read_to_string(&mut contents)?; - Ok::(serde_json::from_str::(&*contents)?) + Ok::(serde_json::from_str::(&contents)?) }).await??; @@ -255,12 +255,12 @@ pub async fn retrieve_data( let mut local_libs : HashMap = HashMap::new(); for lib in &libs { - if lib.downloads.as_ref().map(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).flatten().unwrap_or(false) { + if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) { let mut archive_clone = archive.clone(); let lib_name_clone = lib.name.clone(); let lib_bytes = tokio::task::spawn_blocking(move || { - let mut lib_file = archive_clone.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*lib_name_clone)?))?; + let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?; let mut lib_bytes = Vec::new(); lib_file.read_to_end(&mut lib_bytes)?; @@ -331,7 +331,7 @@ pub async fn retrieve_data( let now = Instant::now(); let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { let artifact_path = - daedalus::get_path_from_artifact(&*lib.name)?; + daedalus::get_path_from_artifact(&lib.name)?; { let mut visited_assets = visited_assets.lock().await; @@ -339,7 +339,7 @@ pub async fn retrieve_data( if visited_assets.contains(&lib.name) { if let Some(ref mut downloads) = lib.downloads { if let Some(ref mut artifact) = downloads.artifact { - artifact.url = format_url(&*format!("maven/{}", artifact_path)); + artifact.url = format_url(&format!("maven/{}", artifact_path)); } } else if lib.url.is_some() { lib.url = Some(format_url("maven/")); @@ -357,14 +357,14 @@ pub async fn retrieve_data( local_libs.get(&lib.name).cloned() } else { Some(daedalus::download_file( - &*artifact.url, + &artifact.url, Some(&*artifact.sha1), ) .await?) }; if res.is_some() { - artifact.url = format_url(&*format!("maven/{}", artifact_path)); + artifact.url = format_url(&format!("maven/{}", artifact_path)); } res @@ -431,7 +431,7 @@ pub async fn retrieve_data( return Ok(Some(LoaderVersion { id: loader_version_full, - url: format_url(&*version_path), + url: format_url(&version_path), stable: false })); } diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 4a4f6766..70a4fa6d 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -1,9 +1,8 @@ use log::{error, warn}; -use rusoto_core::credential::StaticProvider; -use rusoto_core::{HttpClient, Region, RusotoError}; -use rusoto_s3::{PutObjectError, S3Client}; -use rusoto_s3::{PutObjectRequest, S3}; use std::time::Duration; +use s3::{Bucket, Region}; +use s3::creds::Credentials; +use s3::error::S3Error; mod fabric; mod forge; @@ -21,7 +20,7 @@ pub enum Error { TaskError(#[from] tokio::task::JoinError), #[error("Error while uploading file to S3")] S3Error { - inner: RusotoError, + inner: S3Error, file: String, }, #[error("Error while parsing version as semver: {0}")] @@ -70,11 +69,6 @@ async fn main() { Err(err) => error!("{:?}", err), }; } - - match purge_digitalocean_cache(uploaded_files).await { - Ok(..) => {} - Err(err) => error!("{:?}", err), - }; } } @@ -82,13 +76,13 @@ fn check_env_vars() -> bool { let mut failed = false; fn check_var(var: &str) -> bool { - if dotenv::var(var) + if dotenvy::var(var) .ok() .and_then(|s| s.parse::().ok()) .is_none() { warn!( - "Variable `{}` missing in dotenv or not of type `{}`", + "Variable `{}` missing in dotenvy or not of type `{}`", var, std::any::type_name::() ); @@ -107,36 +101,31 @@ fn check_env_vars() -> bool { failed |= check_var::("S3_REGION"); failed |= check_var::("S3_BUCKET_NAME"); - failed |= check_var::("DO_INTEGRATION"); - - let do_integration = dotenv::var("DO_INTEGRATION") - .ok() - .map(|x| x.parse::().ok()) - .flatten() - .unwrap_or(false); - - if do_integration { - failed |= check_var::("DO_ACCESS_KEY"); - failed |= check_var::("DO_ENDPOINT_ID"); - } - failed } + lazy_static::lazy_static! { - static ref CLIENT : S3Client = S3Client::new_with( - HttpClient::new().unwrap(), - StaticProvider::new( - dotenv::var("S3_ACCESS_TOKEN").unwrap(), - dotenv::var("S3_SECRET").unwrap(), - None, - None, - ), + static ref CLIENT : Bucket = Bucket::new( + &dotenvy::var("S3_BUCKET_NAME").unwrap(), + if &*dotenvy::var("S3_REGION").unwrap() == "r2" { + Region::R2 { + account_id: dotenvy::var("S3_URL").unwrap(), + } + } else { Region::Custom { - name: dotenv::var("S3_REGION").unwrap(), - endpoint: dotenv::var("S3_URL").unwrap(), - }, - ); + region: dotenvy::var("S3_REGION").unwrap(), + endpoint: dotenvy::var("S3_URL").unwrap(), + } + }, + Credentials::new( + Some(&*dotenvy::var("S3_ACCESS_TOKEN").unwrap()), + Some(&*dotenvy::var("S3_SECRET").unwrap()), + None, + None, + None, + ).unwrap(), + ).unwrap(); } pub async fn upload_file_to_bucket( @@ -145,23 +134,24 @@ pub async fn upload_file_to_bucket( content_type: Option, uploaded_files: &tokio::sync::Mutex>, ) -> Result<(), Error> { - let key = format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path); + let key = format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path); for attempt in 1..=4 { - let result = CLIENT - .put_object(PutObjectRequest { - bucket: dotenv::var("S3_BUCKET_NAME").unwrap(), - key: key.clone(), - body: Some(bytes.clone().into()), - acl: Some("public-read".to_string()), - content_type: content_type.clone(), - ..Default::default() - }) - .await - .map_err(|err| Error::S3Error { - inner: err, - file: format!("{}/{}", &*dotenv::var("BASE_FOLDER").unwrap(), path), - }); + let result = if let Some(ref content_type) = content_type { + CLIENT.put_object_with_content_type( + key.clone(), + &bytes, + content_type, + ).await + } else { + CLIENT.put_object( + key.clone(), + &bytes, + ).await + }.map_err(|err| Error::S3Error { + inner: err, + file: format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path), + }); match result { Ok(_) => { @@ -185,45 +175,8 @@ pub async fn upload_file_to_bucket( pub fn format_url(path: &str) -> String { format!( "{}/{}/{}", - &*dotenv::var("BASE_URL").unwrap(), - &*dotenv::var("BASE_FOLDER").unwrap(), + &*dotenvy::var("BASE_URL").unwrap(), + &*dotenvy::var("BASE_FOLDER").unwrap(), path ) } - -#[derive(serde::Serialize)] -struct PurgeCacheRequest { - pub files: Vec, -} - -pub async fn purge_digitalocean_cache(files: Vec) -> Result<(), Error> { - if !dotenv::var("DO_INTEGRATION") - .ok() - .map(|x| x.parse::().ok()) - .flatten() - .unwrap_or(false) - { - return Ok(()); - } - - let client = reqwest::Client::new(); - - client - .delete(&format!( - "https://api.digitalocean.com/v2/cdn/endpoints/{}/cache", - &*dotenv::var("DO_ENDPOINT_ID").unwrap() - )) - .header( - "Authorization", - &*format!("Bearer {}", &*dotenv::var("DO_ACCESS_KEY").unwrap()), - ) - .json(&PurgeCacheRequest { files }) - .send() - .await - .map_err(|err| Error::FetchError { - inner: err, - item: "purging digital ocean cache".to_string(), - })?; - - Ok(()) -} diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 58fae665..56007b9f 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -8,7 +8,7 @@ use tokio::sync::Mutex; pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result { let old_manifest = - daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&*format!( + daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&format!( "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION )))) @@ -43,12 +43,12 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result Date: Thu, 22 Dec 2022 20:11:22 -0700 Subject: [PATCH 39/76] bump rust version in container --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c98c954f..6aed673d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.56 +FROM rust:1.65-slim COPY ./ ./ From fb5f7a336d24c5183feb0b2631d005e30b1c1700 Mon Sep 17 00:00:00 2001 From: Jai A Date: Thu, 22 Dec 2022 20:12:05 -0700 Subject: [PATCH 40/76] bump rust version in container (2) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6aed673d..a63ddaec 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.65-slim +FROM rust:1.65 COPY ./ ./ From b7e2d7fb8e3deb686ee96198e4764cda27e89430 Mon Sep 17 00:00:00 2001 From: Jai A Date: Thu, 22 Dec 2022 20:14:26 -0700 Subject: [PATCH 41/76] Update copyright --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 6d21f198..1bc38c7a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright © 2021 Guavy LLC +Copyright © 2022 Rinth, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: From 79ceb56c60a83a8aaff661e7f1fdd5fa6e763636 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 4 Apr 2023 20:25:17 -0700 Subject: [PATCH 42/76] Fix issues --- Dockerfile | 2 +- daedalus/Cargo.toml | 2 +- daedalus/src/lib.rs | 52 ++- daedalus/src/modded.rs | 19 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/fabric.rs | 472 +++++++++++----------- daedalus_client/src/forge.rs | 659 +++++++++++++++---------------- daedalus_client/src/main.rs | 85 ++-- daedalus_client/src/minecraft.rs | 71 ++-- 9 files changed, 740 insertions(+), 624 deletions(-) diff --git a/Dockerfile b/Dockerfile index a63ddaec..eae1ea34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.65 +FROM rust:1.68.2 COPY ./ ./ diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 3909d840..ada94ed8 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.17" +version = "0.1.18" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index 678bf1c4..51221eed 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -46,22 +46,34 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { let name_items = artifact.split(':').collect::>(); let package = name_items.first().ok_or_else(|| { - Error::ParseError(format!("Unable to find package for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find package for library {}", + &artifact + )) })?; let name = name_items.get(1).ok_or_else(|| { - Error::ParseError(format!("Unable to find name for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find name for library {}", + &artifact + )) })?; if name_items.len() == 3 { let version_ext = name_items .get(2) .ok_or_else(|| { - Error::ParseError(format!("Unable to find version for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find version for library {}", + &artifact + )) })? .split('@') .collect::>(); let version = version_ext.first().ok_or_else(|| { - Error::ParseError(format!("Unable to find version for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find version for library {}", + &artifact + )) })?; let ext = version_ext.get(1); @@ -76,18 +88,27 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { )) } else { let version = name_items.get(2).ok_or_else(|| { - Error::ParseError(format!("Unable to find version for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find version for library {}", + &artifact + )) })?; let data_ext = name_items .get(3) .ok_or_else(|| { - Error::ParseError(format!("Unable to find data for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find data for library {}", + &artifact + )) })? .split('@') .collect::>(); let data = data_ext.first().ok_or_else(|| { - Error::ParseError(format!("Unable to find data for library {}", &artifact)) + Error::ParseError(format!( + "Unable to find data for library {}", + &artifact + )) })?; let ext = data_ext.get(1); @@ -126,10 +147,21 @@ pub async fn download_file_mirrors( } /// Downloads a file with retry and checksum functionality -pub async fn download_file(url: &str, sha1: Option<&str>) -> Result { +pub async fn download_file( + url: &str, + sha1: Option<&str>, +) -> Result { + let mut headers = reqwest::header::HeaderMap::new(); + if let Ok(header) = reqwest::header::HeaderValue::from_str(&format!( + "modrinth/daedalus/{} (support@modrinth.com)", + env!("CARGO_PKG_VERSION") + )) { + headers.insert(reqwest::header::USER_AGENT, header); + } let client = reqwest::Client::builder() .tcp_keepalive(Some(std::time::Duration::from_secs(10))) .timeout(std::time::Duration::from_secs(15)) + .default_headers(headers) .build() .map_err(|err| Error::FetchError { inner: err, @@ -183,7 +215,9 @@ pub async fn download_file(url: &str, sha1: Option<&str>) -> Result Result { - let hash = tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest()).await?; + let hash = + tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest()) + .await?; Ok(hash) } diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 2e62c981..2494ded4 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -15,6 +15,9 @@ pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0; /// The latest version of the format the fabric model structs deserialize to pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0; +/// The dummy replace string library names, inheritsFrom, and version names should be replaced with +pub const DUMMY_REPLACE_STRING: &str = "${modrinth.gameVersion}"; + /// A data variable entry that depends on the side of the installation #[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] @@ -93,6 +96,8 @@ pub fn merge_partial_version( partial: PartialVersionInfo, merge: VersionInfo, ) -> VersionInfo { + let merge_id = merge.id.clone(); + VersionInfo { arguments: if let Some(partial_args) = partial.arguments { if let Some(merge_args) = merge.arguments { @@ -126,12 +131,22 @@ pub fn merge_partial_version( asset_index: merge.asset_index, assets: merge.assets, downloads: merge.downloads, - id: partial.id, + id: merge.id, java_version: merge.java_version, libraries: partial .libraries .into_iter() .chain(merge.libraries) + .map(|x| Library { + downloads: x.downloads, + extract: x.extract, + name: x.name.replace(DUMMY_REPLACE_STRING, &merge_id), + url: x.url, + natives: x.natives, + rules: x.rules, + checksums: x.checksums, + include_in_classpath: x.include_in_classpath, + }) .collect::>(), main_class: if let Some(main_class) = partial.main_class { main_class @@ -163,6 +178,8 @@ pub struct Manifest { pub struct Version { /// The minecraft version ID pub id: String, + /// Whether the release is stable or not + pub stable: bool, /// A map that contains loader versions for the game version pub loaders: Vec, } diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 03558872..389c45a6 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.17" +version = "0.1.18" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 3aef4fdc..b40c31ee 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -1,266 +1,284 @@ -use crate::{format_url, upload_file_to_bucket, Error}; -use daedalus::download_file; +use crate::{download_file, format_url, upload_file_to_bucket, Error}; use daedalus::minecraft::{Library, VersionManifest}; -use daedalus::modded::{LoaderVersion, Manifest, PartialVersionInfo, Version}; -use log::info; +use daedalus::modded::{ + LoaderVersion, Manifest, PartialVersionInfo, Version, DUMMY_REPLACE_STRING, +}; use serde::{Deserialize, Serialize}; use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::sync::{Mutex, RwLock}; +use tokio::sync::{Mutex, RwLock, Semaphore}; pub async fn retrieve_data( minecraft_versions: &VersionManifest, uploaded_files: &mut Vec, + semaphore: Arc, ) -> Result<(), Error> { - let mut list = fetch_fabric_versions(None).await?; - let old_manifest = daedalus::modded::fetch_manifest(&format!( + let mut list = fetch_fabric_versions(None, semaphore.clone()).await?; + let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( "fabric/v{}/manifest.json", daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, - )) + ))) .await .ok(); - let versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { + let mut versions = if let Some(old_manifest) = old_manifest { old_manifest.game_versions } else { Vec::new() - })); + }; + let loaders_mutex = RwLock::new(Vec::new()); + + { + let mut loaders = loaders_mutex.write().await; + + for loader in &list.loader { + loaders.push((Box::new(loader.stable), loader.version.clone())) + } + + list.loader + .retain(|x| loaders.iter().any(|val| val.1 == x.version)) + } + + const DUMMY_GAME_VERSION: &str = "1.19.4-rc2"; + + let loader_version_mutex = Mutex::new(Vec::new()); let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); - if let Some(latest) = list.loader.get(0) { - let loaders_mutex = Arc::new(RwLock::new(Vec::new())); - let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); - - { - let mut loaders = loaders_mutex.write().await; - - // for loader in &list.loader { - // loaders.push((Box::new(loader.stable), loader.version.clone())) - // } - - loaders.push((Box::new(latest.stable), latest.version.clone())); - - if !latest.stable { - if let Some(stable) = list.loader.iter().find(|x| x.stable) { - loaders.push((Box::new(stable.stable), stable.version.clone())); - } - } - - list.loader.retain(|x| loaders.iter().any(|val| val.1 == x.version)) - } - - let mut version_futures = Vec::new(); - - for game_version in list.game.iter_mut() { - let visited_artifacts_mutex = Arc::clone(&visited_artifacts_mutex); - let loaders_mutex = Arc::clone(&loaders_mutex); - let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); - - let versions_mutex = Arc::clone(&versions); - version_futures.push(async move { - let loader_version_mutex = Mutex::new(Vec::new()); - - let versions = - futures::future::try_join_all( - loaders_mutex.read().await.clone().into_iter().map( - |(stable, loader)| async { - { - if versions_mutex.lock().await.iter().any(|x| { - x.id == game_version.version - && x.loaders.iter().any(|x| x.id == loader) - }) { - return Ok(None); - } - } - - let version = - fetch_fabric_version(&game_version.version, &loader).await?; - - Ok::, String, PartialVersionInfo)>, Error>(Some( - (stable, loader, version), - )) - }, - ), - ) - .await? - .into_iter() - .flatten(); - - futures::future::try_join_all(versions.map(|(stable, loader, version)| async { - let libs = futures::future::try_join_all(version.libraries.into_iter().map( - |mut lib| async { - { - let mut visited_assets = visited_artifacts_mutex.lock().await; - - if visited_assets.contains(&lib.name) { - lib.url = Some(format_url("maven/")); - - return Ok(lib); - } else { - visited_assets.push(lib.name.clone()) - } - } - - let artifact_path = daedalus::get_path_from_artifact(&lib.name)?; - - let artifact = daedalus::download_file( - &format!( - "{}{}", - lib.url.unwrap_or_else(|| { - "https://maven.fabricmc.net/".to_string() - }), - artifact_path - ), - None, - ) - .await?; - - lib.url = Some(format_url("maven/")); - - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref(), - ) - .await?; - - Ok::(lib) - }, - )) - .await?; - - let version_path = format!( - "fabric/v{}/versions/{}-{}.json", - daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, - version.inherits_from, - &loader - ); - - let inherits_from = version.inherits_from.clone(); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&PartialVersionInfo { - arguments: version.arguments, - id: version.id, - main_class: version.main_class, - release_time: version.release_time, - time: version.time, - type_: version.type_, - inherits_from: version.inherits_from, - libraries: libs, - minecraft_arguments: version.minecraft_arguments, - processors: None, - data: None, - })?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - ) - .await?; - - { - let mut loader_version_map = loader_version_mutex.lock().await; - async move { - loader_version_map.push(LoaderVersion { - id: format!("{}-{}", inherits_from, loader), - url: format_url(&version_path), - stable: *stable, - }); - } - .await; + let loader_versions = futures::future::try_join_all( + loaders_mutex.read().await.clone().into_iter().map( + |(stable, loader)| async { + { + if versions.iter().any(|x| { + x.id == DUMMY_REPLACE_STRING + && x.loaders.iter().any(|x| x.id == loader) + }) { + return Ok(None); } + } - Ok::<(), Error>(()) - })) + let version = fetch_fabric_version( + DUMMY_GAME_VERSION, + &loader, + semaphore.clone(), + ) .await?; - let mut versions = versions_mutex.lock().await; - versions.push(Version { - id: game_version.version.clone(), - loaders: loader_version_mutex.into_inner(), - }); + //println!("{}", loader); + Ok::, String, PartialVersionInfo)>, Error>( + Some((stable, loader, version)), + ) + }, + ), + ) + .await?; - Ok::<(), Error>(()) - }); - } + let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); + futures::future::try_join_all(loader_versions.into_iter() + .flatten().map( + |(stable, loader, version)| async { + let libs = futures::future::try_join_all( + version.libraries.into_iter().map(|mut lib| async { + { + let mut visited_assets = + visited_artifacts_mutex.lock().await; - let mut versions = version_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions.peek().is_some() { - let now = Instant::now(); + if visited_assets.contains(&lib.name) { + lib.url = Some(format_url("maven/")); - let chunk: Vec<_> = versions.by_ref().take(10).collect(); - futures::future::try_join_all(chunk).await?; + return Ok(lib); + } else { + visited_assets.push(lib.name.clone()) + } + } - tokio::time::sleep(Duration::from_secs(1)).await; + if lib.name.contains(DUMMY_GAME_VERSION) { + lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); + lib.url = Some(format_url("maven/")); + futures::future::try_join_all(list.game.clone().into_iter().map(|game_version| async { + let semaphore = semaphore.clone(); + let uploaded_files_mutex = uploaded_files_mutex.clone(); + let lib_name = lib.name.clone(); + let lib_url = lib.url.clone(); - chunk_index += 1; + async move { + let artifact_path = + daedalus::get_path_from_artifact(&lib_name.replace(DUMMY_REPLACE_STRING, &game_version.version))?; - let elapsed = now.elapsed(); - info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); - } + let artifact = download_file( + &format!( + "{}{}", + lib_url.unwrap_or_else(|| { + "https://maven.fabricmc.net/".to_string() + }), + artifact_path + ), + None, + semaphore.clone(), + ) + .await?; + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + &uploaded_files_mutex, + semaphore.clone(), + ) + .await?; + + Ok::<(), Error>(()) + }.await?; + + Ok::<(), Error>(()) + })).await?; + + return Ok(lib); + } + + let artifact_path = + daedalus::get_path_from_artifact(&lib.name)?; + + let artifact = download_file( + &format!( + "{}{}", + lib.url.unwrap_or_else(|| { + "https://maven.fabricmc.net/".to_string() + }), + artifact_path + ), + None, + semaphore.clone(), + ) + .await?; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + &uploaded_files_mutex, + semaphore.clone(), + ) + .await?; + + Ok::(lib) + }), + ) + .await?; + + let version_path = format!( + "fabric/v{}/versions/{}.json", + daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, + &loader + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&PartialVersionInfo { + arguments: version.arguments, + id: version + .id + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), + main_class: version.main_class, + release_time: version.release_time, + time: version.time, + type_: version.type_, + inherits_from: version + .inherits_from + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), + libraries: libs, + minecraft_arguments: version.minecraft_arguments, + processors: None, + data: None, + })?, + Some("application/json".to_string()), + &uploaded_files_mutex, + semaphore.clone(), + ) + .await?; + + { + let mut loader_version_map = loader_version_mutex.lock().await; + async move { + loader_version_map.push(LoaderVersion { + id: loader.to_string(), + url: format_url(&version_path), + stable: *stable, + }); + } + .await; + } + + Ok::<(), Error>(()) + }, + )) + .await?; + + versions.push(Version { + id: DUMMY_REPLACE_STRING.to_string(), + stable: true, + loaders: loader_version_mutex.into_inner(), + }); + + for version in &list.game { + versions.push(Version { + id: version.version.clone(), + stable: version.stable, + loaders: vec![], + }); } - if let Ok(versions) = Arc::try_unwrap(versions) { - let mut versions = versions.into_inner(); + versions.sort_by(|x, y| { + minecraft_versions + .versions + .iter() + .position(|z| x.id == z.id) + .unwrap_or_default() + .cmp( + &minecraft_versions + .versions + .iter() + .position(|z| y.id == z.id) + .unwrap_or_default(), + ) + }); - versions.sort_by(|x, y| { - minecraft_versions - .versions + for version in &mut versions { + version.loaders.sort_by(|x, y| { + list.loader .iter() - .position(|z| x.id == z.id) + .position(|z| { + x.id.split('-').next().unwrap_or_default() == &*z.version + }) .unwrap_or_default() .cmp( - &minecraft_versions - .versions + &list + .loader .iter() - .position(|z| y.id == z.id) + .position(|z| { + y.id.split('-').next().unwrap_or_default() + == z.version + }) .unwrap_or_default(), ) - }); - - for version in &mut versions { - version.loaders.sort_by(|x, y| { - list.loader - .iter() - .position(|z| { - x.id.split('-') - .next() - .unwrap_or_default() - == &*z.version - }) - .unwrap_or_default() - .cmp( - &list - .loader - .iter() - .position(|z| { - y.id.split('-') - .next() - .unwrap_or_default() - == z.version - }) - .unwrap_or_default(), - ) - }) - } - - upload_file_to_bucket( - format!( - "fabric/v{}/manifest.json", - daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, - ), - serde_json::to_vec(&Manifest { - game_versions: versions, - })?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - ) - .await?; + }) } + upload_file_to_bucket( + format!( + "fabric/v{}/manifest.json", + daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, + ), + serde_json::to_vec(&Manifest { + game_versions: versions, + })?, + Some("application/json".to_string()), + &uploaded_files_mutex, + semaphore, + ) + .await?; + if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { uploaded_files.extend(uploaded_files_mutex.into_inner()); } @@ -273,6 +291,7 @@ const FABRIC_META_URL: &str = "https://meta.fabricmc.net/v2"; async fn fetch_fabric_version( version_number: &str, loader_version: &str, + semaphore: Arc, ) -> Result { Ok(serde_json::from_slice( &download_file( @@ -281,6 +300,7 @@ async fn fetch_fabric_version( FABRIC_META_URL, version_number, loader_version ), None, + semaphore, ) .await?, )?) @@ -319,11 +339,15 @@ struct FabricLoaderVersion { pub stable: bool, } /// Fetches the list of fabric versions -async fn fetch_fabric_versions(url: Option<&str>) -> Result { +async fn fetch_fabric_versions( + url: Option<&str>, + semaphore: Arc, +) -> Result { Ok(serde_json::from_slice( &download_file( url.unwrap_or(&*format!("{}/versions", FABRIC_META_URL)), None, + semaphore, ) .await?, )?) diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 25da023a..cb903f15 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -1,8 +1,14 @@ -use crate::{format_url, upload_file_to_bucket, Error}; +use crate::{ + download_file, download_file_mirrors, format_url, upload_file_to_bucket, + Error, +}; use chrono::{DateTime, Utc}; -use daedalus::download_file; -use daedalus::minecraft::{Argument, ArgumentType, Library, VersionManifest, VersionType}; -use daedalus::modded::{LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry}; +use daedalus::minecraft::{ + Argument, ArgumentType, Library, VersionManifest, VersionType, +}; +use daedalus::modded::{ + LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry, +}; use lazy_static::lazy_static; use log::info; use semver::{Version, VersionReq}; @@ -10,8 +16,8 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::io::Read; use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::sync::Mutex; +use std::time::Instant; +use tokio::sync::{Mutex, Semaphore}; lazy_static! { static ref FORGE_MANIFEST_V1_QUERY: VersionReq = @@ -20,14 +26,16 @@ lazy_static! { VersionReq::parse(">=23.5.2851, <31.2.52").unwrap(); static ref FORGE_MANIFEST_V2_QUERY_P2: VersionReq = VersionReq::parse(">=32.0.1, <37.0.0").unwrap(); - static ref FORGE_MANIFEST_V3_QUERY: VersionReq = VersionReq::parse(">=37.0.0").unwrap(); + static ref FORGE_MANIFEST_V3_QUERY: VersionReq = + VersionReq::parse(">=37.0.0").unwrap(); } pub async fn retrieve_data( minecraft_versions: &VersionManifest, uploaded_files: &mut Vec, + semaphore: Arc, ) -> Result<(), Error> { - let maven_metadata = fetch_maven_metadata(None).await?; + let maven_metadata = fetch_maven_metadata(None, semaphore.clone()).await?; let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( "forge/v{}/manifest.json", daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, @@ -35,11 +43,12 @@ pub async fn retrieve_data( .await .ok(); - let old_versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { - old_manifest.game_versions - } else { - Vec::new() - })); + let old_versions = + Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { + old_manifest.game_versions + } else { + Vec::new() + })); let versions = Arc::new(Mutex::new(Vec::new())); @@ -52,13 +61,14 @@ pub async fn retrieve_data( let mut loaders = Vec::new(); for loader_version_full in loader_versions { - let loader_version = loader_version_full.split('-').into_iter().nth(1); + let loader_version = loader_version_full.split('-').nth(1); if let Some(loader_version_raw) = loader_version { // This is a dirty hack to get around Forge not complying with SemVer, but whatever // Most of this is a hack anyways :( // Works for all forge versions! - let split = loader_version_raw.split('.').collect::>(); + let split = + loader_version_raw.split('.').collect::>(); let loader_version = if split.len() >= 4 { if split[0].parse::().unwrap_or(0) < 6 { format!("{}.{}.{}", split[0], split[1], split[3]) @@ -80,203 +90,209 @@ pub async fn retrieve_data( } } } - version_futures.push(async { - let mut loaders_versions = Vec::new(); - { - let mut loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async { - let versions_mutex = Arc::clone(&old_versions); - let visited_assets = Arc::clone(&visited_assets_mutex); - let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); - let minecraft_version = minecraft_version.clone(); + if !loaders.is_empty() { + version_futures.push(async { + let loaders_versions = Vec::new(); - async move { - /// These forge versions are not worth supporting! - const WHITELIST : [&str; 1] = [ - // Not supported due to `data` field being `[]` even though the type is a map - "1.12.2-14.23.5.2851" - ]; + { + let loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async { + let versions_mutex = Arc::clone(&old_versions); + let visited_assets = Arc::clone(&visited_assets_mutex); + let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); + let semaphore = Arc::clone(&semaphore); + let minecraft_version = minecraft_version.clone(); - if WHITELIST.contains(&&*loader_version_full) { - return Ok(None); - } + async move { + /// These forge versions are not worth supporting! + const WHITELIST : [&str; 1] = [ + // Not supported due to `data` field being `[]` even though the type is a map + "1.12.2-14.23.5.2851", + ]; - { - let versions = versions_mutex.lock().await; - let version = versions.iter().find(|x| - x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full)); - - if let Some(version) = version { - return Ok::, Error>(Some(version.clone())); + if WHITELIST.contains(&&*loader_version_full) { + return Ok(None); } - } - info!("Forge - Installer Start {}", loader_version_full.clone()); - let bytes = download_file(&format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?; + { + let versions = versions_mutex.lock().await; + let version = versions.iter().find(|x| + x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full)); - let reader = std::io::Cursor::new(bytes); + if let Some(version) = version { + return Ok::, Error>(Some(version.clone())); + } + } - if let Ok(archive) = zip::ZipArchive::new(reader) { - if FORGE_MANIFEST_V1_QUERY.matches(&version) { - let mut archive_clone = archive.clone(); - let profile = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("install_profile.json")?; + info!("Forge - Installer Start {}", loader_version_full.clone()); + let bytes = download_file(&format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None, semaphore.clone()).await?; - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; + let reader = std::io::Cursor::new(bytes); - Ok::(serde_json::from_str::(&contents)?) - }).await??; + if let Ok(archive) = zip::ZipArchive::new(reader) { + if FORGE_MANIFEST_V1_QUERY.matches(&version) { + let mut archive_clone = archive.clone(); + let profile = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("install_profile.json")?; - let mut archive_clone = archive.clone(); - let file_path = profile.install.file_path.clone(); - let forge_universal_bytes = tokio::task::spawn_blocking(move || { - let mut forge_universal_file = archive_clone.by_name(&file_path)?; - let mut forge_universal = Vec::new(); - forge_universal_file.read_to_end(&mut forge_universal)?; + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + Ok::(serde_json::from_str::(&contents)?) + }).await??; + + let mut archive_clone = archive.clone(); + let file_path = profile.install.file_path.clone(); + let forge_universal_bytes = tokio::task::spawn_blocking(move || { + let mut forge_universal_file = archive_clone.by_name(&file_path)?; + let mut forge_universal = Vec::new(); + forge_universal_file.read_to_end(&mut forge_universal)?; - Ok::(bytes::Bytes::from(forge_universal)) - }).await??; - let forge_universal_path = profile.install.path.clone(); + Ok::(bytes::Bytes::from(forge_universal)) + }).await??; + let forge_universal_path = profile.install.path.clone(); - let now = Instant::now(); - let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { - if let Some(url) = lib.url { - { - let mut visited_assets = visited_assets.lock().await; + let now = Instant::now(); + let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { + if let Some(url) = lib.url { + { + let mut visited_assets = visited_assets.lock().await; - if visited_assets.contains(&lib.name) { - lib.url = Some(format_url("maven/")); + if visited_assets.contains(&lib.name) { + lib.url = Some(format_url("maven/")); - return Ok::(lib); - } else { - visited_assets.push(lib.name.clone()) + return Ok::(lib); + } else { + visited_assets.push(lib.name.clone()) + } } + + let artifact_path = + daedalus::get_path_from_artifact(&lib.name)?; + + let artifact = if lib.name == forge_universal_path { + forge_universal_bytes.clone() + } else { + let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; + + download_file_mirrors( + &artifact_path, + &mirrors, + None, + semaphore.clone(), + ) + .await? + }; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref(), + semaphore.clone(), + ).await?; } - let artifact_path = - daedalus::get_path_from_artifact(&lib.name)?; + Ok::(lib) + })).await?; - let artifact = if lib.name == forge_universal_path { - forge_universal_bytes.clone() - } else { - let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; + let elapsed = now.elapsed(); + info!("Elapsed lib DL: {:.2?}", elapsed); - daedalus::download_file_mirrors( - &artifact_path, - &mirrors, - None, - ) - .await? - }; + let new_profile = PartialVersionInfo { + id: profile.version_info.id, + inherits_from: profile.install.minecraft, + release_time: profile.version_info.release_time, + time: profile.version_info.time, + main_class: profile.version_info.main_class, + minecraft_arguments: profile.version_info.minecraft_arguments.clone(), + arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), + libraries: libs, + type_: profile.version_info.type_, + data: None, + processors: None + }; - lib.url = Some(format_url("maven/")); + let version_path = format!( + "forge/v{}/versions/{}.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + new_profile.id + ); - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref(), - ).await?; + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + uploaded_files_mutex.as_ref(), + semaphore.clone(), + ).await?; + + return Ok(Some(LoaderVersion { + id: loader_version_full, + url: format_url(&version_path), + stable: false + })); + } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { + let mut archive_clone = archive.clone(); + let mut profile = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("install_profile.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + Ok::(serde_json::from_str::(&contents)?) + }).await??; + + let mut archive_clone = archive.clone(); + let version_info = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("version.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + Ok::(serde_json::from_str::(&contents)?) + }).await??; + + + let mut libs : Vec = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library { + downloads: x.downloads, + extract: x.extract, + name: x.name, + url: x.url, + natives: x.natives, + rules: x.rules, + checksums: x.checksums, + include_in_classpath: false + })).collect(); + + let mut local_libs : HashMap = HashMap::new(); + + for lib in &libs { + if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) { + let mut archive_clone = archive.clone(); + let lib_name_clone = lib.name.clone(); + + let lib_bytes = tokio::task::spawn_blocking(move || { + let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?; + let mut lib_bytes = Vec::new(); + lib_file.read_to_end(&mut lib_bytes)?; + + Ok::(bytes::Bytes::from(lib_bytes)) + }).await??; + + local_libs.insert(lib.name.clone(), lib_bytes); + } } - Ok::(lib) - })).await?; + let path = profile.path.clone(); + let version = profile.version.clone(); - let elapsed = now.elapsed(); - info!("Elapsed lib DL: {:.2?}", elapsed); - - let new_profile = PartialVersionInfo { - id: profile.version_info.id, - inherits_from: profile.install.minecraft, - release_time: profile.version_info.release_time, - time: profile.version_info.time, - main_class: profile.version_info.main_class, - minecraft_arguments: profile.version_info.minecraft_arguments.clone(), - arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), - libraries: libs, - type_: profile.version_info.type_, - data: None, - processors: None - }; - - let version_path = format!( - "forge/v{}/versions/{}.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - new_profile.id - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&new_profile)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref() - ).await?; - - return Ok(Some(LoaderVersion { - id: loader_version_full, - url: format_url(&version_path), - stable: false - })); - } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { - let mut archive_clone = archive.clone(); - let mut profile = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("install_profile.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&contents)?) - }).await??; - - let mut archive_clone = archive.clone(); - let version_info = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("version.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&contents)?) - }).await??; - - - let mut libs : Vec = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library { - downloads: x.downloads, - extract: x.extract, - name: x.name, - url: x.url, - natives: x.natives, - rules: x.rules, - checksums: x.checksums, - include_in_classpath: false - })).collect(); - - let mut local_libs : HashMap = HashMap::new(); - - for lib in &libs { - if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) { - let mut archive_clone = archive.clone(); - let lib_name_clone = lib.name.clone(); - - let lib_bytes = tokio::task::spawn_blocking(move || { - let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?; - let mut lib_bytes = Vec::new(); - lib_file.read_to_end(&mut lib_bytes)?; - - Ok::(bytes::Bytes::from(lib_bytes)) - }).await??; - - local_libs.insert(lib.name.clone(), lib_bytes); - } - } - - let path = profile.path.clone(); - let version = profile.version.clone(); - - for entry in profile.data.values_mut() { - if entry.client.starts_with('/') || entry.server.starts_with('/') { - macro_rules! read_data { + for entry in profile.data.values_mut() { + if entry.client.starts_with('/') || entry.server.starts_with('/') { + macro_rules! read_data { ($value:expr) => { let mut archive_clone = archive.clone(); let value_clone = $value.clone(); @@ -315,179 +331,149 @@ pub async fn retrieve_data( } } - if entry.client.starts_with('/') { - read_data!(entry.client); - } - - // Do we really need to support server installs? Keeping this here - // just in case - // - // if entry.server.starts_with('/') { - // read_data!(entry.server); - // } - } - } - - let now = Instant::now(); - let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { - let artifact_path = - daedalus::get_path_from_artifact(&lib.name)?; - - { - let mut visited_assets = visited_assets.lock().await; - - if visited_assets.contains(&lib.name) { - if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - artifact.url = format_url(&format!("maven/{}", artifact_path)); - } - } else if lib.url.is_some() { - lib.url = Some(format_url("maven/")); + if entry.client.starts_with('/') { + read_data!(entry.client); } - return Ok::(lib); - } else { - visited_assets.push(lib.name.clone()) + if entry.server.starts_with('/') { + read_data!(entry.server); + } } } - let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - let res = if artifact.url.is_empty() { + let now = Instant::now(); + let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { + let artifact_path = + daedalus::get_path_from_artifact(&lib.name)?; + + { + let mut visited_assets = visited_assets.lock().await; + + if visited_assets.contains(&lib.name) { + if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { + artifact.url = format_url(&format!("maven/{}", artifact_path)); + } + } else if lib.url.is_some() { + lib.url = Some(format_url("maven/")); + } + + return Ok::(lib); + } else { + visited_assets.push(lib.name.clone()) + } + } + + let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { + let res = if artifact.url.is_empty() { + local_libs.get(&lib.name).cloned() + } else { + Some(download_file( + &artifact.url, + Some(&*artifact.sha1), + semaphore.clone(), + ) + .await?) + }; + + if res.is_some() { + artifact.url = format_url(&format!("maven/{}", artifact_path)); + } + + res + } else { None } + } else if let Some(ref mut url) = lib.url { + let res = if url.is_empty() { local_libs.get(&lib.name).cloned() } else { - Some(daedalus::download_file( - &artifact.url, - Some(&*artifact.sha1), + Some(download_file( + url, + None, + semaphore.clone(), ) .await?) }; if res.is_some() { - artifact.url = format_url(&format!("maven/{}", artifact_path)); + lib.url = Some(format_url("maven/")); } res - } else { None } - } else if let Some(ref mut url) = lib.url { - let res = if url.is_empty() { - local_libs.get(&lib.name).cloned() - } else { - Some(daedalus::download_file( - url, - None, - ) - .await?) - }; + } else { None }; - if res.is_some() { - lib.url = Some(format_url("maven/")); + if let Some(bytes) = artifact_bytes { + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + bytes.to_vec(), + Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref(), + semaphore.clone(), + ).await?; } - res - } else { None }; + Ok::(lib) + })).await?; - if let Some(bytes) = artifact_bytes { - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - bytes.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref() - ).await?; - } + let elapsed = now.elapsed(); + info!("Elapsed lib DL: {:.2?}", elapsed); - Ok::(lib) - })).await?; + let new_profile = PartialVersionInfo { + id: version_info.id, + inherits_from: version_info.inherits_from, + release_time: version_info.release_time, + time: version_info.time, + main_class: version_info.main_class, + minecraft_arguments: version_info.minecraft_arguments, + arguments: version_info.arguments, + libraries: libs, + type_: version_info.type_, + data: Some(profile.data), + processors: Some(profile.processors), + }; - let elapsed = now.elapsed(); - info!("Elapsed lib DL: {:.2?}", elapsed); + let version_path = format!( + "forge/v{}/versions/{}.json", + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + new_profile.id + ); - let new_profile = PartialVersionInfo { - id: version_info.id, - inherits_from: version_info.inherits_from, - release_time: version_info.release_time, - time: version_info.time, - main_class: version_info.main_class, - minecraft_arguments: version_info.minecraft_arguments, - arguments: version_info.arguments, - libraries: libs, - type_: version_info.type_, - data: Some(profile.data), - processors: Some(profile.processors), - }; + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + uploaded_files_mutex.as_ref(), + semaphore.clone(), + ).await?; - let version_path = format!( - "forge/v{}/versions/{}.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - new_profile.id - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&new_profile)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref() - ).await?; - - return Ok(Some(LoaderVersion { - id: loader_version_full, - url: format_url(&version_path), - stable: false - })); + return Ok(Some(LoaderVersion { + id: loader_version_full, + url: format_url(&version_path), + stable: false + })); + } } - } - Ok(None) - }.await - }).into_iter().peekable()/*.into_iter().flatten().collect()*/; + Ok(None) + }.await + }); - let mut chunk_index = 0; - while loaders_futures.peek().is_some() { - info!("Loader Chunk {} Start", chunk_index); - let now = Instant::now(); - - let chunk: Vec<_> = loaders_futures.by_ref().take(10).collect(); - - let res = futures::future::try_join_all(chunk).await?; - loaders_versions.extend(res.into_iter().flatten()); - - tokio::time::sleep(Duration::from_secs(1)).await; - - chunk_index += 1; - - let elapsed = now.elapsed(); - info!("Loader Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + futures::future::try_join_all(loaders_futures).await?; } - } - versions.lock().await.push(daedalus::modded::Version { - id: minecraft_version, - loaders: loaders_versions + versions.lock().await.push(daedalus::modded::Version { + id: minecraft_version, + stable: true, + loaders: loaders_versions + }); + + Ok::<(), Error>(()) }); - - Ok::<(), Error>(()) - }); - } - - { - let mut versions_peek = version_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions_peek.peek().is_some() { - info!("Chunk {} Start", chunk_index); - let now = Instant::now(); - - let chunk: Vec<_> = versions_peek.by_ref().take(1).collect(); - futures::future::try_join_all(chunk).await?; - - tokio::time::sleep(Duration::from_secs(1)).await; - - chunk_index += 1; - - let elapsed = now.elapsed(); - info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); } } + futures::future::try_join_all(version_futures).await?; + if let Ok(versions) = Arc::try_unwrap(versions) { let mut versions = versions.into_inner(); @@ -495,13 +481,17 @@ pub async fn retrieve_data( minecraft_versions .versions .iter() - .position(|z| x.id == z.id) + .position(|z| { + x.id.replace("1.7.10_pre4", "1.7.10-pre4") == z.id + }) .unwrap_or_default() .cmp( &minecraft_versions .versions .iter() - .position(|z| y.id == z.id) + .position(|z| { + y.id.replace("1.7.10_pre4", "1.7.10-pre4") == z.id + }) .unwrap_or_default(), ) }); @@ -524,6 +514,8 @@ pub async fn retrieve_data( } } + println!("{:?}", versions); + upload_file_to_bucket( format!( "forge/v{}/manifest.json", @@ -534,6 +526,7 @@ pub async fn retrieve_data( })?, Some("application/json".to_string()), uploaded_files_mutex.as_ref(), + semaphore, ) .await?; } @@ -554,9 +547,15 @@ const DEFAULT_MAVEN_METADATA_URL: &str = /// the specified Minecraft version pub async fn fetch_maven_metadata( url: Option<&str>, + semaphore: Arc, ) -> Result>, Error> { Ok(serde_json::from_slice( - &download_file(url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), None).await?, + &download_file( + url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), + None, + semaphore, + ) + .await?, )?) } diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 70a4fa6d..cefa4e17 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -1,8 +1,10 @@ use log::{error, warn}; -use std::time::Duration; -use s3::{Bucket, Region}; use s3::creds::Credentials; use s3::error::S3Error; +use s3::{Bucket, Region}; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::Semaphore; mod fabric; mod forge; @@ -19,10 +21,7 @@ pub enum Error { #[error("Error while managing asynchronous tasks")] TaskError(#[from] tokio::task::JoinError), #[error("Error while uploading file to S3")] - S3Error { - inner: S3Error, - file: String, - }, + S3Error { inner: S3Error, file: String }, #[error("Error while parsing version as semver: {0}")] SemVerError(#[from] semver::Error), #[error("Error while reading zip file: {0}")] @@ -31,6 +30,8 @@ pub enum Error { IoError(#[from] std::io::Error), #[error("Error while obtaining strong reference to Arc")] ArcError, + #[error("Error acquiring semaphore: {0}")] + AcquireError(#[from] tokio::sync::AcquireError), } #[tokio::main] @@ -43,14 +44,20 @@ async fn main() { return; } - let mut timer = tokio::time::interval(Duration::from_secs(10 * 60)); + let mut timer = tokio::time::interval(Duration::from_secs(30 * 60)); + let semaphore = Arc::new(Semaphore::new(50)); loop { timer.tick().await; let mut uploaded_files = Vec::new(); - let versions = match minecraft::retrieve_data(&mut uploaded_files).await { + let versions = match minecraft::retrieve_data( + &mut uploaded_files, + semaphore.clone(), + ) + .await + { Ok(res) => Some(res), Err(err) => { error!("{:?}", err); @@ -60,15 +67,22 @@ async fn main() { }; if let Some(manifest) = versions { - match fabric::retrieve_data(&manifest, &mut uploaded_files).await { + match fabric::retrieve_data( + &manifest, + &mut uploaded_files, + semaphore.clone(), + ) + .await + { Ok(..) => {} Err(err) => error!("{:?}", err), }; - match forge::retrieve_data(&manifest, &mut uploaded_files).await { + match forge::retrieve_data(&manifest, &mut uploaded_files, semaphore.clone()).await { Ok(..) => {} Err(err) => error!("{:?}", err), }; } + break; } } @@ -104,7 +118,6 @@ fn check_env_vars() -> bool { failed } - lazy_static::lazy_static! { static ref CLIENT : Bucket = Bucket::new( &dotenvy::var("S3_BUCKET_NAME").unwrap(), @@ -133,24 +146,26 @@ pub async fn upload_file_to_bucket( bytes: Vec, content_type: Option, uploaded_files: &tokio::sync::Mutex>, + semaphore: Arc, ) -> Result<(), Error> { + let _permit = semaphore.acquire().await?; let key = format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path); for attempt in 1..=4 { let result = if let Some(ref content_type) = content_type { - CLIENT.put_object_with_content_type( - key.clone(), - &bytes, - content_type, - ).await + CLIENT + .put_object_with_content_type(key.clone(), &bytes, content_type) + .await } else { - CLIENT.put_object( - key.clone(), - &bytes, - ).await - }.map_err(|err| Error::S3Error { + CLIENT.put_object(key.clone(), &bytes).await + } + .map_err(|err| Error::S3Error { inner: err, - file: format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path), + file: format!( + "{}/{}", + &*dotenvy::var("BASE_FOLDER").unwrap(), + path + ), }); match result { @@ -180,3 +195,29 @@ pub fn format_url(path: &str) -> String { path ) } + +pub async fn download_file( + url: &str, + sha1: Option<&str>, + semaphore: Arc, +) -> Result { + let _permit = semaphore.acquire().await?; + println!("{} url start", url); + + let val = daedalus::download_file(url, sha1).await?; + println!("{} url end", url); + Ok(val) +} + +pub async fn download_file_mirrors( + base: &str, + mirrors: &[&str], + sha1: Option<&str>, + semaphore: Arc, +) -> Result { + let _permit = semaphore.acquire().await?; + + let val = daedalus::download_file_mirrors(base, mirrors, sha1).await?; + + Ok(val) +} diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 56007b9f..256eef71 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,21 +1,26 @@ +use crate::download_file; use crate::{format_url, upload_file_to_bucket, Error}; -use daedalus::download_file; use daedalus::minecraft::VersionManifest; use log::info; use std::sync::Arc; -use std::time::{Duration, Instant}; -use tokio::sync::Mutex; +use std::time::Instant; +use tokio::sync::{Mutex, Semaphore}; -pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result { - let old_manifest = - daedalus::minecraft::fetch_version_manifest(Some(&*crate::format_url(&format!( +pub async fn retrieve_data( + uploaded_files: &mut Vec, + semaphore: Arc, +) -> Result { + let old_manifest = daedalus::minecraft::fetch_version_manifest(Some( + &*format_url(&format!( "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION - )))) - .await - .ok(); + )), + )) + .await + .ok(); - let mut manifest = daedalus::minecraft::fetch_version_manifest(None).await?; + let mut manifest = + daedalus::minecraft::fetch_version_manifest(None).await?; let cloned_manifest = Arc::new(Mutex::new(manifest.clone())); let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); @@ -42,13 +47,16 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result) -> Result) -> Result) -> Result) -> Result) -> Result = versions.by_ref().take(100).collect(); - futures::future::try_join_all(chunk).await?; - - tokio::time::sleep(Duration::from_secs(1)).await; - - chunk_index += 1; - - let elapsed = now.elapsed(); - info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); - } - } + futures::future::try_join_all(version_futures).await?; upload_file_to_bucket( format!( @@ -157,6 +157,7 @@ pub async fn retrieve_data(uploaded_files: &mut Vec) -> Result Date: Tue, 4 Apr 2023 20:26:10 -0700 Subject: [PATCH 43/76] fix debug stuff --- daedalus_client/src/fabric.rs | 1 - daedalus_client/src/forge.rs | 2 -- daedalus_client/src/main.rs | 12 +++++++----- 3 files changed, 7 insertions(+), 8 deletions(-) diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index b40c31ee..e3625dab 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -63,7 +63,6 @@ pub async fn retrieve_data( ) .await?; - //println!("{}", loader); Ok::, String, PartialVersionInfo)>, Error>( Some((stable, loader, version)), ) diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index cb903f15..0de8cf1e 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -514,8 +514,6 @@ pub async fn retrieve_data( } } - println!("{:?}", versions); - upload_file_to_bucket( format!( "forge/v{}/manifest.json", diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index cefa4e17..de97bc07 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -77,12 +77,17 @@ async fn main() { Ok(..) => {} Err(err) => error!("{:?}", err), }; - match forge::retrieve_data(&manifest, &mut uploaded_files, semaphore.clone()).await { + match forge::retrieve_data( + &manifest, + &mut uploaded_files, + semaphore.clone(), + ) + .await + { Ok(..) => {} Err(err) => error!("{:?}", err), }; } - break; } } @@ -202,10 +207,7 @@ pub async fn download_file( semaphore: Arc, ) -> Result { let _permit = semaphore.acquire().await?; - println!("{} url start", url); - let val = daedalus::download_file(url, sha1).await?; - println!("{} url end", url); Ok(val) } From 9754f2d1c51dee70c3833192926809fbd2608619 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 4 Apr 2023 21:17:19 -0700 Subject: [PATCH 44/76] Add limiter for forge downloading --- .env | 1 - daedalus_client/src/forge.rs | 36 +++++++++++++++++++++++++++++--- daedalus_client/src/main.rs | 22 +++++++++---------- daedalus_client/src/minecraft.rs | 17 ++++++++++++++- 4 files changed, 59 insertions(+), 17 deletions(-) diff --git a/.env b/.env index 34bf98cd..c7298bc7 100644 --- a/.env +++ b/.env @@ -1,7 +1,6 @@ RUST_LOG=info,error BASE_URL=https://modrinth-cdn-staging.nyc3.digitaloceanspaces.com -BASE_FOLDER=gamedata S3_ACCESS_TOKEN=none S3_SECRET=none diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 0de8cf1e..7cbe8479 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -16,7 +16,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::io::Read; use std::sync::Arc; -use std::time::Instant; +use std::time::{Duration, Instant}; use tokio::sync::{Mutex, Semaphore}; lazy_static! { @@ -458,7 +458,22 @@ pub async fn retrieve_data( }.await }); - futures::future::try_join_all(loaders_futures).await?; + { + let mut versions = loaders_futures.into_iter().peekable(); + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); + + let chunk: Vec<_> = versions.by_ref().take(1).collect(); + futures::future::try_join_all(chunk).await?; + + chunk_index += 1; + + let elapsed = now.elapsed(); + info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } + } + //futures::future::try_join_all(loaders_futures).await?; } versions.lock().await.push(daedalus::modded::Version { @@ -472,7 +487,22 @@ pub async fn retrieve_data( } } - futures::future::try_join_all(version_futures).await?; + { + let mut versions = version_futures.into_iter().peekable(); + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); + + let chunk: Vec<_> = versions.by_ref().take(10).collect(); + futures::future::try_join_all(chunk).await?; + + chunk_index += 1; + + let elapsed = now.elapsed(); + info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } + } + //futures::future::try_join_all(version_futures).await?; if let Ok(versions) = Arc::try_unwrap(versions) { let mut versions = versions.into_inner(); diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index de97bc07..56b524bf 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -1,4 +1,4 @@ -use log::{error, warn}; +use log::{error, info, warn}; use s3::creds::Credentials; use s3::error::S3Error; use s3::{Bucket, Region}; @@ -112,7 +112,6 @@ fn check_env_vars() -> bool { } failed |= check_var::("BASE_URL"); - failed |= check_var::("BASE_FOLDER"); failed |= check_var::("S3_ACCESS_TOKEN"); failed |= check_var::("S3_SECRET"); @@ -154,7 +153,8 @@ pub async fn upload_file_to_bucket( semaphore: Arc, ) -> Result<(), Error> { let _permit = semaphore.acquire().await?; - let key = format!("{}/{}", &*dotenvy::var("BASE_FOLDER").unwrap(), path); + info!("{} started uploading", path); + let key = path.clone(); for attempt in 1..=4 { let result = if let Some(ref content_type) = content_type { @@ -166,16 +166,13 @@ pub async fn upload_file_to_bucket( } .map_err(|err| Error::S3Error { inner: err, - file: format!( - "{}/{}", - &*dotenvy::var("BASE_FOLDER").unwrap(), - path - ), + file: path.clone(), }); match result { Ok(_) => { { + info!("{} done uploading", path); let mut uploaded_files = uploaded_files.lock().await; uploaded_files.push(key); } @@ -188,15 +185,13 @@ pub async fn upload_file_to_bucket( } } } - unreachable!() } pub fn format_url(path: &str) -> String { format!( - "{}/{}/{}", + "{}/{}", &*dotenvy::var("BASE_URL").unwrap(), - &*dotenvy::var("BASE_FOLDER").unwrap(), path ) } @@ -207,7 +202,9 @@ pub async fn download_file( semaphore: Arc, ) -> Result { let _permit = semaphore.acquire().await?; + info!("{} started downloading", url); let val = daedalus::download_file(url, sha1).await?; + info!("{} finished downloading", url); Ok(val) } @@ -218,8 +215,9 @@ pub async fn download_file_mirrors( semaphore: Arc, ) -> Result { let _permit = semaphore.acquire().await?; - + info!("{} started downloading", base); let val = daedalus::download_file_mirrors(base, mirrors, sha1).await?; + info!("{} finished downloading", base); Ok(val) } diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 256eef71..024ce3d0 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -147,7 +147,22 @@ pub async fn retrieve_data( }) } - futures::future::try_join_all(version_futures).await?; + { + let mut versions = version_futures.into_iter().peekable(); + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); + + let chunk: Vec<_> = versions.by_ref().take(100).collect(); + futures::future::try_join_all(chunk).await?; + + chunk_index += 1; + + let elapsed = now.elapsed(); + info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + } + } + //futures::future::try_join_all(version_futures).await?; upload_file_to_bucket( format!( From aa84f21fde4954775f2f7b442a5a45eb442315e7 Mon Sep 17 00:00:00 2001 From: Jai A Date: Wed, 5 Apr 2023 12:02:00 -0700 Subject: [PATCH 45/76] Fix forge syncing not working --- .env | 2 +- Dockerfile | 22 ++++++++++++++++++---- daedalus/Cargo.toml | 2 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/fabric.rs | 3 ++- daedalus_client/src/forge.rs | 11 +++++++---- daedalus_client/src/main.rs | 2 +- 7 files changed, 31 insertions(+), 13 deletions(-) diff --git a/.env b/.env index c7298bc7..bbcfb5cb 100644 --- a/.env +++ b/.env @@ -1,4 +1,4 @@ -RUST_LOG=info,error +RUST_LOG=info BASE_URL=https://modrinth-cdn-staging.nyc3.digitaloceanspaces.com diff --git a/Dockerfile b/Dockerfile index eae1ea34..a2a341da 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,21 @@ -FROM rust:1.68.2 - -COPY ./ ./ +FROM rust:1.68.2 as build +ENV PKG_CONFIG_ALLOW_CROSS=1 +WORKDIR /usr/src/daedalus +COPY . . RUN cargo build --release -CMD ["./target/release/daedalus_client"] \ No newline at end of file + +FROM debian:bullseye-slim + +RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +RUN update-ca-certificates + +COPY --from=build /usr/src/daedalus/target/release/daedalus_client /daedalus/daedalus_client +WORKDIR /daedalus_client + +CMD /daedalus/daedalus_client diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index ada94ed8..09c54758 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.18" +version = "0.1.19" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 389c45a6..8b531712 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.18" +version = "0.1.19" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index e3625dab..6aa8f91f 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -82,6 +82,7 @@ pub async fn retrieve_data( visited_artifacts_mutex.lock().await; if visited_assets.contains(&lib.name) { + lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); lib.url = Some(format_url("maven/")); return Ok(lib); @@ -92,7 +93,6 @@ pub async fn retrieve_data( if lib.name.contains(DUMMY_GAME_VERSION) { lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); - lib.url = Some(format_url("maven/")); futures::future::try_join_all(list.game.clone().into_iter().map(|game_version| async { let semaphore = semaphore.clone(); let uploaded_files_mutex = uploaded_files_mutex.clone(); @@ -130,6 +130,7 @@ pub async fn retrieve_data( Ok::<(), Error>(()) })).await?; + lib.url = Some(format_url("maven/")); return Ok(lib); } diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 7cbe8479..5335f20d 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -93,7 +93,7 @@ pub async fn retrieve_data( if !loaders.is_empty() { version_futures.push(async { - let loaders_versions = Vec::new(); + let mut loaders_versions = Vec::new(); { let loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async { @@ -464,8 +464,9 @@ pub async fn retrieve_data( while versions.peek().is_some() { let now = Instant::now(); - let chunk: Vec<_> = versions.by_ref().take(1).collect(); - futures::future::try_join_all(chunk).await?; + let chunk: Vec<_> = versions.by_ref().take(10).collect(); + let res = futures::future::try_join_all(chunk).await?; + loaders_versions.extend(res.into_iter().flatten()); chunk_index += 1; @@ -493,7 +494,7 @@ pub async fn retrieve_data( while versions.peek().is_some() { let now = Instant::now(); - let chunk: Vec<_> = versions.by_ref().take(10).collect(); + let chunk: Vec<_> = versions.by_ref().take(1).collect(); futures::future::try_join_all(chunk).await?; chunk_index += 1; @@ -544,6 +545,8 @@ pub async fn retrieve_data( } } + println!("{}", serde_json::to_string(&versions).unwrap()); + upload_file_to_bucket( format!( "forge/v{}/manifest.json", diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 56b524bf..000f003a 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -44,7 +44,7 @@ async fn main() { return; } - let mut timer = tokio::time::interval(Duration::from_secs(30 * 60)); + let mut timer = tokio::time::interval(Duration::from_secs(60 * 60)); let semaphore = Arc::new(Semaphore::new(50)); loop { From bf5a25a96f0da836b20e16f1998cc9eb7bebc3f9 Mon Sep 17 00:00:00 2001 From: Jai A Date: Wed, 5 Apr 2023 12:02:47 -0700 Subject: [PATCH 46/76] fmt + clippy --- daedalus_client/src/forge.rs | 4 +--- daedalus_client/src/main.rs | 6 +----- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 5335f20d..7415cc65 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -16,7 +16,7 @@ use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::io::Read; use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::Instant; use tokio::sync::{Mutex, Semaphore}; lazy_static! { @@ -545,8 +545,6 @@ pub async fn retrieve_data( } } - println!("{}", serde_json::to_string(&versions).unwrap()); - upload_file_to_bucket( format!( "forge/v{}/manifest.json", diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 000f003a..92fee4cd 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -189,11 +189,7 @@ pub async fn upload_file_to_bucket( } pub fn format_url(path: &str) -> String { - format!( - "{}/{}", - &*dotenvy::var("BASE_URL").unwrap(), - path - ) + format!("{}/{}", &*dotenvy::var("BASE_URL").unwrap(), path) } pub async fn download_file( From 0c2e9137a2cb45ac9cb422d822e8c9eb8d676486 Mon Sep 17 00:00:00 2001 From: Jai A Date: Wed, 5 Apr 2023 12:39:16 -0700 Subject: [PATCH 47/76] Update id merging --- daedalus/src/modded.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 2494ded4..2519719c 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -131,7 +131,7 @@ pub fn merge_partial_version( asset_index: merge.asset_index, assets: merge.assets, downloads: merge.downloads, - id: merge.id, + id: partial.id.replace(DUMMY_REPLACE_STRING, &merge_id), java_version: merge.java_version, libraries: partial .libraries From 89e56ae279392f41abfc8e35658099449071a3e6 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 25 Apr 2023 19:36:21 -0700 Subject: [PATCH 48/76] Support for ARM + Quilt --- LICENSE | 2 +- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 82 ++++++- daedalus/src/modded.rs | 2 + daedalus_client/Cargo.toml | 2 +- daedalus_client/src/fabric.rs | 29 ++- daedalus_client/src/forge.rs | 17 +- daedalus_client/src/main.rs | 13 +- daedalus_client/src/minecraft.rs | 61 +++++- daedalus_client/src/quilt.rs | 362 +++++++++++++++++++++++++++++++ 10 files changed, 542 insertions(+), 30 deletions(-) create mode 100644 daedalus_client/src/quilt.rs diff --git a/LICENSE b/LICENSE index 1bc38c7a..75e4f779 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright © 2022 Rinth, Inc. +Copyright © 2023 Rinth, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 09c54758..94cbde71 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.19" +version = "0.1.20" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 247274a5..49c87fe4 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -149,11 +149,12 @@ pub struct Download { } #[cfg_attr(feature = "bincode", derive(Encode, Decode))] -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// Download information of a library pub struct LibraryDownload { + #[serde(skip_serializing_if = "Option::is_none")] /// The path that the library should be saved to - pub path: String, + pub path: Option, /// The SHA1 hash of the library pub sha1: String, /// The size of the library @@ -163,7 +164,7 @@ pub struct LibraryDownload { } #[cfg_attr(feature = "bincode", derive(Encode, Decode))] -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// A list of files that should be downloaded for libraries pub struct LibraryDownloads { #[serde(skip_serializing_if = "Option::is_none")] @@ -188,17 +189,25 @@ pub enum RuleAction { #[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] -#[serde(rename_all = "snake_case")] +#[serde(rename_all = "kebab-case")] /// An enum representing the different types of operating systems pub enum Os { - /// MacOS + /// MacOS (x86) Osx, - /// Windows + /// M1-Based Macs + OsxArm64, + /// Windows (x86) Windows, - /// Linux and its derivatives + /// Windows ARM + WindowsArm64, + /// Linux (x86) and its derivatives Linux, + /// Linux ARM 64 + LinuxArm64, + /// Linux ARM 32 + LinuxArm32, /// The OS is unknown - Unknown, + Unknown } #[cfg_attr(feature = "bincode", derive(Encode, Decode))] @@ -243,7 +252,7 @@ pub struct Rule { } #[cfg_attr(feature = "bincode", derive(Encode, Decode))] -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// Information delegating the extraction of the library pub struct LibraryExtract { #[serde(skip_serializing_if = "Option::is_none")] @@ -263,7 +272,7 @@ pub struct JavaVersion { } #[cfg_attr(feature = "bincode", derive(Encode, Decode))] -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Clone)] /// A library which the game relies on to run pub struct Library { #[serde(skip_serializing_if = "Option::is_none")] @@ -291,6 +300,59 @@ pub struct Library { pub include_in_classpath: bool, } +#[derive(Deserialize, Debug, Clone)] +/// A partial library which should be merged with a full library +pub struct PartialLibrary { + /// The files the library has + pub downloads: Option, + /// Rules of the extraction of the file + pub extract: Option, + /// The maven name of the library. The format is `groupId:artifactId:version` + pub name: Option, + /// The URL to the repository where the library can be downloaded + pub url: Option, + /// Native files that the library relies on + pub natives: Option>, + /// Rules deciding whether the library should be downloaded or not + pub rules: Option>, + /// SHA1 Checksums for validating the library's integrity. Only present for forge libraries + pub checksums: Option>, + /// Whether the library should be included in the classpath at the game's launch + pub include_in_classpath: Option, +} + +pub fn merge_partial_library( + partial: PartialLibrary, + mut merge: Library, +) -> Library { + if let Some(downloads) = partial.downloads { + merge.downloads = Some(downloads) + } + if let Some(extract) = partial.extract { + merge.extract = Some(extract) + } + if let Some(name) = partial.name { + merge.name = name + } + if let Some(url) = partial.url { + merge.url = Some(url) + } + if let Some(natives) = partial.natives { + merge.natives = Some(natives) + } + if let Some(rules) = partial.rules { + merge.rules = Some(rules) + } + if let Some(checksums) = partial.checksums { + merge.checksums = Some(checksums) + } + if let Some(include_in_classpath) = partial.include_in_classpath { + merge.include_in_classpath = include_in_classpath + } + + merge +} + fn default_include_in_classpath() -> bool { true } diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 2519719c..8f91798f 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -14,6 +14,8 @@ use bincode::{Decode, Encode}; pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0; /// The latest version of the format the fabric model structs deserialize to pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0; +/// The latest version of the format the quilt model structs deserialize to +pub const CURRENT_QUILT_FORMAT_VERSION: usize = 0; /// The dummy replace string library names, inheritsFrom, and version names should be replaced with pub const DUMMY_REPLACE_STRING: &str = "${modrinth.gameVersion}"; diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 8b531712..42b93a80 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.19" +version = "0.1.20" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 6aa8f91f..d57cfd85 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -215,18 +215,27 @@ pub async fn retrieve_data( )) .await?; - versions.push(Version { - id: DUMMY_REPLACE_STRING.to_string(), - stable: true, - loaders: loader_version_mutex.into_inner(), - }); + let mut loader_version_mutex = loader_version_mutex.into_inner(); + if !loader_version_mutex.is_empty() { + if let Some(version) = versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) { + version.loaders.append(&mut loader_version_mutex); + } else { + versions.push(Version { + id: DUMMY_REPLACE_STRING.to_string(), + stable: true, + loaders: loader_version_mutex, + }); + } + } for version in &list.game { - versions.push(Version { - id: version.version.clone(), - stable: version.stable, - loaders: vec![], - }); + if !versions.iter().any(|x| x.id == version.version) { + versions.push(Version { + id: version.version.clone(), + stable: version.stable, + loaders: vec![], + }); + } } versions.sort_by(|x, y| { diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 7415cc65..fc3bf2f7 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -105,9 +105,16 @@ pub async fn retrieve_data( async move { /// These forge versions are not worth supporting! - const WHITELIST : [&str; 1] = [ + const WHITELIST : &[&str] = &[ // Not supported due to `data` field being `[]` even though the type is a map "1.12.2-14.23.5.2851", + // Malformed Archives + "1.6.1-8.9.0.749", + "1.6.1-8.9.0.751", + "1.6.4-9.11.1.960", + "1.6.4-9.11.1.961", + "1.6.4-9.11.1.963", + "1.6.4-9.11.1.964", ]; if WHITELIST.contains(&&*loader_version_full) { @@ -459,19 +466,20 @@ pub async fn retrieve_data( }); { + let len = loaders_futures.len(); let mut versions = loaders_futures.into_iter().peekable(); let mut chunk_index = 0; while versions.peek().is_some() { let now = Instant::now(); - let chunk: Vec<_> = versions.by_ref().take(10).collect(); + let chunk: Vec<_> = versions.by_ref().take(1).collect(); let res = futures::future::try_join_all(chunk).await?; loaders_versions.extend(res.into_iter().flatten()); chunk_index += 1; let elapsed = now.elapsed(); - info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + info!("Loader Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed); } } //futures::future::try_join_all(loaders_futures).await?; @@ -489,6 +497,7 @@ pub async fn retrieve_data( } { + let len = version_futures.len(); let mut versions = version_futures.into_iter().peekable(); let mut chunk_index = 0; while versions.peek().is_some() { @@ -500,7 +509,7 @@ pub async fn retrieve_data( chunk_index += 1; let elapsed = now.elapsed(); - info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); + info!("Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed); } } //futures::future::try_join_all(version_futures).await?; diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 92fee4cd..c4bb2471 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -9,6 +9,7 @@ use tokio::sync::Semaphore; mod fabric; mod forge; mod minecraft; +mod quilt; #[derive(thiserror::Error, Debug)] pub enum Error { @@ -45,7 +46,7 @@ async fn main() { } let mut timer = tokio::time::interval(Duration::from_secs(60 * 60)); - let semaphore = Arc::new(Semaphore::new(50)); + let semaphore = Arc::new(Semaphore::new(10)); loop { timer.tick().await; @@ -87,6 +88,16 @@ async fn main() { Ok(..) => {} Err(err) => error!("{:?}", err), }; + match quilt::retrieve_data( + &manifest, + &mut uploaded_files, + semaphore.clone(), + ) + .await + { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; } } } diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 024ce3d0..282fd934 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,10 +1,12 @@ use crate::download_file; use crate::{format_url, upload_file_to_bucket, Error}; -use daedalus::minecraft::VersionManifest; +use daedalus::minecraft::{Library, merge_partial_library, PartialLibrary, VersionManifest}; use log::info; use std::sync::Arc; use std::time::Instant; use tokio::sync::{Mutex, Semaphore}; +use serde::Deserialize; +use daedalus::get_hash; pub async fn retrieve_data( uploaded_files: &mut Vec, @@ -23,6 +25,9 @@ pub async fn retrieve_data( daedalus::minecraft::fetch_version_manifest(None).await?; let cloned_manifest = Arc::new(Mutex::new(manifest.clone())); + let patches = fetch_library_patches(None, semaphore.clone()).await?; + let cloned_patches = Arc::new(&patches); + let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); @@ -48,6 +53,7 @@ pub async fn retrieve_data( let cloned_manifest_mutex = Arc::clone(&cloned_manifest); let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); let semaphore = Arc::clone(&semaphore); + let patches = Arc::clone(&cloned_patches); let assets_hash = old_version.and_then(|x| x.assets_index_sha1.clone()); @@ -55,9 +61,30 @@ pub async fn retrieve_data( async move { let mut upload_futures = Vec::new(); - let version_info = + let mut version_info = daedalus::minecraft::fetch_version_info(version).await?; + let mut new_libraries = Vec::new(); + for library in version_info.libraries { + if let Some(patch) = patches.iter().find(|x| x.match_.contains(&library.name)) { + if let Some(additional_libraries) = &patch.additional_libraries { + new_libraries.push(library); + for additional_library in additional_libraries { + new_libraries.push(additional_library.clone()); + } + } else if let Some(override_) = &patch.override_ { + new_libraries.push(merge_partial_library(override_.clone(), library)); + } else { + new_libraries.push(library); + } + } else { + new_libraries.push(library); + } + } + version_info.libraries = new_libraries; + + let version_info_hash = get_hash(bytes::Bytes::from(serde_json::to_vec(&version_info)?)).await?; + let version_path = format!( "minecraft/v{}/versions/{}.json", daedalus::minecraft::CURRENT_FORMAT_VERSION, @@ -85,6 +112,7 @@ pub async fn retrieve_data( Some(version_info.asset_index.sha1.clone()); cloned_manifest.versions[position].assets_index_url = Some(format_url(&assets_path)); + cloned_manifest.versions[position].sha1 = version_info_hash; } let mut download_assets = false; @@ -187,3 +215,32 @@ pub async fn retrieve_data( .map_err(|_| Error::ArcError)? .into_inner()) } + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +/// A version of the fabric loader +struct LibraryPatch { + #[serde(rename = "_comment")] + pub _comment: String, + #[serde(rename = "match")] + pub match_: Vec, + pub additional_libraries: Option>, + #[serde(rename = "override")] + pub override_: Option, + pub patch_additional_libraries: Option, +} + +/// Fetches the list of fabric versions +async fn fetch_library_patches( + url: Option<&str>, + semaphore: Arc, +) -> Result, Error> { + Ok(serde_json::from_slice( + &download_file( + url.unwrap_or(&format_url("library-patches.json")), + None, + semaphore, + ) + .await?, + )?) +} diff --git a/daedalus_client/src/quilt.rs b/daedalus_client/src/quilt.rs new file mode 100644 index 00000000..9b122967 --- /dev/null +++ b/daedalus_client/src/quilt.rs @@ -0,0 +1,362 @@ +use crate::{download_file, format_url, upload_file_to_bucket, Error}; +use daedalus::minecraft::{Library, VersionManifest}; +use daedalus::modded::{ + LoaderVersion, Manifest, PartialVersionInfo, Version, DUMMY_REPLACE_STRING, +}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use tokio::sync::{Mutex, RwLock, Semaphore}; + +pub async fn retrieve_data( + minecraft_versions: &VersionManifest, + uploaded_files: &mut Vec, + semaphore: Arc, +) -> Result<(), Error> { + let mut list = fetch_quilt_versions(None, semaphore.clone()).await?; + let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( + "quilt/v{}/manifest.json", + daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, + ))) + .await + .ok(); + + let mut versions = if let Some(old_manifest) = old_manifest { + old_manifest.game_versions + } else { + Vec::new() + }; + + let loaders_mutex = RwLock::new(Vec::new()); + + { + let mut loaders = loaders_mutex.write().await; + + for loader in &list.loader { + loaders.push((Box::new(false), loader.version.clone())) + } + + list.loader + .retain(|x| loaders.iter().any(|val| val.1 == x.version)) + } + + const DUMMY_GAME_VERSION: &str = "1.19.4-rc2"; + + let loader_version_mutex = Mutex::new(Vec::new()); + let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); + + let loader_versions = futures::future::try_join_all( + loaders_mutex.read().await.clone().into_iter().map( + |(stable, loader)| async { + { + if versions.iter().any(|x| { + x.id == DUMMY_REPLACE_STRING + && x.loaders.iter().any(|x| x.id == loader) + }) { + return Ok(None); + } + } + + let version = fetch_quilt_version( + DUMMY_GAME_VERSION, + &loader, + semaphore.clone(), + ) + .await?; + + Ok::, String, PartialVersionInfo)>, Error>( + Some((stable, loader, version)), + ) + }, + ), + ) + .await?; + + let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); + futures::future::try_join_all(loader_versions.into_iter() + .flatten().map( + |(stable, loader, version)| async { + let libs = futures::future::try_join_all( + version.libraries.into_iter().map(|mut lib| async { + { + let mut visited_assets = + visited_artifacts_mutex.lock().await; + + if visited_assets.contains(&lib.name) { + lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); + lib.url = Some(format_url("maven/")); + + return Ok(lib); + } else { + visited_assets.push(lib.name.clone()) + } + } + + if lib.name.contains(DUMMY_GAME_VERSION) { + lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); + futures::future::try_join_all(list.game.clone().into_iter().map(|game_version| async { + let semaphore = semaphore.clone(); + let uploaded_files_mutex = uploaded_files_mutex.clone(); + let lib_name = lib.name.clone(); + let lib_url = lib.url.clone(); + + async move { + let artifact_path = + daedalus::get_path_from_artifact(&lib_name.replace(DUMMY_REPLACE_STRING, &game_version.version))?; + + let artifact = download_file( + &format!( + "{}{}", + lib_url.unwrap_or_else(|| { + "https://maven.quiltmc.org/".to_string() + }), + artifact_path + ), + None, + semaphore.clone(), + ) + .await?; + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + &uploaded_files_mutex, + semaphore.clone(), + ) + .await?; + + Ok::<(), Error>(()) + }.await?; + + Ok::<(), Error>(()) + })).await?; + lib.url = Some(format_url("maven/")); + + return Ok(lib); + } + + let artifact_path = + daedalus::get_path_from_artifact(&lib.name)?; + + let artifact = download_file( + &format!( + "{}{}", + lib.url.unwrap_or_else(|| { + "https://maven.quiltmc.org/".to_string() + }), + artifact_path + ), + None, + semaphore.clone(), + ) + .await?; + + lib.url = Some(format_url("maven/")); + + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + artifact.to_vec(), + Some("application/java-archive".to_string()), + &uploaded_files_mutex, + semaphore.clone(), + ) + .await?; + + Ok::(lib) + }), + ) + .await?; + + let version_path = format!( + "quilt/v{}/versions/{}.json", + daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, + &loader + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&PartialVersionInfo { + arguments: version.arguments, + id: version + .id + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), + main_class: version.main_class, + release_time: version.release_time, + time: version.time, + type_: version.type_, + inherits_from: version + .inherits_from + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), + libraries: libs, + minecraft_arguments: version.minecraft_arguments, + processors: None, + data: None, + })?, + Some("application/json".to_string()), + &uploaded_files_mutex, + semaphore.clone(), + ) + .await?; + + { + let mut loader_version_map = loader_version_mutex.lock().await; + async move { + loader_version_map.push(LoaderVersion { + id: loader.to_string(), + url: format_url(&version_path), + stable: *stable, + }); + } + .await; + } + + Ok::<(), Error>(()) + }, + )) + .await?; + + let mut loader_version_mutex = loader_version_mutex.into_inner(); + if !loader_version_mutex.is_empty() { + if let Some(version) = versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) { + version.loaders.append(&mut loader_version_mutex); + } else { + versions.push(Version { + id: DUMMY_REPLACE_STRING.to_string(), + stable: true, + loaders: loader_version_mutex, + }); + } + } + + for version in &list.game { + if !versions.iter().any(|x| x.id == version.version) { + versions.push(Version { + id: version.version.clone(), + stable: version.stable, + loaders: vec![], + }); + } + } + + versions.sort_by(|x, y| { + minecraft_versions + .versions + .iter() + .position(|z| x.id == z.id) + .unwrap_or_default() + .cmp( + &minecraft_versions + .versions + .iter() + .position(|z| y.id == z.id) + .unwrap_or_default(), + ) + }); + + for version in &mut versions { + version.loaders.sort_by(|x, y| { + list.loader + .iter() + .position(|z| { + x.id.split('-').next().unwrap_or_default() == &*z.version + }) + .unwrap_or_default() + .cmp( + &list + .loader + .iter() + .position(|z| { + y.id.split('-').next().unwrap_or_default() + == z.version + }) + .unwrap_or_default(), + ) + }) + } + + upload_file_to_bucket( + format!( + "quilt/v{}/manifest.json", + daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, + ), + serde_json::to_vec(&Manifest { + game_versions: versions, + })?, + Some("application/json".to_string()), + &uploaded_files_mutex, + semaphore, + ) + .await?; + + if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { + uploaded_files.extend(uploaded_files_mutex.into_inner()); + } + + Ok(()) +} + +const QUILT_META_URL: &str = "https://meta.quiltmc.org/v3"; + +async fn fetch_quilt_version( + version_number: &str, + loader_version: &str, + semaphore: Arc, +) -> Result { + Ok(serde_json::from_slice( + &download_file( + &format!( + "{}/versions/loader/{}/{}/profile/json", + QUILT_META_URL, version_number, loader_version + ), + None, + semaphore, + ) + .await?, + )?) +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// Versions of quilt components +struct QuiltVersions { + /// Versions of Minecraft that quilt supports + pub game: Vec, + /// Available versions of the quilt loader + pub loader: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// A version of Minecraft that quilt supports +struct QuiltGameVersion { + /// The version number of the game + pub version: String, + /// Whether the Minecraft version is stable or not + pub stable: bool, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// A version of the quilt loader +struct QuiltLoaderVersion { + /// The separator to get the build number + pub separator: String, + /// The build number + pub build: u32, + /// The maven artifact + pub maven: String, + /// The version number of the quilt loader + pub version: String, +} + +/// Fetches the list of quilt versions +async fn fetch_quilt_versions( + url: Option<&str>, + semaphore: Arc, +) -> Result { + Ok(serde_json::from_slice( + &download_file( + url.unwrap_or(&*format!("{}/versions", QUILT_META_URL)), + None, + semaphore, + ) + .await?, + )?) +} From 568e5a9bb89b7b2bd18f419d752eb2fc319224f7 Mon Sep 17 00:00:00 2001 From: Jai A Date: Tue, 25 Apr 2023 19:37:56 -0700 Subject: [PATCH 49/76] run fmt --- daedalus/src/minecraft.rs | 3 ++- daedalus_client/src/fabric.rs | 4 +++- daedalus_client/src/main.rs | 2 +- daedalus_client/src/minecraft.rs | 31 ++++++++++++++++++++++--------- daedalus_client/src/quilt.rs | 4 +++- 5 files changed, 31 insertions(+), 13 deletions(-) diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 49c87fe4..efa89ed9 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -207,7 +207,7 @@ pub enum Os { /// Linux ARM 32 LinuxArm32, /// The OS is unknown - Unknown + Unknown, } #[cfg_attr(feature = "bincode", derive(Encode, Decode))] @@ -321,6 +321,7 @@ pub struct PartialLibrary { pub include_in_classpath: Option, } +/// Merges a partial library to make a complete library pub fn merge_partial_library( partial: PartialLibrary, mut merge: Library, diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index d57cfd85..39c4d413 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -217,7 +217,9 @@ pub async fn retrieve_data( let mut loader_version_mutex = loader_version_mutex.into_inner(); if !loader_version_mutex.is_empty() { - if let Some(version) = versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) { + if let Some(version) = + versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) + { version.loaders.append(&mut loader_version_mutex); } else { versions.push(Version { diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index c4bb2471..db3639de 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -93,7 +93,7 @@ async fn main() { &mut uploaded_files, semaphore.clone(), ) - .await + .await { Ok(..) => {} Err(err) => error!("{:?}", err), diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 282fd934..30cf7f09 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,12 +1,14 @@ use crate::download_file; use crate::{format_url, upload_file_to_bucket, Error}; -use daedalus::minecraft::{Library, merge_partial_library, PartialLibrary, VersionManifest}; +use daedalus::get_hash; +use daedalus::minecraft::{ + merge_partial_library, Library, PartialLibrary, VersionManifest, +}; use log::info; +use serde::Deserialize; use std::sync::Arc; use std::time::Instant; use tokio::sync::{Mutex, Semaphore}; -use serde::Deserialize; -use daedalus::get_hash; pub async fn retrieve_data( uploaded_files: &mut Vec, @@ -66,14 +68,22 @@ pub async fn retrieve_data( let mut new_libraries = Vec::new(); for library in version_info.libraries { - if let Some(patch) = patches.iter().find(|x| x.match_.contains(&library.name)) { - if let Some(additional_libraries) = &patch.additional_libraries { + if let Some(patch) = patches + .iter() + .find(|x| x.match_.contains(&library.name)) + { + if let Some(additional_libraries) = + &patch.additional_libraries + { new_libraries.push(library); for additional_library in additional_libraries { new_libraries.push(additional_library.clone()); } } else if let Some(override_) = &patch.override_ { - new_libraries.push(merge_partial_library(override_.clone(), library)); + new_libraries.push(merge_partial_library( + override_.clone(), + library, + )); } else { new_libraries.push(library); } @@ -83,7 +93,10 @@ pub async fn retrieve_data( } version_info.libraries = new_libraries; - let version_info_hash = get_hash(bytes::Bytes::from(serde_json::to_vec(&version_info)?)).await?; + let version_info_hash = get_hash(bytes::Bytes::from( + serde_json::to_vec(&version_info)?, + )) + .await?; let version_path = format!( "minecraft/v{}/versions/{}.json", @@ -227,7 +240,7 @@ struct LibraryPatch { pub additional_libraries: Option>, #[serde(rename = "override")] pub override_: Option, - pub patch_additional_libraries: Option, + // pub patch_additional_libraries: Option, } /// Fetches the list of fabric versions @@ -241,6 +254,6 @@ async fn fetch_library_patches( None, semaphore, ) - .await?, + .await?, )?) } diff --git a/daedalus_client/src/quilt.rs b/daedalus_client/src/quilt.rs index 9b122967..19ebeb45 100644 --- a/daedalus_client/src/quilt.rs +++ b/daedalus_client/src/quilt.rs @@ -217,7 +217,9 @@ pub async fn retrieve_data( let mut loader_version_mutex = loader_version_mutex.into_inner(); if !loader_version_mutex.is_empty() { - if let Some(version) = versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) { + if let Some(version) = + versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) + { version.loaders.append(&mut loader_version_mutex); } else { versions.push(Version { From 3afda713499294d6e1f9744019df7b5d72ccfda6 Mon Sep 17 00:00:00 2001 From: Jai A Date: Wed, 26 Apr 2023 10:41:43 -0700 Subject: [PATCH 50/76] fix some data --- daedalus_client/src/minecraft.rs | 64 ++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 30cf7f09..113456a0 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -66,30 +66,48 @@ pub async fn retrieve_data( let mut version_info = daedalus::minecraft::fetch_version_info(version).await?; + fn patch_library(patches: &Vec, mut library: Library) -> Vec { + let mut val = Vec::new(); + + let actual_patches = patches + .iter() + .filter(|x| x.match_.contains(&library.name)) + .collect::>(); + + if !actual_patches.is_empty() + { + for patch in actual_patches { + if let Some(additional_libraries) = + &patch.additional_libraries + { + for additional_library in additional_libraries { + if patch.patch_additional_libraries.unwrap_or(false) { + let mut libs = patch_library(patches, additional_library.clone()); + val.append(&mut libs) + } else { + val.push(additional_library.clone()); + } + } + } else if let Some(override_) = &patch.override_ { + library = merge_partial_library( + override_.clone(), + library, + ); + } + } + + val.push(library); + } else { + val.push(library); + } + + val + } + let mut new_libraries = Vec::new(); for library in version_info.libraries { - if let Some(patch) = patches - .iter() - .find(|x| x.match_.contains(&library.name)) - { - if let Some(additional_libraries) = - &patch.additional_libraries - { - new_libraries.push(library); - for additional_library in additional_libraries { - new_libraries.push(additional_library.clone()); - } - } else if let Some(override_) = &patch.override_ { - new_libraries.push(merge_partial_library( - override_.clone(), - library, - )); - } else { - new_libraries.push(library); - } - } else { - new_libraries.push(library); - } + let mut libs = patch_library(&patches, library); + new_libraries.append(&mut libs) } version_info.libraries = new_libraries; @@ -240,7 +258,7 @@ struct LibraryPatch { pub additional_libraries: Option>, #[serde(rename = "override")] pub override_: Option, - // pub patch_additional_libraries: Option, + pub patch_additional_libraries: Option, } /// Fetches the list of fabric versions From 3db00534c24edfab910f0b02dfca75b1adb9448c Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Fri, 12 May 2023 20:42:18 -0700 Subject: [PATCH 51/76] Fix launcher overrides (#7) * Fix launcher overrides * Remove none set * Patch to work with legacy versions * remove mc --- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 33 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/library-patches.json | 2020 ++++++++++++++++++++++++++ daedalus_client/src/minecraft.rs | 36 +- 5 files changed, 2068 insertions(+), 25 deletions(-) create mode 100644 daedalus_client/library-patches.json diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 94cbde71..fbb6c0e4 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.20" +version = "0.1.21" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index efa89ed9..87506720 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -327,7 +327,22 @@ pub fn merge_partial_library( mut merge: Library, ) -> Library { if let Some(downloads) = partial.downloads { - merge.downloads = Some(downloads) + if let Some(merge_downloads) = &mut merge.downloads { + if let Some(artifact) = downloads.artifact { + merge_downloads.artifact = Some(artifact); + } + if let Some(classifiers) = downloads.classifiers { + if let Some(merge_classifiers) = &mut merge_downloads.classifiers { + for classifier in classifiers { + merge_classifiers.insert(classifier.0, classifier.1); + } + } else { + merge_downloads.classifiers = Some(classifiers); + } + } + } else { + merge.downloads = Some(downloads) + } } if let Some(extract) = partial.extract { merge.extract = Some(extract) @@ -339,10 +354,22 @@ pub fn merge_partial_library( merge.url = Some(url) } if let Some(natives) = partial.natives { - merge.natives = Some(natives) + if let Some(merge_natives) = &mut merge.natives { + for native in natives { + merge_natives.insert(native.0, native.1); + } + } else { + merge.natives = Some(natives); + } } if let Some(rules) = partial.rules { - merge.rules = Some(rules) + if let Some(merge_rules) = &mut merge.rules { + for rule in rules { + merge_rules.push(rule); + } + } else { + merge.rules = Some(rules) + } } if let Some(checksums) = partial.checksums { merge.checksums = Some(checksums) diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 42b93a80..48a59975 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.20" +version = "0.1.21" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/library-patches.json b/daedalus_client/library-patches.json new file mode 100644 index 00000000..31a6a8cb --- /dev/null +++ b/daedalus_client/library-patches.json @@ -0,0 +1,2020 @@ +[ + { + "_comment": "Add missing tinyfd to the broken LWJGL 3.2.2 variant", + "match": [ + "org.lwjgl:lwjgl:3.2.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "fcbe606c8f8da6f8f9a05e2c540eb1ee8632b0e9", + "size": 7092, + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl-tinyfd/3.2.2/lwjgl-tinyfd-3.2.2.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.2.2" + }, + { + "downloads": { + "artifact": { + "sha1": "fcbe606c8f8da6f8f9a05e2c540eb1ee8632b0e9", + "size": 7092, + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl-tinyfd/3.2.2/lwjgl-tinyfd-3.2.2.jar" + }, + "classifiers": { + "natives-linux": { + "sha1": "39e35b161c130635d9c8918ce04e887a30c5b687", + "size": 38804, + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl-tinyfd/3.2.2/lwjgl-tinyfd-3.2.2-natives-linux.jar" + }, + "natives-macos": { + "sha1": "46d0798228b8a28e857a2a0f02310fd6ba2a4eab", + "size": 42136, + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl-tinyfd/3.2.2/lwjgl-tinyfd-3.2.2-natives-macos.jar" + }, + "natives-windows": { + "sha1": "e9115958773644e863332a6a06488d26f9e1fc9f", + "size": 208314, + "url": + "https://libraries.minecraft.net/org/lwjgl/lwjgl-tinyfd/3.2.2/lwjgl-tinyfd-3.2.2-natives-windows.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.2.2", + "natives": { + "linux": "natives-linux", + "osx": "natives-macos", + "windows": "natives-windows" + } + } + ], + "patchAdditionalLibraries": true + }, + { + "_comment": "Add additional library just for osx-arm64. No override needed", + "match": [ + "ca.weblite:java-objc-bridge:1.0.0" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "369a83621e3c65496348491e533cb97fe5f2f37d", + "size": 91947, + "url": + "https://github.com/MinecraftMachina/Java-Objective-C-Bridge/releases/download/1.1.0-mmachina.1/java-objc-bridge-1.1.jar" + } + }, + "name": "ca.weblite:java-objc-bridge:1.1.0-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add additional classifiers for jinput-platform", + "match": [ + "net.java.jinput:jinput-platform:2.0.5" + ], + "override": { + "downloads": { + "classifiers": { + "natives-osx-arm64": { + "sha1": "5189eb40db3087fb11ca063b68fa4f4c20b199dd", + "size": 10031, + "url": "https://github.com/r58Playz/jinput-m1/raw/main/plugins/OSX/bin/jinput-platform-2.0.5.jar" + }, + "natives-linux-arm64": { + "sha1": "42b388ccb7c63cec4e9f24f4dddef33325f8b212", + "size": 10932, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-2.9.4/jinput-platform-2.0.5-natives-linux.jar" + }, + "natives-linux-arm32": { + "sha1": "f3c455b71c5146acb5f8a9513247fc06db182fd5", + "size": 4521, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-2.9.4/jinput-platform-2.0.5-natives-linux.jar" + } + } + }, + "natives": { + "linux-arm64": "natives-linux-arm64", + "linux-arm32": "natives-linux-arm32", + "osx-arm64": "natives-osx-arm64" + } + } + }, + { + "_comment": "Use a newer version on osx-arm64", + "match": [ + "com.mojang:text2speech:1.0.10", + "com.mojang:text2speech:1.5", + "com.mojang:text2speech:1.6", + "com.mojang:text2speech:1.7", + "com.mojang:text2speech:1.10.1", + "com.mojang:text2speech:1.10.3", + "com.mojang:text2speech:1.11.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "f378f889797edd7df8d32272c06ca80a1b6b0f58", + "size": 13164, + "url": "https://libraries.minecraft.net/com/mojang/text2speech/1.11.3/text2speech-1.11.3.jar" + } + }, + "name": "com.mojang:text2speech:1.11.3", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl.lwjgl:lwjgl:2.9.3", + "org.lwjgl.lwjgl:lwjgl:2.9.1-nightly-20131120", + "org.lwjgl.lwjgl:lwjgl:2.9.1-nightly-20131017", + "org.lwjgl.lwjgl:lwjgl:2.9.1-nightly-20130708-debug3", + "org.lwjgl.lwjgl:lwjgl:2.9.1", + "org.lwjgl.lwjgl:lwjgl:2.9.0" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "697517568c68e78ae0b4544145af031c81082dfe", + "size": 1047168, + "url": + "https://libraries.minecraft.net/org/lwjgl/lwjgl/lwjgl/2.9.4-nightly-20150209/lwjgl-2.9.4-nightly-20150209.jar" + } + }, + "name": "org.lwjgl.lwjgl:lwjgl:2.9.4-nightly-20150209", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + }, + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl.lwjgl:lwjgl_util:2.9.3", + "org.lwjgl.lwjgl:lwjgl_util:2.9.1-nightly-20131120", + "org.lwjgl.lwjgl:lwjgl_util:2.9.1-nightly-20131017", + "org.lwjgl.lwjgl:lwjgl_util:2.9.1-nightly-20130708-debug3", + "org.lwjgl.lwjgl:lwjgl_util:2.9.1", + "org.lwjgl.lwjgl:lwjgl_util:2.9.0" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "d51a7c040a721d13efdfbd34f8b257b2df882ad0", + "size": 173887, + "url": + "https://libraries.minecraft.net/org/lwjgl/lwjgl/lwjgl_util/2.9.4-nightly-20150209/lwjgl_util-2.9.4-nightly-20150209.jar" + } + }, + "name": "org.lwjgl.lwjgl:lwjgl_util:2.9.4-nightly-20150209", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + }, + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl.lwjgl:lwjgl-platform:2.9.4-nightly-20150209", + "org.lwjgl.lwjgl:lwjgl-platform:2.9.3", + "org.lwjgl.lwjgl:lwjgl-platform:2.9.1-nightly-20131120", + "org.lwjgl.lwjgl:lwjgl-platform:2.9.1-nightly-20131017", + "org.lwjgl.lwjgl:lwjgl-platform:2.9.1-nightly-20130708-debug3", + "org.lwjgl.lwjgl:lwjgl-platform:2.9.1", + "org.lwjgl.lwjgl:lwjgl-platform:2.9.0" + ], + "override": { + "downloads": { + "classifiers": { + "natives-osx-arm64": { + "sha1": "eff546c0b319d6ffc7a835652124c18089c67f36", + "size": 488316, + "url": + "https://github.com/MinecraftMachina/lwjgl/releases/download/2.9.4-20150209-mmachina.2/lwjgl-platform-2.9.4-nightly-20150209-natives-osx.jar" + }, + "natives-linux-arm64": { + "sha1": "63ac7da0f4a4785c7eadc0f8edc1e9dcc4dd08cb", + "size": 579979, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-2.9.4/lwjgl-platform-2.9.4-nightly-20150209-natives-linux.jar" + }, + "natives-linux-arm32": { + "sha1": "fa483e540a9a753a5ffbb23dcf7879a5bf752611", + "size": 475177, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-2.9.4/lwjgl-platform-2.9.4-nightly-20150209-natives-linux.jar" + } + } + }, + "natives": { + "linux-arm64": "natives-linux-arm64", + "linux-arm32": "natives-linux-arm32", + "osx-arm64": "natives-osx-arm64" + } + } + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl-glfw:3.2.2", + "org.lwjgl:lwjgl-glfw:3.2.1", + "org.lwjgl:lwjgl-glfw:3.1.6", + "org.lwjgl:lwjgl-glfw:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "155d175037efc76630940c197ca6dea2b17d7e18", + "size": 108691, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-glfw.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "155d175037efc76630940c197ca6dea2b17d7e18", + "size": 108691, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-glfw.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "074ad243761147df0d060fbefc814614d2ff75cc", + "size": 85072, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-glfw-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-glfw:3.2.2-gman64.1", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "99e9a39fa8ed4167e3ff9e04d47eb32c9e69804d", + "size": 108691, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-glfw.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "99e9a39fa8ed4167e3ff9e04d47eb32c9e69804d", + "size": 108691, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-glfw.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "4265f2fbe3b9d642591165165a17cf406cf7b98e", + "size": 80186, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-glfw-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-glfw:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "e9a101bca4fa30d26b21b526ff28e7c2d8927f1b", + "size": 130128, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-glfw.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "e9a101bca4fa30d26b21b526ff28e7c2d8927f1b", + "size": 130128, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-glfw.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "71d793d0a5a42e3dfe78eb882abc2523a2c6b496", + "size": 129076, + "url": + "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-glfw-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-glfw:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl-jemalloc:3.2.2", + "org.lwjgl:lwjgl-jemalloc:3.2.1", + "org.lwjgl:lwjgl-jemalloc:3.1.6", + "org.lwjgl:lwjgl-jemalloc:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "cc04eec29b2fa8c298791af9800a3766d9617954", + "size": 33790, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-jemalloc.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "cc04eec29b2fa8c298791af9800a3766d9617954", + "size": 33790, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-jemalloc.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "762d7d80c9cdf3a3f3fc80c8a5f86612255edfe0", + "size": 156343, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-jemalloc-patched-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-jemalloc:3.2.2-gman64.2", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "8224ae2e8fc6d8e1a0fc7d84dc917aa3c440620c", + "size": 33790, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-jemalloc.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "8224ae2e8fc6d8e1a0fc7d84dc917aa3c440620c", + "size": 33790, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-jemalloc.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "9163a2a5559ef87bc13ead8fea84417ea3928748", + "size": 134237, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-jemalloc-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-jemalloc:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "4fb94224378d3588d52d2beb172f2eeafea2d546", + "size": 36976, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-jemalloc.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "4fb94224378d3588d52d2beb172f2eeafea2d546", + "size": 36976, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-jemalloc.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "b0be721188d2e7195798780b1c5fe7eafe8091c1", + "size": 103478, + "url": + "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-jemalloc-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-jemalloc:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl-openal:3.2.2", + "org.lwjgl:lwjgl-openal:3.2.1", + "org.lwjgl:lwjgl-openal:3.1.6", + "org.lwjgl:lwjgl-openal:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "6dfce9dc6a9629c75b2ae01a8df7e7be80ba0261", + "size": 79582, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-openal.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "6dfce9dc6a9629c75b2ae01a8df7e7be80ba0261", + "size": 79582, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-openal.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "948e415b5b2a2c650c25b377a4a9f443b21ce92e", + "size": 469432, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-openal-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-openal:3.2.2-gman64.1", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "304f0571fd5971621ee6da86a4c1e90f6f52e2ee", + "size": 79582, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-openal.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "304f0571fd5971621ee6da86a4c1e90f6f52e2ee", + "size": 79582, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-openal.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "ecbc981fdd996492a1f6334f003ed62e5a8c0cd5", + "size": 398418, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-openal-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-openal:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "d48e753d85916fc8a200ccddc709b36e3865cc4e", + "size": 88880, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-openal.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "d48e753d85916fc8a200ccddc709b36e3865cc4e", + "size": 88880, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-openal.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "6b80fc0b982a0723b141e88859c42d6f71bd723f", + "size": 346131, + "url": + "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-openal-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-openal:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl-opengl:3.2.2", + "org.lwjgl:lwjgl-opengl:3.2.1", + "org.lwjgl:lwjgl-opengl:3.1.6", + "org.lwjgl:lwjgl-opengl:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "198bc2f72e0b2eb401eb6f5999aea52909b31ac4", + "size": 937609, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-opengl.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "198bc2f72e0b2eb401eb6f5999aea52909b31ac4", + "size": 937609, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-opengl.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "bd40897077bf7d12f562da898b18ac2c68e1f9d7", + "size": 56109, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-opengl-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-opengl:3.2.2-gman64.1", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "9762ae928d02147e716cd82e929b74a97ea9600a", + "size": 937609, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-opengl.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "9762ae928d02147e716cd82e929b74a97ea9600a", + "size": 937609, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-opengl.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "3af5599c74dd76dd8dbb567b3f9b4963a6abeed5", + "size": 56388, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-opengl-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-opengl:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "962c2a8d2a8cdd3b89de3d78d766ab5e2133c2f4", + "size": 929233, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-opengl.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "962c2a8d2a8cdd3b89de3d78d766ab5e2133c2f4", + "size": 929233, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-opengl.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "bb575058e0372f515587b5d2d04ff7db185f3ffe", + "size": 41667, + "url": + "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-opengl-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-opengl:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl-stb:3.2.2", + "org.lwjgl:lwjgl-stb:3.2.1", + "org.lwjgl:lwjgl-stb:3.1.6", + "org.lwjgl:lwjgl-stb:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "46a5735f3eb9d17eb5dcbdd5afa194066d2a6555", + "size": 104075, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-stb.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "46a5735f3eb9d17eb5dcbdd5afa194066d2a6555", + "size": 104075, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-stb.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "077efa7d7ea41b32df5c6078e912e724cccd06db", + "size": 202038, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-stb-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-stb:3.2.2-gman64.1", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "ea979b0af45b8e689f5f47c989aa8550c148d8a2", + "size": 104075, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-stb.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "ea979b0af45b8e689f5f47c989aa8550c148d8a2", + "size": 104075, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-stb.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "ec9d70aaebd0ff76dfeecf8f00b56118bf3706b1", + "size": 149387, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-stb-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-stb:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "703e4b533e2542560e9f94d6d8bd148be1c1d572", + "size": 113273, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-stb.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "703e4b533e2542560e9f94d6d8bd148be1c1d572", + "size": 113273, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-stb.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "98f0ad956c754723ef354d50057cc30417ef376a", + "size": 178409, + "url": + "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-stb-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-stb:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl-tinyfd:3.2.2", + "org.lwjgl:lwjgl-tinyfd:3.2.1", + "org.lwjgl:lwjgl-tinyfd:3.1.6", + "org.lwjgl:lwjgl-tinyfd:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "3a75b9811607633bf33c978f53964df1534a4bc1", + "size": 5571, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-tinyfd.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "3a75b9811607633bf33c978f53964df1534a4bc1", + "size": 5571, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-tinyfd.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "37c744ca289b5d7ae155d79e39029488b3254e5b", + "size": 37893, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-tinyfd-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.2.2-gman64.1", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "a8c09f5b7fa24bd53ec329c231b566497a163d5b", + "size": 5571, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-tinyfd.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "a8c09f5b7fa24bd53ec329c231b566497a163d5b", + "size": 5571, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-tinyfd.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "82d16054ada6633297a3108fb6d8bae98800c76f", + "size": 41663, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-tinyfd-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "1203660b3131cbb8681b17ce6437412545be95e0", + "size": 6802, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-tinyfd.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "1203660b3131cbb8681b17ce6437412545be95e0", + "size": 6802, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-tinyfd.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "015b931a2daba8f0c317d84c9d14e8e98ae56e0c", + "size": 41384, + "url": + "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-tinyfd-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl-tinyfd:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Use a newer patched version on osx-arm64, linux-arm64, and linux-arm32", + "match": [ + "org.lwjgl:lwjgl:3.2.2", + "org.lwjgl:lwjgl:3.2.1", + "org.lwjgl:lwjgl:3.1.6", + "org.lwjgl:lwjgl:3.1.2" + ], + "override": { + "rules": [ + { + "action": "allow" + }, + { + "action": "disallow", + "os": { + "name": "linux-arm64" + } + }, + { + "action": "disallow", + "os": { + "name": "linux-arm32" + } + }, + { + "action": "disallow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "360899386df83d6a8407844a94478607af937f97", + "size": 318833, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-core.jar" + } + }, + "name": "org.lwjgl:lwjgl:3.2.2-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "360899386df83d6a8407844a94478607af937f97", + "size": 318833, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-core.jar" + }, + "classifiers": { + "natives-linux-arm64": { + "sha1": "612efd57d12b2e48e554858eb35e7e2eb46ebb4c", + "size": 87121, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-natives-linux-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl:3.2.2-gman64.1", + "natives": { + "linux-arm64": "natives-linux-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "16ea3934fca417368250d1ddac01a30c1809d317", + "size": 318413, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-core.jar" + } + }, + "name": "org.lwjgl:lwjgl:3.2.2-gman32.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "16ea3934fca417368250d1ddac01a30c1809d317", + "size": 318413, + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-core.jar" + }, + "classifiers": { + "natives-linux-arm32": { + "sha1": "6bd0b37fef777a309936a72dc7f63126e8c79ea5", + "size": 90296, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-natives-linux-arm32.jar" + } + } + }, + "name": "org.lwjgl:lwjgl:3.2.2-gman32.1", + "natives": { + "linux-arm32": "natives-linux-arm32" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "8e664dd69ad7bbcf2053da23efc7848e39e498db", + "size": 719038, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl.jar" + } + }, + "name": "org.lwjgl:lwjgl:3.3.1-mmachina.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + }, + { + "downloads": { + "artifact": { + "sha1": "8e664dd69ad7bbcf2053da23efc7848e39e498db", + "size": 719038, + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl.jar" + }, + "classifiers": { + "natives-osx-arm64": { + "sha1": "984df31fadaab86838877b112e5b4e4f68a00ccf", + "size": 42693, + "url": + "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-natives-macos-arm64.jar" + } + } + }, + "name": "org.lwjgl:lwjgl:3.3.1-mmachina.1", + "natives": { + "osx-arm64": "natives-osx-arm64" + }, + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Only allow osx-arm64 for existing LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-glfw-natives-macos-arm64:3.3.1", + "org.lwjgl:lwjgl-jemalloc-natives-macos-arm64:3.3.1", + "org.lwjgl:lwjgl-openal-natives-macos-arm64:3.3.1", + "org.lwjgl:lwjgl-opengl-natives-macos-arm64:3.3.1", + "org.lwjgl:lwjgl-stb-natives-macos-arm64:3.3.1", + "org.lwjgl:lwjgl-tinyfd-natives-macos-arm64:3.3.1", + "org.lwjgl:lwjgl-natives-macos-arm64:3.3.1" + ], + "override": { + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + }, + { + "_comment": "Only allow osx-arm64 for existing java-objc-bridge:1.1", + "match": [ + "ca.weblite:java-objc-bridge:1.1" + ], + "override": { + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + }, + { + "_comment": "Only allow windows-arm64 for existing LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-glfw-natives-windows-arm64:3.3.1", + "org.lwjgl:lwjgl-jemalloc-natives-windows-arm64:3.3.1", + "org.lwjgl:lwjgl-openal-natives-windows-arm64:3.3.1", + "org.lwjgl:lwjgl-opengl-natives-windows-arm64:3.3.1", + "org.lwjgl:lwjgl-stb-natives-windows-arm64:3.3.1", + "org.lwjgl:lwjgl-tinyfd-natives-windows-arm64:3.3.1", + "org.lwjgl:lwjgl-natives-windows-arm64:3.3.1" + ], + "override": { + "rules": [ + { + "action": "allow", + "os": { + "name": "windows-arm64" + } + } + ] + } + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-glfw:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "513eb39b866d0fe131a18d5c517087805433b029", + "size": 112350, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-glfw/lwjgl-glfw-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux-arm64:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-jemalloc:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "749be48a9b86ee2c3a2da5fd77511208adcfb33b", + "size": 159993, + "url": + "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.3.1/lwjgl-jemalloc-patched-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm64:3.3.1-gman64.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-openal:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "cf4e303257e82981b8b2e31bba3d7f8f7b8f42b2", + "size": 470743, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-openal/lwjgl-openal-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal-natives-linux-arm64:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-opengl:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "1c528fb258a6e63e8fceb4482d8db0f3af10a634", + "size": 57908, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-opengl/lwjgl-opengl-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl-natives-linux-arm64:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-stb:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "8e8348a1813aad7f30aaf75ea197151ebb7beba9", + "size": 205491, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-stb/lwjgl-stb-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb-natives-linux-arm64:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-tinyfd:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "964f628b7a82fd909def086c0dd9a4b84bb259ae", + "size": 42654, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-tinyfd/lwjgl-tinyfd-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd-natives-linux-arm64:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "b597401014acb7196c76d97e15a6288f54f1f692", + "size": 86308, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl/lwjgl-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-natives-linux-arm64:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-glfw:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "816d935933f2dd743074c4e717cc25b55720f294", + "size": 104027, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-glfw/lwjgl-glfw-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-jemalloc:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "a96a6d6cb3876d7813fcee53c3c24f246aeba3b3", + "size": 136157, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-jemalloc/lwjgl-jemalloc-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-openal:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "ffbe35d7fa5ec9b7eca136a7c71f24d4025a510b", + "size": 400129, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-openal/lwjgl-openal-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-opengl:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "e3550fa91097fd56e361b4370fa822220fef3595", + "size": 58474, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-opengl/lwjgl-opengl-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-stb:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "b08226bab162c06ae69337d8a1b0ee0a3fdf0b90", + "size": 153889, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-stb/lwjgl-stb-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl-tinyfd:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "d53d331e859217a61298fcbcf8d79137f3df345c", + "size": 48061, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl-tinyfd/lwjgl-tinyfd-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.1", + "match": [ + "org.lwjgl:lwjgl:3.3.1" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "41a3c1dd15d6b964eb8196dde69720a3e3e5e969", + "size": 82374, + "url": "https://build.lwjgl.org/release/3.3.1/bin/lwjgl/lwjgl-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-natives-linux-arm32:3.3.1-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Replace glfw from 3.3.1 with version from 3.3.2 to prevent stack smashing", + "match": [ + "org.lwjgl:lwjgl-glfw-natives-linux:3.3.1" + ], + "override": { + "downloads": { + "artifact": { + "sha1": "0766bb0e8e829598b1c8052fd8173c62af741c52", + "size": 115553, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-glfw/lwjgl-glfw-natives-linux.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux:3.3.2-lwjgl.1" + } + } +] + diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 113456a0..352bd6fc 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,9 +1,7 @@ use crate::download_file; use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::get_hash; -use daedalus::minecraft::{ - merge_partial_library, Library, PartialLibrary, VersionManifest, -}; +use daedalus::minecraft::{merge_partial_library, Library, PartialLibrary, VersionManifest, VersionInfo}; use log::info; use serde::Deserialize; use std::sync::Arc; @@ -66,7 +64,7 @@ pub async fn retrieve_data( let mut version_info = daedalus::minecraft::fetch_version_info(version).await?; - fn patch_library(patches: &Vec, mut library: Library) -> Vec { + fn patch_library(patches: &Vec, mut library: Library, version_info: &VersionInfo) -> Vec { let mut val = Vec::new(); let actual_patches = patches @@ -77,22 +75,26 @@ pub async fn retrieve_data( if !actual_patches.is_empty() { for patch in actual_patches { + println!("{} {}", version_info.id, patch._comment); + + if let Some(override_) = &patch.override_ { + library = merge_partial_library( + override_.clone(), + library, + ); + } + if let Some(additional_libraries) = &patch.additional_libraries { for additional_library in additional_libraries { if patch.patch_additional_libraries.unwrap_or(false) { - let mut libs = patch_library(patches, additional_library.clone()); + let mut libs = patch_library(patches, additional_library.clone(), &version_info); val.append(&mut libs) } else { val.push(additional_library.clone()); } } - } else if let Some(override_) = &patch.override_ { - library = merge_partial_library( - override_.clone(), - library, - ); } } @@ -105,8 +107,8 @@ pub async fn retrieve_data( } let mut new_libraries = Vec::new(); - for library in version_info.libraries { - let mut libs = patch_library(&patches, library); + for library in version_info.libraries.clone() { + let mut libs = patch_library(&patches, library, &version_info); new_libraries.append(&mut libs) } version_info.libraries = new_libraries; @@ -266,12 +268,6 @@ async fn fetch_library_patches( url: Option<&str>, semaphore: Arc, ) -> Result, Error> { - Ok(serde_json::from_slice( - &download_file( - url.unwrap_or(&format_url("library-patches.json")), - None, - semaphore, - ) - .await?, - )?) + let patches = include_bytes!("../library-patches.json"); + Ok(serde_json::from_slice(patches)?) } From 6c628afe5d9067b19ccfb03291b9bbb29e94548b Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Fri, 2 Jun 2023 16:13:00 -0700 Subject: [PATCH 52/76] Auto doing fixes (#8) --- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 4 +++- daedalus_client/Cargo.toml | 4 ++-- daedalus_client/src/minecraft.rs | 41 ++++++++++++++++++-------------- 4 files changed, 29 insertions(+), 22 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index fbb6c0e4..2ecbc192 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.21" +version = "0.1.22" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 87506720..18ff41fa 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -332,7 +332,9 @@ pub fn merge_partial_library( merge_downloads.artifact = Some(artifact); } if let Some(classifiers) = downloads.classifiers { - if let Some(merge_classifiers) = &mut merge_downloads.classifiers { + if let Some(merge_classifiers) = + &mut merge_downloads.classifiers + { for classifier in classifiers { merge_classifiers.insert(classifier.0, classifier.1); } diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 48a59975..fc7f2f41 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.21" +version = "0.1.22" authors = ["Jai A "] edition = "2018" @@ -22,4 +22,4 @@ zip = "0.6.3" semver = "1.0" chrono = { version = "0.4", features = ["serde"] } bytes = "1.3.0" -rust-s3 = "0.32.3" +rust-s3 = "0.33.0" diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 352bd6fc..231a0343 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,7 +1,10 @@ use crate::download_file; use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::get_hash; -use daedalus::minecraft::{merge_partial_library, Library, PartialLibrary, VersionManifest, VersionInfo}; +use daedalus::minecraft::{ + merge_partial_library, Library, PartialLibrary, + VersionManifest, +}; use log::info; use serde::Deserialize; use std::sync::Arc; @@ -25,7 +28,7 @@ pub async fn retrieve_data( daedalus::minecraft::fetch_version_manifest(None).await?; let cloned_manifest = Arc::new(Mutex::new(manifest.clone())); - let patches = fetch_library_patches(None, semaphore.clone()).await?; + let patches = fetch_library_patches()?; let cloned_patches = Arc::new(&patches); let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); @@ -43,10 +46,8 @@ pub async fn retrieve_data( None }; - if let Some(old_version) = old_version { - if old_version.sha1 == version.sha1 { - return Ok(()); - } + if old_version.is_some() { + return Ok(()); } let visited_assets_mutex = Arc::clone(&visited_assets_mutex); @@ -64,7 +65,10 @@ pub async fn retrieve_data( let mut version_info = daedalus::minecraft::fetch_version_info(version).await?; - fn patch_library(patches: &Vec, mut library: Library, version_info: &VersionInfo) -> Vec { + fn patch_library( + patches: &Vec, + mut library: Library, + ) -> Vec { let mut val = Vec::new(); let actual_patches = patches @@ -72,11 +76,8 @@ pub async fn retrieve_data( .filter(|x| x.match_.contains(&library.name)) .collect::>(); - if !actual_patches.is_empty() - { + if !actual_patches.is_empty() { for patch in actual_patches { - println!("{} {}", version_info.id, patch._comment); - if let Some(override_) = &patch.override_ { library = merge_partial_library( override_.clone(), @@ -88,8 +89,14 @@ pub async fn retrieve_data( &patch.additional_libraries { for additional_library in additional_libraries { - if patch.patch_additional_libraries.unwrap_or(false) { - let mut libs = patch_library(patches, additional_library.clone(), &version_info); + if patch + .patch_additional_libraries + .unwrap_or(false) + { + let mut libs = patch_library( + patches, + additional_library.clone(), + ); val.append(&mut libs) } else { val.push(additional_library.clone()); @@ -108,7 +115,8 @@ pub async fn retrieve_data( let mut new_libraries = Vec::new(); for library in version_info.libraries.clone() { - let mut libs = patch_library(&patches, library, &version_info); + let mut libs = + patch_library(&patches, library); new_libraries.append(&mut libs) } version_info.libraries = new_libraries; @@ -264,10 +272,7 @@ struct LibraryPatch { } /// Fetches the list of fabric versions -async fn fetch_library_patches( - url: Option<&str>, - semaphore: Arc, -) -> Result, Error> { +fn fetch_library_patches() -> Result, Error> { let patches = include_bytes!("../library-patches.json"); Ok(serde_json::from_slice(patches)?) } From 0d56127758944ff0d8fb23bf2fa12daa9d2d05fc Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Sun, 11 Jun 2023 13:40:31 -0700 Subject: [PATCH 53/76] Add new mojang args (#9) --- daedalus/Cargo.toml | 2 +- daedalus/src/minecraft.rs | 15 +++++++++++++-- daedalus_client/Cargo.toml | 2 +- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 2ecbc192..f00c0ae8 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.22" +version = "0.1.23" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 18ff41fa..747fa8a0 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -233,8 +233,19 @@ pub struct FeatureRule { /// Whether the user is in demo mode pub is_demo_user: Option, #[serde(skip_serializing_if = "Option::is_none")] - /// Whether the user is using the demo resolution - pub has_demo_resolution: Option, + /// Whether the user is using a custom resolution + pub has_custom_resolution: Option, + #[serde(skip_serializing_if = "Option::is_none")] + /// Whether the launcher has quick plays support + pub has_quick_plays_support: Option, + #[serde(skip_serializing_if = "Option::is_none")] + /// Whether the instance is being launched to a single-player world + pub is_quick_play_singleplayer: Option, + #[serde(skip_serializing_if = "Option::is_none")] + /// Whether the instance is being launched to a multi-player world + pub is_quick_play_multiplayer: Option, + /// Whether the instance is being launched to a realms world + pub is_quick_play_realms: Option, } #[cfg_attr(feature = "bincode", derive(Encode, Decode))] diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index fc7f2f41..a0d74911 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.22" +version = "0.1.23" authors = ["Jai A "] edition = "2018" From 10e7b66f38217f8ab735a80b1d67de956bd6aed6 Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Fri, 28 Jul 2023 16:22:38 -0700 Subject: [PATCH 54/76] Fix loader ordering (#10) --- daedalus_client/src/fabric.rs | 9 ++------- daedalus_client/src/minecraft.rs | 6 ++---- daedalus_client/src/quilt.rs | 9 ++------- 3 files changed, 6 insertions(+), 18 deletions(-) diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 39c4d413..94d2488f 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -259,18 +259,13 @@ pub async fn retrieve_data( version.loaders.sort_by(|x, y| { list.loader .iter() - .position(|z| { - x.id.split('-').next().unwrap_or_default() == &*z.version - }) + .position(|z| x.id == *z.version) .unwrap_or_default() .cmp( &list .loader .iter() - .position(|z| { - y.id.split('-').next().unwrap_or_default() - == z.version - }) + .position(|z| y.id == z.version) .unwrap_or_default(), ) }) diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 231a0343..6badc9c4 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -2,8 +2,7 @@ use crate::download_file; use crate::{format_url, upload_file_to_bucket, Error}; use daedalus::get_hash; use daedalus::minecraft::{ - merge_partial_library, Library, PartialLibrary, - VersionManifest, + merge_partial_library, Library, PartialLibrary, VersionManifest, }; use log::info; use serde::Deserialize; @@ -115,8 +114,7 @@ pub async fn retrieve_data( let mut new_libraries = Vec::new(); for library in version_info.libraries.clone() { - let mut libs = - patch_library(&patches, library); + let mut libs = patch_library(&patches, library); new_libraries.append(&mut libs) } version_info.libraries = new_libraries; diff --git a/daedalus_client/src/quilt.rs b/daedalus_client/src/quilt.rs index 19ebeb45..827ad00e 100644 --- a/daedalus_client/src/quilt.rs +++ b/daedalus_client/src/quilt.rs @@ -259,18 +259,13 @@ pub async fn retrieve_data( version.loaders.sort_by(|x, y| { list.loader .iter() - .position(|z| { - x.id.split('-').next().unwrap_or_default() == &*z.version - }) + .position(|z| x.id == *z.version) .unwrap_or_default() .cmp( &list .loader .iter() - .position(|z| { - y.id.split('-').next().unwrap_or_default() - == z.version - }) + .position(|z| y.id == z.version) .unwrap_or_default(), ) }) From 2fa8371bae0e91c79f2c64b237fabe50e42188f0 Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Mon, 21 Aug 2023 14:34:22 -0400 Subject: [PATCH 55/76] Neoforge support (#11) --- daedalus/src/modded.rs | 2 + daedalus_client/Cargo.toml | 1 + daedalus_client/src/main.rs | 13 + daedalus_client/src/neo.rs | 470 ++++++++++++++++++++++++++++++++++++ 4 files changed, 486 insertions(+) create mode 100644 daedalus_client/src/neo.rs diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 8f91798f..83c579ab 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -16,6 +16,8 @@ pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0; pub const CURRENT_FORGE_FORMAT_VERSION: usize = 0; /// The latest version of the format the quilt model structs deserialize to pub const CURRENT_QUILT_FORMAT_VERSION: usize = 0; +/// The latest version of the format the neoforge model structs deserialize to +pub const CURRENT_NEOFORGE_FORMAT_VERSION: usize = 0; /// The dummy replace string library names, inheritsFrom, and version names should be replaced with pub const DUMMY_REPLACE_STRING: &str = "${modrinth.gameVersion}"; diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index a0d74911..ca19f014 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -15,6 +15,7 @@ log = "0.4.17" env_logger= "0.10.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +serde-xml-rs = "0.6.0" lazy_static = "1.4.0" thiserror = "1.0" reqwest = "0.11.13" diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index db3639de..e706fa07 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -9,6 +9,7 @@ use tokio::sync::Semaphore; mod fabric; mod forge; mod minecraft; +mod neo; mod quilt; #[derive(thiserror::Error, Debug)] @@ -17,6 +18,8 @@ pub enum Error { DaedalusError(#[from] daedalus::Error), #[error("Error while deserializing JSON")] SerdeError(#[from] serde_json::Error), + #[error("Error while deserializing XML")] + XMLError(#[from] serde_xml_rs::Error), #[error("Unable to fetch {item}")] FetchError { inner: reqwest::Error, item: String }, #[error("Error while managing asynchronous tasks")] @@ -98,6 +101,16 @@ async fn main() { Ok(..) => {} Err(err) => error!("{:?}", err), }; + match neo::retrieve_data( + &manifest, + &mut uploaded_files, + semaphore.clone(), + ) + .await + { + Ok(..) => {} + Err(err) => error!("{:?}", err), + }; } } } diff --git a/daedalus_client/src/neo.rs b/daedalus_client/src/neo.rs new file mode 100644 index 00000000..172a5667 --- /dev/null +++ b/daedalus_client/src/neo.rs @@ -0,0 +1,470 @@ +use crate::{download_file, format_url, upload_file_to_bucket, Error}; +use daedalus::minecraft::{Library, VersionManifest}; +use daedalus::modded::{ + LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry, +}; +use log::info; +use semver::Version; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::io::Read; +use std::sync::Arc; +use std::time::Instant; +use tokio::sync::{Mutex, Semaphore}; + +pub async fn retrieve_data( + minecraft_versions: &VersionManifest, + uploaded_files: &mut Vec, + semaphore: Arc, +) -> Result<(), Error> { + let maven_metadata = fetch_maven_metadata(None, semaphore.clone()).await?; + let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( + "neo/v{}/manifest.json", + daedalus::modded::CURRENT_NEOFORGE_FORMAT_VERSION, + ))) + .await + .ok(); + + let old_versions = + Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { + old_manifest.game_versions + } else { + Vec::new() + })); + + let versions = Arc::new(Mutex::new(Vec::new())); + + let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); + let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); + + let mut version_futures = Vec::new(); + + for (minecraft_version, loader_versions) in maven_metadata.clone() { + let mut loaders = Vec::new(); + + for (full, loader_version) in loader_versions { + let version = Version::parse(&loader_version)?; + + loaders.push((full, version)) + } + + if !loaders.is_empty() { + version_futures.push(async { + let mut loaders_versions = Vec::new(); + + { + let loaders_futures = loaders.into_iter().map(|(loader_version_full, _)| async { + let versions_mutex = Arc::clone(&old_versions); + let visited_assets = Arc::clone(&visited_assets_mutex); + let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); + let semaphore = Arc::clone(&semaphore); + let minecraft_version = minecraft_version.clone(); + + async move { + { + let versions = versions_mutex.lock().await; + let version = versions.iter().find(|x| + x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full)); + + if let Some(version) = version { + return Ok::, Error>(Some(version.clone())); + } + } + + info!("Forge - Installer Start {}", loader_version_full.clone()); + let bytes = download_file(&format!("https://maven.neoforged.net/net/neoforged/forge/{0}/forge-{0}-installer.jar", loader_version_full), None, semaphore.clone()).await?; + + let reader = std::io::Cursor::new(bytes); + + if let Ok(archive) = zip::ZipArchive::new(reader) { + let mut archive_clone = archive.clone(); + let mut profile = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("install_profile.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + Ok::(serde_json::from_str::(&contents)?) + }).await??; + + let mut archive_clone = archive.clone(); + let version_info = tokio::task::spawn_blocking(move || { + let mut install_profile = archive_clone.by_name("version.json")?; + + let mut contents = String::new(); + install_profile.read_to_string(&mut contents)?; + + Ok::(serde_json::from_str::(&contents)?) + }).await??; + + + let mut libs : Vec = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library { + downloads: x.downloads, + extract: x.extract, + name: x.name, + url: x.url, + natives: x.natives, + rules: x.rules, + checksums: x.checksums, + include_in_classpath: false + })).collect(); + + let mut local_libs : HashMap = HashMap::new(); + + for lib in &libs { + if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) { + let mut archive_clone = archive.clone(); + let lib_name_clone = lib.name.clone(); + + let lib_bytes = tokio::task::spawn_blocking(move || { + let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?; + let mut lib_bytes = Vec::new(); + lib_file.read_to_end(&mut lib_bytes)?; + + Ok::(bytes::Bytes::from(lib_bytes)) + }).await??; + + local_libs.insert(lib.name.clone(), lib_bytes); + } + } + + let path = profile.path.clone(); + let version = profile.version.clone(); + + for entry in profile.data.values_mut() { + if entry.client.starts_with('/') || entry.server.starts_with('/') { + macro_rules! read_data { + ($value:expr) => { + let mut archive_clone = archive.clone(); + let value_clone = $value.clone(); + let lib_bytes = tokio::task::spawn_blocking(move || { + let mut lib_file = archive_clone.by_name(&value_clone[1..value_clone.len()])?; + let mut lib_bytes = Vec::new(); + lib_file.read_to_end(&mut lib_bytes)?; + + Ok::(bytes::Bytes::from(lib_bytes)) + }).await??; + + let split = $value.split('/').last(); + + if let Some(last) = split { + let mut file = last.split('.'); + + if let Some(file_name) = file.next() { + if let Some(ext) = file.next() { + let path = format!("{}:{}@{}", path.as_deref().unwrap_or(&*format!("net.minecraftforge:forge:{}", version)), file_name, ext); + $value = format!("[{}]", &path); + local_libs.insert(path.clone(), bytes::Bytes::from(lib_bytes)); + + libs.push(Library { + downloads: None, + extract: None, + name: path, + url: Some("".to_string()), + natives: None, + rules: None, + checksums: None, + include_in_classpath: false, + }); + } + } + } + } + } + + if entry.client.starts_with('/') { + read_data!(entry.client); + } + + if entry.server.starts_with('/') { + read_data!(entry.server); + } + } + } + + let now = Instant::now(); + let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { + let artifact_path = + daedalus::get_path_from_artifact(&lib.name)?; + + { + let mut visited_assets = visited_assets.lock().await; + + if visited_assets.contains(&lib.name) { + if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { + artifact.url = format_url(&format!("maven/{}", artifact_path)); + } + } else if lib.url.is_some() { + lib.url = Some(format_url("maven/")); + } + + return Ok::(lib); + } else { + visited_assets.push(lib.name.clone()) + } + } + + let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { + if let Some(ref mut artifact) = downloads.artifact { + let res = if artifact.url.is_empty() { + local_libs.get(&lib.name).cloned() + } else { + Some(download_file( + &artifact.url, + Some(&*artifact.sha1), + semaphore.clone(), + ) + .await?) + }; + + if res.is_some() { + artifact.url = format_url(&format!("maven/{}", artifact_path)); + } + + res + } else { None } + } else if let Some(ref mut url) = lib.url { + let res = if url.is_empty() { + local_libs.get(&lib.name).cloned() + } else { + Some(download_file( + url, + None, + semaphore.clone(), + ) + .await?) + }; + + if res.is_some() { + lib.url = Some(format_url("maven/")); + } + + res + } else { None }; + + if let Some(bytes) = artifact_bytes { + upload_file_to_bucket( + format!("{}/{}", "maven", artifact_path), + bytes.to_vec(), + Some("application/java-archive".to_string()), + uploaded_files_mutex.as_ref(), + semaphore.clone(), + ).await?; + } + + Ok::(lib) + })).await?; + + let elapsed = now.elapsed(); + info!("Elapsed lib DL: {:.2?}", elapsed); + + let new_profile = PartialVersionInfo { + id: version_info.id, + inherits_from: version_info.inherits_from, + release_time: version_info.release_time, + time: version_info.time, + main_class: version_info.main_class, + minecraft_arguments: version_info.minecraft_arguments, + arguments: version_info.arguments, + libraries: libs, + type_: version_info.type_, + data: Some(profile.data), + processors: Some(profile.processors), + }; + + let version_path = format!( + "neo/v{}/versions/{}.json", + daedalus::modded::CURRENT_NEOFORGE_FORMAT_VERSION, + new_profile.id + ); + + upload_file_to_bucket( + version_path.clone(), + serde_json::to_vec(&new_profile)?, + Some("application/json".to_string()), + uploaded_files_mutex.as_ref(), + semaphore.clone(), + ).await?; + + return Ok(Some(LoaderVersion { + id: loader_version_full, + url: format_url(&version_path), + stable: false + })); + } + + Ok(None) + }.await + }); + + { + let len = loaders_futures.len(); + let mut versions = loaders_futures.into_iter().peekable(); + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); + + let chunk: Vec<_> = versions.by_ref().take(1).collect(); + let res = futures::future::try_join_all(chunk).await?; + loaders_versions.extend(res.into_iter().flatten()); + + chunk_index += 1; + + let elapsed = now.elapsed(); + info!("Loader Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed); + } + } + } + + versions.lock().await.push(daedalus::modded::Version { + id: minecraft_version, + stable: true, + loaders: loaders_versions + }); + + Ok::<(), Error>(()) + }); + } + } + + { + let len = version_futures.len(); + let mut versions = version_futures.into_iter().peekable(); + let mut chunk_index = 0; + while versions.peek().is_some() { + let now = Instant::now(); + + let chunk: Vec<_> = versions.by_ref().take(1).collect(); + futures::future::try_join_all(chunk).await?; + + chunk_index += 1; + + let elapsed = now.elapsed(); + info!("Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed); + } + } + + if let Ok(versions) = Arc::try_unwrap(versions) { + let mut versions = versions.into_inner(); + + versions.sort_by(|x, y| { + minecraft_versions + .versions + .iter() + .position(|z| x.id == z.id) + .unwrap_or_default() + .cmp( + &minecraft_versions + .versions + .iter() + .position(|z| y.id == z.id) + .unwrap_or_default(), + ) + }); + + for version in &mut versions { + let loader_versions = maven_metadata.get(&version.id); + if let Some(loader_versions) = loader_versions { + version.loaders.sort_by(|x, y| { + loader_versions + .iter() + .position(|z| y.id == z.1) + .unwrap_or_default() + .cmp( + &loader_versions + .iter() + .position(|z| x.id == z.1) + .unwrap_or_default(), + ) + }); + version.loaders.reverse(); + } + } + + upload_file_to_bucket( + format!( + "neo/v{}/manifest.json", + daedalus::modded::CURRENT_NEOFORGE_FORMAT_VERSION, + ), + serde_json::to_vec(&Manifest { + game_versions: versions, + })?, + Some("application/json".to_string()), + uploaded_files_mutex.as_ref(), + semaphore, + ) + .await?; + } + + if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { + uploaded_files.extend(uploaded_files_mutex.into_inner()); + } + + Ok(()) +} + +const DEFAULT_MAVEN_METADATA_URL: &str = + "https://maven.neoforged.net/net/neoforged/forge/maven-metadata.xml"; + +#[derive(Debug, Deserialize)] +struct Metadata { + versioning: Versioning, +} + +#[derive(Debug, Deserialize)] +struct Versioning { + versions: Versions, +} + +#[derive(Debug, Deserialize)] +struct Versions { + version: Vec, +} + +pub async fn fetch_maven_metadata( + url: Option<&str>, + semaphore: Arc, +) -> Result>, Error> { + let values = serde_xml_rs::from_str::( + &String::from_utf8( + download_file( + url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), + None, + semaphore, + ) + .await? + .to_vec(), + ) + .unwrap_or_default(), + )?; + + let mut map: HashMap> = HashMap::new(); + + for value in values.versioning.versions.version { + let original = value.clone(); + + let parts: Vec<&str> = value.split('-').collect(); + if parts.len() == 2 { + map.entry(parts[0].to_string()) + .or_insert(Vec::new()) + .push((original, parts[1].to_string())); + } + } + + Ok(map) +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde(rename_all = "camelCase")] +struct ForgeInstallerProfileV2 { + pub spec: i32, + pub profile: String, + pub version: String, + pub json: String, + pub path: Option, + pub minecraft: String, + pub data: HashMap, + pub libraries: Vec, + pub processors: Vec, +} From c1d28381e8da98153f9a665ce4a32fcda524d729 Mon Sep 17 00:00:00 2001 From: Jai A Date: Mon, 21 Aug 2023 14:36:43 -0400 Subject: [PATCH 56/76] actually bump version --- daedalus/Cargo.toml | 2 +- daedalus_client/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index f00c0ae8..75262c27 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.23" +version = "0.1.25" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index ca19f014..8cd0fd2f 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.23" +version = "0.1.25" authors = ["Jai A "] edition = "2018" From 57b1932b5eb64a53f4947bcd8b97fef985e05b2a Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Tue, 12 Sep 2023 14:19:24 -0400 Subject: [PATCH 57/76] Fix minecraft meta links (#12) * Fix minecraft meta links * remove debug * bump v * no mut --- daedalus/Cargo.toml | 2 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/minecraft.rs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 75262c27..6aff4dd4 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.25" +version = "0.1.26" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 8cd0fd2f..87b3d417 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.25" +version = "0.1.26" authors = ["Jai A "] edition = "2018" diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 6badc9c4..e284c27c 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -25,7 +25,7 @@ pub async fn retrieve_data( let mut manifest = daedalus::minecraft::fetch_version_manifest(None).await?; - let cloned_manifest = Arc::new(Mutex::new(manifest.clone())); + let cloned_manifest = Arc::new(Mutex::new(old_manifest.clone().unwrap_or(manifest.clone()))); let patches = fetch_library_patches()?; let cloned_patches = Arc::new(&patches); From b1ca2cc2b6f558fea82396b1ad8ce954ca6e2d73 Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Sat, 21 Oct 2023 13:09:21 -0700 Subject: [PATCH 58/76] Fix quilt + fabric intermediary syncing issues (#13) * Fix quilt + fabric intermediary syncing issues * fix comp * Fix random versions not working/updating meta --- daedalus_client/library-patches.json | 544 ++++++++++++++++++++++----- daedalus_client/src/fabric.rs | 47 ++- daedalus_client/src/main.rs | 47 ++- daedalus_client/src/minecraft.rs | 3 +- daedalus_client/src/neo.rs | 2 +- daedalus_client/src/quilt.rs | 46 ++- docker-compose.yml | 17 + 7 files changed, 559 insertions(+), 147 deletions(-) create mode 100644 docker-compose.yml diff --git a/daedalus_client/library-patches.json b/daedalus_client/library-patches.json index 31a6a8cb..ebde4ed0 100644 --- a/daedalus_client/library-patches.json +++ b/daedalus_client/library-patches.json @@ -1,4 +1,48 @@ [ + { + "_comment": "Only allow osx-arm64 for existing LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-glfw-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-jemalloc-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-openal-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-opengl-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-stb-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-tinyfd-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-natives-macos-arm64:3.3.2" + ], + "override": { + "rules": [ + { + "action": "allow", + "os": { + "name": "osx-arm64" + } + } + ] + } + }, + { + "_comment": "Only allow windows-arm64 for existing LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-glfw-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-jemalloc-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-openal-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-opengl-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-stb-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-tinyfd-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-natives-windows-arm64:3.3.2" + ], + "override": { + "rules": [ + { + "action": "allow", + "os": { + "name": "windows-arm64" + } + } + ] + } + }, { "_comment": "Add missing tinyfd to the broken LWJGL 3.2.2 variant", "match": [ @@ -36,8 +80,7 @@ "natives-windows": { "sha1": "e9115958773644e863332a6a06488d26f9e1fc9f", "size": 208314, - "url": - "https://libraries.minecraft.net/org/lwjgl/lwjgl-tinyfd/3.2.2/lwjgl-tinyfd-3.2.2-natives-windows.jar" + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl-tinyfd/3.2.2/lwjgl-tinyfd-3.2.2-natives-windows.jar" } } }, @@ -62,8 +105,7 @@ "artifact": { "sha1": "369a83621e3c65496348491e533cb97fe5f2f37d", "size": 91947, - "url": - "https://github.com/MinecraftMachina/Java-Objective-C-Bridge/releases/download/1.1.0-mmachina.1/java-objc-bridge-1.1.jar" + "url": "https://github.com/MinecraftMachina/Java-Objective-C-Bridge/releases/download/1.1.0-mmachina.1/java-objc-bridge-1.1.jar" } }, "name": "ca.weblite:java-objc-bridge:1.1.0-mmachina.1", @@ -94,14 +136,12 @@ "natives-linux-arm64": { "sha1": "42b388ccb7c63cec4e9f24f4dddef33325f8b212", "size": 10932, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-2.9.4/jinput-platform-2.0.5-natives-linux.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-2.9.4/jinput-platform-2.0.5-natives-linux.jar" }, "natives-linux-arm32": { "sha1": "f3c455b71c5146acb5f8a9513247fc06db182fd5", "size": 4521, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-2.9.4/jinput-platform-2.0.5-natives-linux.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-2.9.4/jinput-platform-2.0.5-natives-linux.jar" } } }, @@ -125,9 +165,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -169,9 +207,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -198,8 +234,7 @@ "artifact": { "sha1": "697517568c68e78ae0b4544145af031c81082dfe", "size": 1047168, - "url": - "https://libraries.minecraft.net/org/lwjgl/lwjgl/lwjgl/2.9.4-nightly-20150209/lwjgl-2.9.4-nightly-20150209.jar" + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl/lwjgl/2.9.4-nightly-20150209/lwjgl-2.9.4-nightly-20150209.jar" } }, "name": "org.lwjgl.lwjgl:lwjgl:2.9.4-nightly-20150209", @@ -238,9 +273,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -267,8 +300,7 @@ "artifact": { "sha1": "d51a7c040a721d13efdfbd34f8b257b2df882ad0", "size": 173887, - "url": - "https://libraries.minecraft.net/org/lwjgl/lwjgl/lwjgl_util/2.9.4-nightly-20150209/lwjgl_util-2.9.4-nightly-20150209.jar" + "url": "https://libraries.minecraft.net/org/lwjgl/lwjgl/lwjgl_util/2.9.4-nightly-20150209/lwjgl_util-2.9.4-nightly-20150209.jar" } }, "name": "org.lwjgl.lwjgl:lwjgl_util:2.9.4-nightly-20150209", @@ -312,20 +344,17 @@ "natives-osx-arm64": { "sha1": "eff546c0b319d6ffc7a835652124c18089c67f36", "size": 488316, - "url": - "https://github.com/MinecraftMachina/lwjgl/releases/download/2.9.4-20150209-mmachina.2/lwjgl-platform-2.9.4-nightly-20150209-natives-osx.jar" + "url": "https://github.com/MinecraftMachina/lwjgl/releases/download/2.9.4-20150209-mmachina.2/lwjgl-platform-2.9.4-nightly-20150209-natives-osx.jar" }, "natives-linux-arm64": { "sha1": "63ac7da0f4a4785c7eadc0f8edc1e9dcc4dd08cb", "size": 579979, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-2.9.4/lwjgl-platform-2.9.4-nightly-20150209-natives-linux.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-2.9.4/lwjgl-platform-2.9.4-nightly-20150209-natives-linux.jar" }, "natives-linux-arm32": { "sha1": "fa483e540a9a753a5ffbb23dcf7879a5bf752611", "size": 475177, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-2.9.4/lwjgl-platform-2.9.4-nightly-20150209-natives-linux.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-2.9.4/lwjgl-platform-2.9.4-nightly-20150209-natives-linux.jar" } } }, @@ -346,9 +375,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -399,8 +426,7 @@ "natives-linux-arm64": { "sha1": "074ad243761147df0d060fbefc814614d2ff75cc", "size": 85072, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-glfw-natives-linux-arm64.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-glfw-natives-linux-arm64.jar" } } }, @@ -446,8 +472,7 @@ "natives-linux-arm32": { "sha1": "4265f2fbe3b9d642591165165a17cf406cf7b98e", "size": 80186, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-glfw-natives-linux-arm32.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-glfw-natives-linux-arm32.jar" } } }, @@ -493,8 +518,7 @@ "natives-osx-arm64": { "sha1": "71d793d0a5a42e3dfe78eb882abc2523a2c6b496", "size": 129076, - "url": - "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-glfw-natives-macos-arm64.jar" + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-glfw-natives-macos-arm64.jar" } } }, @@ -523,9 +547,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -576,8 +598,7 @@ "natives-linux-arm64": { "sha1": "762d7d80c9cdf3a3f3fc80c8a5f86612255edfe0", "size": 156343, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-jemalloc-patched-natives-linux-arm64.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-jemalloc-patched-natives-linux-arm64.jar" } } }, @@ -623,8 +644,7 @@ "natives-linux-arm32": { "sha1": "9163a2a5559ef87bc13ead8fea84417ea3928748", "size": 134237, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-jemalloc-natives-linux-arm32.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-jemalloc-natives-linux-arm32.jar" } } }, @@ -670,8 +690,7 @@ "natives-osx-arm64": { "sha1": "b0be721188d2e7195798780b1c5fe7eafe8091c1", "size": 103478, - "url": - "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-jemalloc-natives-macos-arm64.jar" + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-jemalloc-natives-macos-arm64.jar" } } }, @@ -700,9 +719,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -753,8 +770,7 @@ "natives-linux-arm64": { "sha1": "948e415b5b2a2c650c25b377a4a9f443b21ce92e", "size": 469432, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-openal-natives-linux-arm64.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-openal-natives-linux-arm64.jar" } } }, @@ -800,8 +816,7 @@ "natives-linux-arm32": { "sha1": "ecbc981fdd996492a1f6334f003ed62e5a8c0cd5", "size": 398418, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-openal-natives-linux-arm32.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-openal-natives-linux-arm32.jar" } } }, @@ -847,8 +862,7 @@ "natives-osx-arm64": { "sha1": "6b80fc0b982a0723b141e88859c42d6f71bd723f", "size": 346131, - "url": - "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-openal-natives-macos-arm64.jar" + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-openal-natives-macos-arm64.jar" } } }, @@ -877,9 +891,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -930,8 +942,7 @@ "natives-linux-arm64": { "sha1": "bd40897077bf7d12f562da898b18ac2c68e1f9d7", "size": 56109, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-opengl-natives-linux-arm64.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-opengl-natives-linux-arm64.jar" } } }, @@ -977,8 +988,7 @@ "natives-linux-arm32": { "sha1": "3af5599c74dd76dd8dbb567b3f9b4963a6abeed5", "size": 56388, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-opengl-natives-linux-arm32.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-opengl-natives-linux-arm32.jar" } } }, @@ -1024,8 +1034,7 @@ "natives-osx-arm64": { "sha1": "bb575058e0372f515587b5d2d04ff7db185f3ffe", "size": 41667, - "url": - "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-opengl-natives-macos-arm64.jar" + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-opengl-natives-macos-arm64.jar" } } }, @@ -1054,9 +1063,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -1107,8 +1114,7 @@ "natives-linux-arm64": { "sha1": "077efa7d7ea41b32df5c6078e912e724cccd06db", "size": 202038, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-stb-natives-linux-arm64.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-stb-natives-linux-arm64.jar" } } }, @@ -1154,8 +1160,7 @@ "natives-linux-arm32": { "sha1": "ec9d70aaebd0ff76dfeecf8f00b56118bf3706b1", "size": 149387, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-stb-natives-linux-arm32.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-stb-natives-linux-arm32.jar" } } }, @@ -1201,8 +1206,7 @@ "natives-osx-arm64": { "sha1": "98f0ad956c754723ef354d50057cc30417ef376a", "size": 178409, - "url": - "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-stb-natives-macos-arm64.jar" + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-stb-natives-macos-arm64.jar" } } }, @@ -1231,9 +1235,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -1284,8 +1286,7 @@ "natives-linux-arm64": { "sha1": "37c744ca289b5d7ae155d79e39029488b3254e5b", "size": 37893, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-tinyfd-natives-linux-arm64.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-tinyfd-natives-linux-arm64.jar" } } }, @@ -1331,8 +1332,7 @@ "natives-linux-arm32": { "sha1": "82d16054ada6633297a3108fb6d8bae98800c76f", "size": 41663, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-tinyfd-natives-linux-arm32.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-tinyfd-natives-linux-arm32.jar" } } }, @@ -1378,8 +1378,7 @@ "natives-osx-arm64": { "sha1": "015b931a2daba8f0c317d84c9d14e8e98ae56e0c", "size": 41384, - "url": - "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-tinyfd-natives-macos-arm64.jar" + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-tinyfd-natives-macos-arm64.jar" } } }, @@ -1408,9 +1407,7 @@ ], "override": { "rules": [ - { - "action": "allow" - }, + { "action": "disallow", "os": { @@ -1461,8 +1458,7 @@ "natives-linux-arm64": { "sha1": "612efd57d12b2e48e554858eb35e7e2eb46ebb4c", "size": 87121, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-natives-linux-arm64.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.2.2/lwjgl-natives-linux-arm64.jar" } } }, @@ -1508,8 +1504,7 @@ "natives-linux-arm32": { "sha1": "6bd0b37fef777a309936a72dc7f63126e8c79ea5", "size": 90296, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-natives-linux-arm32.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm32/raw/lwjgl-3.2.2/lwjgl-natives-linux-arm32.jar" } } }, @@ -1555,8 +1550,7 @@ "natives-osx-arm64": { "sha1": "984df31fadaab86838877b112e5b4e4f68a00ccf", "size": 42693, - "url": - "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-natives-macos-arm64.jar" + "url": "https://github.com/MinecraftMachina/lwjgl3/releases/download/3.3.1-mmachina.1/lwjgl-natives-macos-arm64.jar" } } }, @@ -1672,8 +1666,7 @@ "artifact": { "sha1": "749be48a9b86ee2c3a2da5fd77511208adcfb33b", "size": 159993, - "url": - "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.3.1/lwjgl-jemalloc-patched-natives-linux-arm64.jar" + "url": "https://github.com/theofficialgman/lwjgl3-binaries-arm64/raw/lwjgl-3.3.1/lwjgl-jemalloc-patched-natives-linux-arm64.jar" } }, "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm64:3.3.1-gman64.1", @@ -2000,6 +1993,370 @@ } ] }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-glfw:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "bc49e64bae0f7ff103a312ee8074a34c4eb034c7", + "size": 120168, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-glfw/lwjgl-glfw-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-jemalloc:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "5249f18a9ae20ea86c5816bc3107a888ce7a17d2", + "size": 206402, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-jemalloc/lwjgl-jemalloc-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-openal:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "22408980cc579709feaf9acb807992d3ebcf693f", + "size": 590865, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-openal/lwjgl-openal-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-opengl:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "bb9eb56da6d1d549d6a767218e675e36bc568eb9", + "size": 58627, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-opengl/lwjgl-opengl-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-stb:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "11a380c37b0f03cb46db235e064528f84d736ff7", + "size": 207419, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-stb/lwjgl-stb-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-tinyfd:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "93f8c5bc1984963cd79109891fb5a9d1e580373e", + "size": 43381, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-tinyfd/lwjgl-tinyfd-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "8bd89332c90a90e6bc4aa997a25c05b7db02c90a", + "size": 90795, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl/lwjgl-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-glfw:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "5907d9a6b7c44fb0612a63bb1cff5992588f65be", + "size": 110067, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-glfw/lwjgl-glfw-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-jemalloc:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "9367437ce192e4d6f5725d53d85520644c0b0d6f", + "size": 177571, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-jemalloc/lwjgl-jemalloc-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-openal:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "7c82bbc33ef49ee4094b216c940db564b2998224", + "size": 503352, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-openal/lwjgl-openal-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-opengl:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "821f9a2d1d583c44893f42b96f6977682b48a99b", + "size": 59265, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-opengl/lwjgl-opengl-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-stb:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "ca9333da184aade20757151f4615f1e27ca521ae", + "size": 154928, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-stb/lwjgl-stb-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-tinyfd:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "807e220913aa0740449ff90d3b3d825cf5f359ed", + "size": 48788, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-tinyfd/lwjgl-tinyfd-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "afcbfaaa46f217e98a6da4208550f71de1f2a225", + "size": 89347, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl/lwjgl-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, { "_comment": "Replace glfw from 3.3.1 with version from 3.3.2 to prevent stack smashing", "match": [ @@ -2016,5 +2373,4 @@ "name": "org.lwjgl:lwjgl-glfw-natives-linux:3.3.2-lwjgl.1" } } -] - +] \ No newline at end of file diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 94d2488f..c7e55fae 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -31,8 +31,25 @@ pub async fn retrieve_data( { let mut loaders = loaders_mutex.write().await; - for loader in &list.loader { - loaders.push((Box::new(loader.stable), loader.version.clone())) + for (index, loader) in list.loader.iter().enumerate() { + if versions.iter().any(|x| { + x.id == DUMMY_REPLACE_STRING + && x.loaders.iter().any(|x| x.id == loader.version) + }) { + if index == 0 { + loaders.push(( + Box::new(loader.stable), + loader.version.clone(), + Box::new(true), + )) + } + } else { + loaders.push(( + Box::new(loader.stable), + loader.version.clone(), + Box::new(false), + )) + } } list.loader @@ -46,16 +63,7 @@ pub async fn retrieve_data( let loader_versions = futures::future::try_join_all( loaders_mutex.read().await.clone().into_iter().map( - |(stable, loader)| async { - { - if versions.iter().any(|x| { - x.id == DUMMY_REPLACE_STRING - && x.loaders.iter().any(|x| x.id == loader) - }) { - return Ok(None); - } - } - + |(stable, loader, skip_upload)| async { let version = fetch_fabric_version( DUMMY_GAME_VERSION, &loader, @@ -63,8 +71,8 @@ pub async fn retrieve_data( ) .await?; - Ok::, String, PartialVersionInfo)>, Error>( - Some((stable, loader, version)), + Ok::<(Box, String, PartialVersionInfo, Box), Error>( + (stable, loader, version, skip_upload), ) }, ), @@ -73,8 +81,8 @@ pub async fn retrieve_data( let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); futures::future::try_join_all(loader_versions.into_iter() - .flatten().map( - |(stable, loader, version)| async { + .map( + |(stable, loader, version, skip_upload)| async { let libs = futures::future::try_join_all( version.libraries.into_iter().map(|mut lib| async { { @@ -167,6 +175,13 @@ pub async fn retrieve_data( ) .await?; + if async move { + *skip_upload + }.await { + return Ok::<(), Error>(()) + } + + let version_path = format!( "fabric/v{}/versions/{}.json", daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index e706fa07..af34ce3a 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -147,26 +147,35 @@ fn check_env_vars() -> bool { } lazy_static::lazy_static! { - static ref CLIENT : Bucket = Bucket::new( - &dotenvy::var("S3_BUCKET_NAME").unwrap(), - if &*dotenvy::var("S3_REGION").unwrap() == "r2" { - Region::R2 { - account_id: dotenvy::var("S3_URL").unwrap(), - } + static ref CLIENT : Bucket = { + let region = dotenvy::var("S3_REGION").unwrap(); + let b = Bucket::new( + &dotenvy::var("S3_BUCKET_NAME").unwrap(), + if &*region == "r2" { + Region::R2 { + account_id: dotenvy::var("S3_URL").unwrap(), + } + } else { + Region::Custom { + region: region.clone(), + endpoint: dotenvy::var("S3_URL").unwrap(), + } + }, + Credentials::new( + Some(&*dotenvy::var("S3_ACCESS_TOKEN").unwrap()), + Some(&*dotenvy::var("S3_SECRET").unwrap()), + None, + None, + None, + ).unwrap(), + ).unwrap(); + + if region == "path-style" { + b.with_path_style() } else { - Region::Custom { - region: dotenvy::var("S3_REGION").unwrap(), - endpoint: dotenvy::var("S3_URL").unwrap(), - } - }, - Credentials::new( - Some(&*dotenvy::var("S3_ACCESS_TOKEN").unwrap()), - Some(&*dotenvy::var("S3_SECRET").unwrap()), - None, - None, - None, - ).unwrap(), - ).unwrap(); + b + } + }; } pub async fn upload_file_to_bucket( diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index e284c27c..99ed51bc 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -25,7 +25,8 @@ pub async fn retrieve_data( let mut manifest = daedalus::minecraft::fetch_version_manifest(None).await?; - let cloned_manifest = Arc::new(Mutex::new(old_manifest.clone().unwrap_or(manifest.clone()))); + let cloned_manifest = + Arc::new(Mutex::new(old_manifest.clone().unwrap_or(manifest.clone()))); let patches = fetch_library_patches()?; let cloned_patches = Arc::new(&patches); diff --git a/daedalus_client/src/neo.rs b/daedalus_client/src/neo.rs index 172a5667..fd4c6bb4 100644 --- a/daedalus_client/src/neo.rs +++ b/daedalus_client/src/neo.rs @@ -447,7 +447,7 @@ pub async fn fetch_maven_metadata( let parts: Vec<&str> = value.split('-').collect(); if parts.len() == 2 { map.entry(parts[0].to_string()) - .or_insert(Vec::new()) + .or_default() .push((original, parts[1].to_string())); } } diff --git a/daedalus_client/src/quilt.rs b/daedalus_client/src/quilt.rs index 827ad00e..caa4a674 100644 --- a/daedalus_client/src/quilt.rs +++ b/daedalus_client/src/quilt.rs @@ -31,8 +31,25 @@ pub async fn retrieve_data( { let mut loaders = loaders_mutex.write().await; - for loader in &list.loader { - loaders.push((Box::new(false), loader.version.clone())) + for (index, loader) in list.loader.iter().enumerate() { + if versions.iter().any(|x| { + x.id == DUMMY_REPLACE_STRING + && x.loaders.iter().any(|x| x.id == loader.version) + }) { + if index == 0 { + loaders.push(( + Box::new(false), + loader.version.clone(), + Box::new(true), + )) + } + } else { + loaders.push(( + Box::new(false), + loader.version.clone(), + Box::new(false), + )) + } } list.loader @@ -46,16 +63,7 @@ pub async fn retrieve_data( let loader_versions = futures::future::try_join_all( loaders_mutex.read().await.clone().into_iter().map( - |(stable, loader)| async { - { - if versions.iter().any(|x| { - x.id == DUMMY_REPLACE_STRING - && x.loaders.iter().any(|x| x.id == loader) - }) { - return Ok(None); - } - } - + |(stable, loader, skip_upload)| async { let version = fetch_quilt_version( DUMMY_GAME_VERSION, &loader, @@ -63,8 +71,8 @@ pub async fn retrieve_data( ) .await?; - Ok::, String, PartialVersionInfo)>, Error>( - Some((stable, loader, version)), + Ok::<(Box, String, PartialVersionInfo, Box), Error>( + (stable, loader, version, skip_upload), ) }, ), @@ -73,8 +81,8 @@ pub async fn retrieve_data( let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); futures::future::try_join_all(loader_versions.into_iter() - .flatten().map( - |(stable, loader, version)| async { + .map( + |(stable, loader, version, skip_upload)| async { let libs = futures::future::try_join_all( version.libraries.into_iter().map(|mut lib| async { { @@ -167,6 +175,12 @@ pub async fn retrieve_data( ) .await?; + if async move { + *skip_upload + }.await { + return Ok::<(), Error>(()) + } + let version_path = format!( "quilt/v{}/versions/{}.json", daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..4e74e5ad --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,17 @@ +version: '3' + +services: + minio: + image: quay.io/minio/minio + volumes: + - minio-data:/data + ports: + - "9000:9000" + - "9001:9001" + environment: + MINIO_ROOT_USER: minioadmin + MINIO_ROOT_PASSWORD: miniosecret + command: server /data --console-address ":9001" + +volumes: + minio-data: \ No newline at end of file From 6fe1fa34556ce7e170f6e0cbeb71dd7500a58219 Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Wed, 8 Nov 2023 17:57:37 -0700 Subject: [PATCH 59/76] Fix crash when MC updates (#14) * Fix crash when MC updates * Fix fabric ordering + add neoforge support * run clippy * run clippy --- daedalus/src/modded.rs | 19 +++++++- daedalus_client/src/fabric.rs | 5 +-- daedalus_client/src/minecraft.rs | 39 +++++++++++++---- daedalus_client/src/neo.rs | 75 +++++++++++++++++++++----------- daedalus_client/src/quilt.rs | 5 +-- 5 files changed, 100 insertions(+), 43 deletions(-) diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index 83c579ab..d678c1e3 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -4,7 +4,7 @@ use crate::minecraft::{ Argument, ArgumentType, Library, VersionInfo, VersionType, }; use chrono::{DateTime, Utc}; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; use std::collections::HashMap; #[cfg(feature = "bincode")] @@ -32,6 +32,21 @@ pub struct SidedDataEntry { pub server: String, } +fn deserialize_date<'de, D>(deserializer: D) -> Result, D::Error> +where + D: Deserializer<'de>, +{ + let s = String::deserialize(deserializer)?; + + DateTime::parse_from_rfc3339(&s) + .map(|dt| dt.with_timezone(&Utc)) + .or_else(|_| { + DateTime::parse_from_str(&s, "%Y-%m-%dT%H:%M:%S%.f") + .map(|dt| dt.with_timezone(&Utc)) + }) + .map_err(serde::de::Error::custom) +} + #[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] @@ -43,9 +58,11 @@ pub struct PartialVersionInfo { pub inherits_from: String, /// The time that the version was released #[cfg_attr(feature = "bincode", bincode(with_serde))] + #[serde(deserialize_with = "deserialize_date")] pub release_time: DateTime, /// The latest time a file in this version was updated #[cfg_attr(feature = "bincode", bincode(with_serde))] + #[serde(deserialize_with = "deserialize_date")] pub time: DateTime, #[serde(skip_serializing_if = "Option::is_none")] /// The classpath to the main class to launch the game diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index c7e55fae..1a997da7 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -12,7 +12,7 @@ pub async fn retrieve_data( uploaded_files: &mut Vec, semaphore: Arc, ) -> Result<(), Error> { - let mut list = fetch_fabric_versions(None, semaphore.clone()).await?; + let list = fetch_fabric_versions(None, semaphore.clone()).await?; let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( "fabric/v{}/manifest.json", daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, @@ -51,9 +51,6 @@ pub async fn retrieve_data( )) } } - - list.loader - .retain(|x| loaders.iter().any(|val| val.1 == x.version)) } const DUMMY_GAME_VERSION: &str = "1.19.4-rc2"; diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 99ed51bc..1f0e5b7c 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -141,18 +141,39 @@ pub async fn retrieve_data( let mut cloned_manifest = cloned_manifest_mutex.lock().await; - let position = cloned_manifest + if let Some(position) = cloned_manifest .versions .iter() .position(|x| version.id == x.id) - .unwrap(); - cloned_manifest.versions[position].url = - format_url(&version_path); - cloned_manifest.versions[position].assets_index_sha1 = - Some(version_info.asset_index.sha1.clone()); - cloned_manifest.versions[position].assets_index_url = - Some(format_url(&assets_path)); - cloned_manifest.versions[position].sha1 = version_info_hash; + { + cloned_manifest.versions[position].url = + format_url(&version_path); + cloned_manifest.versions[position].assets_index_sha1 = + Some(version_info.asset_index.sha1.clone()); + cloned_manifest.versions[position].assets_index_url = + Some(format_url(&assets_path)); + cloned_manifest.versions[position].sha1 = + version_info_hash; + } else { + cloned_manifest.versions.insert( + 0, + daedalus::minecraft::Version { + id: version_info.id.clone(), + type_: version_info.type_.clone(), + url: format_url(&version_path), + time: version_info.time, + release_time: version_info.release_time, + sha1: version_info_hash, + compliance_level: 1, + assets_index_url: Some( + version_info.asset_index.sha1.clone(), + ), + assets_index_sha1: Some( + version_info.asset_index.sha1.clone(), + ), + }, + ) + } } let mut download_assets = false; diff --git a/daedalus_client/src/neo.rs b/daedalus_client/src/neo.rs index fd4c6bb4..ddc07564 100644 --- a/daedalus_client/src/neo.rs +++ b/daedalus_client/src/neo.rs @@ -17,7 +17,7 @@ pub async fn retrieve_data( uploaded_files: &mut Vec, semaphore: Arc, ) -> Result<(), Error> { - let maven_metadata = fetch_maven_metadata(None, semaphore.clone()).await?; + let maven_metadata = fetch_maven_metadata(semaphore.clone()).await?; let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( "neo/v{}/manifest.json", daedalus::modded::CURRENT_NEOFORGE_FORMAT_VERSION, @@ -42,10 +42,10 @@ pub async fn retrieve_data( for (minecraft_version, loader_versions) in maven_metadata.clone() { let mut loaders = Vec::new(); - for (full, loader_version) in loader_versions { + for (full, loader_version, new_forge) in loader_versions { let version = Version::parse(&loader_version)?; - loaders.push((full, version)) + loaders.push((full, version, new_forge.to_string())) } if !loaders.is_empty() { @@ -53,7 +53,7 @@ pub async fn retrieve_data( let mut loaders_versions = Vec::new(); { - let loaders_futures = loaders.into_iter().map(|(loader_version_full, _)| async { + let loaders_futures = loaders.into_iter().map(|(loader_version_full, _, new_forge)| async { let versions_mutex = Arc::clone(&old_versions); let visited_assets = Arc::clone(&visited_assets_mutex); let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); @@ -72,13 +72,13 @@ pub async fn retrieve_data( } info!("Forge - Installer Start {}", loader_version_full.clone()); - let bytes = download_file(&format!("https://maven.neoforged.net/net/neoforged/forge/{0}/forge-{0}-installer.jar", loader_version_full), None, semaphore.clone()).await?; + let bytes = download_file(&format!("https://maven.neoforged.net/net/neoforged/{1}/{0}/{1}-{0}-installer.jar", loader_version_full, if &*new_forge == "true" { "neoforge" } else { "forge" }), None, semaphore.clone()).await?; let reader = std::io::Cursor::new(bytes); if let Ok(archive) = zip::ZipArchive::new(reader) { let mut archive_clone = archive.clone(); - let mut profile = tokio::task::spawn_blocking(move || { + let mut profile = tokio::task::spawn_blocking(move || { let mut install_profile = archive_clone.by_name("install_profile.json")?; let mut contents = String::new(); @@ -404,8 +404,10 @@ pub async fn retrieve_data( Ok(()) } -const DEFAULT_MAVEN_METADATA_URL: &str = +const DEFAULT_MAVEN_METADATA_URL_1: &str = "https://maven.neoforged.net/net/neoforged/forge/maven-metadata.xml"; +const DEFAULT_MAVEN_METADATA_URL_2: &str = + "https://maven.neoforged.net/net/neoforged/neoforge/maven-metadata.xml"; #[derive(Debug, Deserialize)] struct Metadata { @@ -423,32 +425,55 @@ struct Versions { } pub async fn fetch_maven_metadata( - url: Option<&str>, semaphore: Arc, -) -> Result>, Error> { - let values = serde_xml_rs::from_str::( - &String::from_utf8( - download_file( - url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), - None, - semaphore, +) -> Result>, Error> { + async fn fetch_values( + url: &str, + semaphore: Arc, + ) -> Result { + Ok(serde_xml_rs::from_str( + &String::from_utf8( + download_file(url, None, semaphore).await?.to_vec(), ) - .await? - .to_vec(), - ) - .unwrap_or_default(), - )?; + .unwrap_or_default(), + )?) + } - let mut map: HashMap> = HashMap::new(); + let forge_values = + fetch_values(DEFAULT_MAVEN_METADATA_URL_1, semaphore.clone()).await?; + let neo_values = + fetch_values(DEFAULT_MAVEN_METADATA_URL_2, semaphore).await?; - for value in values.versioning.versions.version { + let mut map: HashMap> = HashMap::new(); + + for value in forge_values.versioning.versions.version { let original = value.clone(); let parts: Vec<&str> = value.split('-').collect(); if parts.len() == 2 { - map.entry(parts[0].to_string()) - .or_default() - .push((original, parts[1].to_string())); + map.entry(parts[0].to_string()).or_default().push(( + original, + parts[1].to_string(), + false, + )); + } + } + + for value in neo_values.versioning.versions.version { + let original = value.clone(); + + let mut parts = value.split('.'); + + if let Some(major) = parts.next() { + if let Some(minor) = parts.next() { + let game_version = format!("1.{}.{}", major, minor); + + map.entry(game_version.clone()).or_default().push(( + original.clone(), + format!("{}-{}", game_version, original), + true, + )); + } } } diff --git a/daedalus_client/src/quilt.rs b/daedalus_client/src/quilt.rs index caa4a674..cd0df2da 100644 --- a/daedalus_client/src/quilt.rs +++ b/daedalus_client/src/quilt.rs @@ -12,7 +12,7 @@ pub async fn retrieve_data( uploaded_files: &mut Vec, semaphore: Arc, ) -> Result<(), Error> { - let mut list = fetch_quilt_versions(None, semaphore.clone()).await?; + let list = fetch_quilt_versions(None, semaphore.clone()).await?; let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( "quilt/v{}/manifest.json", daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, @@ -51,9 +51,6 @@ pub async fn retrieve_data( )) } } - - list.loader - .retain(|x| loaders.iter().any(|val| val.1 == x.version)) } const DUMMY_GAME_VERSION: &str = "1.19.4-rc2"; From ac07ac52348a6257470d7ed182cb65ab2b16a1a5 Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Wed, 15 Nov 2023 16:35:33 -0700 Subject: [PATCH 60/76] Fix date parsing for partial versions (#15) --- daedalus/Cargo.toml | 2 +- daedalus/src/modded.rs | 10 +++------- daedalus_client/Cargo.toml | 2 +- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 6aff4dd4..5c20d6ab 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.1.26" +version = "0.1.27" authors = ["Jai A "] edition = "2018" license = "MIT" diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index d678c1e3..e72eafea 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -3,7 +3,7 @@ use crate::{download_file, Error}; use crate::minecraft::{ Argument, ArgumentType, Library, VersionInfo, VersionType, }; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, TimeZone, Utc}; use serde::{Deserialize, Deserializer, Serialize}; use std::collections::HashMap; @@ -38,12 +38,8 @@ where { let s = String::deserialize(deserializer)?; - DateTime::parse_from_rfc3339(&s) - .map(|dt| dt.with_timezone(&Utc)) - .or_else(|_| { - DateTime::parse_from_str(&s, "%Y-%m-%dT%H:%M:%S%.f") - .map(|dt| dt.with_timezone(&Utc)) - }) + serde_json::from_str::>(&format!("\"{s}\"")) + .or_else(|_| Utc.datetime_from_str(&s, "%Y-%m-%dT%H:%M:%S%.9f")) .map_err(serde::de::Error::custom) } diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 87b3d417..47f8aa9a 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.1.26" +version = "0.1.27" authors = ["Jai A "] edition = "2018" From 8b16cd1b363b525c17bc9e93e4393389c3677492 Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Tue, 25 Jun 2024 15:47:27 -0700 Subject: [PATCH 61/76] Daedalus Rewrite + Code Cleanup (#16) * [wip] rewrite daedalus, vanilla, fabric, and quilt * finish forge + neo * fix docker * fix neoforge 1.21+ * update concurrency limit * finish * remove mac garb --- .env | 16 +- .github/workflows/run.yml | 49 + .gitignore | 1 + .idea/daedalus.iml | 1 + Dockerfile | 2 +- daedalus/Cargo.toml | 10 +- daedalus/src/lib.rs | 121 --- daedalus/src/minecraft.rs | 75 +- daedalus/src/modded.rs | 38 +- daedalus_client/Cargo.toml | 25 +- daedalus_client/library-patches.json | 543 +++++++++- daedalus_client/src/error.rs | 63 ++ daedalus_client/src/fabric.rs | 574 +++++------ daedalus_client/src/forge.rs | 1366 ++++++++++++++------------ daedalus_client/src/main.rs | 331 +++---- daedalus_client/src/minecraft.rs | 414 ++++---- daedalus_client/src/neo.rs | 495 ---------- daedalus_client/src/quilt.rs | 370 ------- daedalus_client/src/util.rs | 369 +++++++ 19 files changed, 2334 insertions(+), 2529 deletions(-) create mode 100644 .github/workflows/run.yml create mode 100644 daedalus_client/src/error.rs delete mode 100644 daedalus_client/src/neo.rs delete mode 100644 daedalus_client/src/quilt.rs create mode 100644 daedalus_client/src/util.rs diff --git a/.env b/.env index bbcfb5cb..9aac220a 100644 --- a/.env +++ b/.env @@ -1,9 +1,15 @@ -RUST_LOG=info +RUST_LOG=warn,daedalus_client=trace -BASE_URL=https://modrinth-cdn-staging.nyc3.digitaloceanspaces.com +BASE_URL=http://localhost:9000/meta + +CONCURRENCY_LIMIT=10 S3_ACCESS_TOKEN=none S3_SECRET=none -S3_URL=none -S3_REGION=none -S3_BUCKET_NAME=none +S3_URL=http://localhost:9000 +S3_REGION=path-style +S3_BUCKET_NAME=meta + +CLOUDFLARE_INTEGRATION=false +CLOUDFLARE_TOKEN=none +CLOUDFLARE_ZONE_ID=none \ No newline at end of file diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml new file mode 100644 index 00000000..b97d2bdd --- /dev/null +++ b/.github/workflows/run.yml @@ -0,0 +1,49 @@ +name: Run Meta + +on: + schedule: + - cron: '*/5 * * * *' + +jobs: + run-docker: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + + - name: Pull Docker image from GHCR + run: docker pull ghcr.io/modrinth/daedalus:latest + + - name: Run Docker container + env: + BASE_URL: ${{ secrets.BASE_URL }} + S3_ACCESS_TOKEN: ${{ secrets.S3_ACCESS_TOKEN }} + S3_SECRET: ${{ secrets.S3_SECRET }} + S3_URL: ${{ secrets.S3_URL }} + S3_REGION: ${{ secrets.S3_REGION }} + S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }} + CLOUDFLARE_INTEGRATION: ${{ secrets.CLOUDFLARE_INTEGRATION }} + CLOUDFLARE_TOKEN: ${{ secrets.CLOUDFLARE_TOKEN }} + CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }} + run: | + docker run -d \ + --name daedalus \ + -e BASE_URL=$BASE_URL \ + -e S3_ACCESS_TOKEN=$S3_ACCESS_TOKEN \ + -e S3_SECRET=$S3_SECRET \ + -e S3_URL=$S3_URL \ + -e S3_REGION=$S3_REGION \ + -e S3_BUCKET_NAME=$S3_BUCKET_NAME \ + -e CLOUDFLARE_INTEGRATION=$CLOUDFLARE_INTEGRATION \ + -e CLOUDFLARE_TOKEN=$CLOUDFLARE_TOKEN \ + -e CLOUDFLARE_ZONE_ID=$CLOUDFLARE_ZONE_ID \ + ghcr.io/modrinth/daedalus:latest \ No newline at end of file diff --git a/.gitignore b/.gitignore index dae6eff4..06b153ac 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ ### Intellij ### # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 +caches/ # User-specific stuff .idea/**/workspace.xml diff --git a/.idea/daedalus.iml b/.idea/daedalus.iml index ec7bb013..cc8620ac 100644 --- a/.idea/daedalus.iml +++ b/.idea/daedalus.iml @@ -6,6 +6,7 @@ + diff --git a/Dockerfile b/Dockerfile index a2a341da..c015f674 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.68.2 as build +FROM rust:1.79.0 as build ENV PKG_CONFIG_ALLOW_CROSS=1 WORKDIR /usr/src/daedalus diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 5c20d6ab..1acc5253 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "daedalus" -version = "0.1.27" -authors = ["Jai A "] -edition = "2018" +version = "0.2.0" +authors = ["Jai A "] +edition = "2021" license = "MIT" description = "Utilities for querying and parsing Minecraft metadata" repository = "https://github.com/modrinth/daedalus/" @@ -14,12 +14,8 @@ readme = "README.md" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -reqwest = { version = "0.11", features = ["json"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" chrono = { version = "0.4", features = ["serde"] } bytes = "1" thiserror = "1.0" -tokio = { version = "1", features = ["full"] } -sha1 = { version = "0.6.1", features = ["std"]} -bincode = {version = "2.0.0-rc.2", features = ["serde"], optional = true} diff --git a/daedalus/src/lib.rs b/daedalus/src/lib.rs index 51221eed..79008889 100644 --- a/daedalus/src/lib.rs +++ b/daedalus/src/lib.rs @@ -12,30 +12,6 @@ pub mod modded; #[derive(thiserror::Error, Debug)] /// An error type representing possible errors when fetching metadata pub enum Error { - #[error("Failed to validate file checksum at url {url} with hash {hash} after {tries} tries")] - /// A checksum was failed to validate for a file - ChecksumFailure { - /// The checksum's hash - hash: String, - /// The URL of the file attempted to be downloaded - url: String, - /// The amount of tries that the file was downloaded until failure - tries: u32, - }, - /// There was an error while deserializing metadata - #[error("Error while deserializing JSON")] - SerdeError(#[from] serde_json::Error), - /// There was a network error when fetching an object - #[error("Unable to fetch {item}")] - FetchError { - /// The internal reqwest error - inner: reqwest::Error, - /// The item that was failed to be fetched - item: String, - }, - /// There was an error when managing async tasks - #[error("Error while managing asynchronous tasks")] - TaskError(#[from] tokio::task::JoinError), /// Error while parsing input #[error("{0}")] ParseError(String), @@ -124,100 +100,3 @@ pub fn get_path_from_artifact(artifact: &str) -> Result { )) } } - -/// Downloads a file from specified mirrors -pub async fn download_file_mirrors( - base: &str, - mirrors: &[&str], - sha1: Option<&str>, -) -> Result { - if mirrors.is_empty() { - return Err(Error::ParseError("No mirrors provided!".to_string())); - } - - for (index, mirror) in mirrors.iter().enumerate() { - let result = download_file(&format!("{}{}", mirror, base), sha1).await; - - if result.is_ok() || (result.is_err() && index == (mirrors.len() - 1)) { - return result; - } - } - - unreachable!() -} - -/// Downloads a file with retry and checksum functionality -pub async fn download_file( - url: &str, - sha1: Option<&str>, -) -> Result { - let mut headers = reqwest::header::HeaderMap::new(); - if let Ok(header) = reqwest::header::HeaderValue::from_str(&format!( - "modrinth/daedalus/{} (support@modrinth.com)", - env!("CARGO_PKG_VERSION") - )) { - headers.insert(reqwest::header::USER_AGENT, header); - } - let client = reqwest::Client::builder() - .tcp_keepalive(Some(std::time::Duration::from_secs(10))) - .timeout(std::time::Duration::from_secs(15)) - .default_headers(headers) - .build() - .map_err(|err| Error::FetchError { - inner: err, - item: url.to_string(), - })?; - - for attempt in 1..=4 { - let result = client.get(url).send().await; - - match result { - Ok(x) => { - let bytes = x.bytes().await; - - if let Ok(bytes) = bytes { - if let Some(sha1) = sha1 { - if &*get_hash(bytes.clone()).await? != sha1 { - if attempt <= 3 { - continue; - } else { - return Err(Error::ChecksumFailure { - hash: sha1.to_string(), - url: url.to_string(), - tries: attempt, - }); - } - } - } - - return Ok(bytes); - } else if attempt <= 3 { - continue; - } else if let Err(err) = bytes { - return Err(Error::FetchError { - inner: err, - item: url.to_string(), - }); - } - } - Err(_) if attempt <= 3 => continue, - Err(err) => { - return Err(Error::FetchError { - inner: err, - item: url.to_string(), - }) - } - } - } - - unreachable!() -} - -/// Computes a checksum of the input bytes -pub async fn get_hash(bytes: bytes::Bytes) -> Result { - let hash = - tokio::task::spawn_blocking(|| sha1::Sha1::from(bytes).hexdigest()) - .await?; - - Ok(hash) -} diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 747fa8a0..021c91ab 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -1,16 +1,11 @@ use crate::modded::{Processor, SidedDataEntry}; -use crate::{download_file, Error}; use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; -#[cfg(feature = "bincode")] -use bincode::{Decode, Encode}; - /// The latest version of the format the model structs deserialize to pub const CURRENT_FORMAT_VERSION: usize = 0; -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "snake_case")] /// The version type @@ -37,7 +32,6 @@ impl VersionType { } } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] /// A game version of Minecraft @@ -50,26 +44,18 @@ pub struct Version { /// A link to additional information about the version pub url: String, /// The latest time a file in this version was updated - #[cfg_attr(feature = "bincode", bincode(with_serde))] pub time: DateTime, /// The time this version was released - #[cfg_attr(feature = "bincode", bincode(with_serde))] pub release_time: DateTime, /// The SHA1 hash of the additional information about the version pub sha1: String, /// Whether the version supports the latest player safety features pub compliance_level: u32, #[serde(skip_serializing_if = "Option::is_none")] - /// (Modrinth Provided) The link to the assets index for this version - /// This is only available when using the Modrinth mirror - pub assets_index_url: Option, - #[serde(skip_serializing_if = "Option::is_none")] - /// (Modrinth Provided) The SHA1 hash of the assets index for this version - /// This is only available when using the Modrinth mirror - pub assets_index_sha1: Option, + /// (Modrinth Provided) The SHA1 hash of the original unmodified Minecraft versions JSON + pub original_sha1: Option, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// The latest snapshot and release of the game pub struct LatestVersion { @@ -79,7 +65,6 @@ pub struct LatestVersion { pub snapshot: String, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// Data of all game versions of Minecraft pub struct VersionManifest { @@ -93,16 +78,6 @@ pub struct VersionManifest { pub const VERSION_MANIFEST_URL: &str = "https://piston-meta.mojang.com/mc/game/version_manifest_v2.json"; -/// Fetches a version manifest from the specified URL. If no URL is specified, the default is used. -pub async fn fetch_version_manifest( - url: Option<&str>, -) -> Result { - Ok(serde_json::from_slice( - &download_file(url.unwrap_or(VERSION_MANIFEST_URL), None).await?, - )?) -} - -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// Information about the assets of the game @@ -119,7 +94,6 @@ pub struct AssetIndex { pub url: String, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)] #[serde(rename_all = "snake_case")] /// The type of download @@ -136,7 +110,6 @@ pub enum DownloadType { WindowsServer, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// Download information of a file pub struct Download { @@ -148,7 +121,6 @@ pub struct Download { pub url: String, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// Download information of a library pub struct LibraryDownload { @@ -163,7 +135,6 @@ pub struct LibraryDownload { pub url: String, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A list of files that should be downloaded for libraries pub struct LibraryDownloads { @@ -176,7 +147,6 @@ pub struct LibraryDownloads { pub classifiers: Option>, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "snake_case")] /// The action a rule can follow @@ -187,7 +157,6 @@ pub enum RuleAction { Disallow, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone)] #[serde(rename_all = "kebab-case")] /// An enum representing the different types of operating systems @@ -210,7 +179,6 @@ pub enum Os { Unknown, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A rule which depends on what OS the user is on pub struct OsRule { @@ -225,7 +193,6 @@ pub struct OsRule { pub arch: Option, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A rule which depends on the toggled features of the launcher pub struct FeatureRule { @@ -248,7 +215,6 @@ pub struct FeatureRule { pub is_quick_play_realms: Option, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A rule deciding whether a file is downloaded, an argument is used, etc. pub struct Rule { @@ -262,7 +228,6 @@ pub struct Rule { pub features: Option, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// Information delegating the extraction of the library pub struct LibraryExtract { @@ -271,7 +236,6 @@ pub struct LibraryExtract { pub exclude: Option>, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// Information about the java version the game needs @@ -282,7 +246,6 @@ pub struct JavaVersion { pub major_version: u32, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A library which the game relies on to run pub struct Library { @@ -309,6 +272,9 @@ pub struct Library { #[serde(default = "default_include_in_classpath")] /// Whether the library should be included in the classpath at the game's launch pub include_in_classpath: bool, + #[serde(default = "default_downloadable")] + /// Whether the library should be downloaded + pub downloadable: bool, } #[derive(Deserialize, Debug, Clone)] @@ -397,8 +363,10 @@ pub fn merge_partial_library( fn default_include_in_classpath() -> bool { true } +fn default_downloadable() -> bool { + true +} -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(untagged)] /// A container for an argument or multiple arguments @@ -409,7 +377,6 @@ pub enum ArgumentValue { Many(Vec), } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(untagged)] /// A command line argument passed to a program @@ -425,7 +392,6 @@ pub enum Argument { }, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash, Clone, Copy)] #[serde(rename_all = "snake_case")] /// The type of argument @@ -436,7 +402,6 @@ pub enum ArgumentType { Jvm, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// Information about a version @@ -481,16 +446,6 @@ pub struct VersionInfo { pub processors: Option>, } -/// Fetches detailed information about a version from the manifest -pub async fn fetch_version_info( - version: &Version, -) -> Result { - Ok(serde_json::from_slice( - &download_file(&version.url, Some(&version.sha1)).await?, - )?) -} - -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// An asset of the game pub struct Asset { @@ -500,23 +455,9 @@ pub struct Asset { pub size: u32, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] /// An index containing all assets the game needs pub struct AssetsIndex { /// A hashmap containing the filename (key) and asset (value) pub objects: HashMap, } - -/// Fetches the assets index from the version info -pub async fn fetch_assets_index( - version: &VersionInfo, -) -> Result { - Ok(serde_json::from_slice( - &download_file( - &version.asset_index.url, - Some(&version.asset_index.sha1), - ) - .await?, - )?) -} diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index e72eafea..c510fe78 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -1,5 +1,3 @@ -use crate::{download_file, Error}; - use crate::minecraft::{ Argument, ArgumentType, Library, VersionInfo, VersionType, }; @@ -7,9 +5,6 @@ use chrono::{DateTime, TimeZone, Utc}; use serde::{Deserialize, Deserializer, Serialize}; use std::collections::HashMap; -#[cfg(feature = "bincode")] -use bincode::{Decode, Encode}; - /// The latest version of the format the fabric model structs deserialize to pub const CURRENT_FABRIC_FORMAT_VERSION: usize = 0; /// The latest version of the format the fabric model structs deserialize to @@ -23,7 +18,6 @@ pub const CURRENT_NEOFORGE_FORMAT_VERSION: usize = 0; pub const DUMMY_REPLACE_STRING: &str = "${modrinth.gameVersion}"; /// A data variable entry that depends on the side of the installation -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] pub struct SidedDataEntry { /// The value on the client @@ -43,7 +37,6 @@ where .map_err(serde::de::Error::custom) } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// A partial version returned by fabric meta @@ -53,11 +46,9 @@ pub struct PartialVersionInfo { /// The version ID this partial version inherits from pub inherits_from: String, /// The time that the version was released - #[cfg_attr(feature = "bincode", bincode(with_serde))] #[serde(deserialize_with = "deserialize_date")] pub release_time: DateTime, /// The latest time a file in this version was updated - #[cfg_attr(feature = "bincode", bincode(with_serde))] #[serde(deserialize_with = "deserialize_date")] pub time: DateTime, #[serde(skip_serializing_if = "Option::is_none")] @@ -83,7 +74,6 @@ pub struct PartialVersionInfo { } /// A processor to be ran after downloading the files -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug)] pub struct Processor { /// Maven coordinates for the JAR library of this processor. @@ -101,13 +91,6 @@ pub struct Processor { pub sides: Option>, } -/// Fetches the version manifest of a game version's URL -pub async fn fetch_partial_version( - url: &str, -) -> Result { - Ok(serde_json::from_slice(&download_file(url, None).await?)?) -} - /// Merges a partial version into a complete one pub fn merge_partial_version( partial: PartialVersionInfo, @@ -154,15 +137,10 @@ pub fn merge_partial_version( .libraries .into_iter() .chain(merge.libraries) - .map(|x| Library { - downloads: x.downloads, - extract: x.extract, - name: x.name.replace(DUMMY_REPLACE_STRING, &merge_id), - url: x.url, - natives: x.natives, - rules: x.rules, - checksums: x.checksums, - include_in_classpath: x.include_in_classpath, + .map(|mut x| { + x.name = x.name.replace(DUMMY_REPLACE_STRING, &merge_id); + + x }) .collect::>(), main_class: if let Some(main_class) = partial.main_class { @@ -180,7 +158,6 @@ pub fn merge_partial_version( } } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] /// A manifest containing information about a mod loader's versions @@ -189,7 +166,6 @@ pub struct Manifest { pub game_versions: Vec, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A game version of Minecraft pub struct Version { @@ -201,7 +177,6 @@ pub struct Version { pub loaders: Vec, } -#[cfg_attr(feature = "bincode", derive(Encode, Decode))] #[derive(Serialize, Deserialize, Debug, Clone)] /// A version of a Minecraft mod loader pub struct LoaderVersion { @@ -212,8 +187,3 @@ pub struct LoaderVersion { /// Whether the loader is stable or not pub stable: bool, } - -/// Fetches the manifest of a mod loader -pub async fn fetch_manifest(url: &str) -> Result { - Ok(serde_json::from_slice(&download_file(url, None).await?)?) -} diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index 47f8aa9a..ce8204fa 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,8 +1,8 @@ [package] name = "daedalus_client" -version = "0.1.27" -authors = ["Jai A "] -edition = "2018" +version = "0.2.0" +authors = ["Jai A "] +edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html @@ -11,16 +11,23 @@ daedalus = { path = "../daedalus" } tokio = { version = "1", features = ["full"] } futures = "0.3.25" dotenvy = "0.15.6" -log = "0.4.17" -env_logger= "0.10.0" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" serde-xml-rs = "0.6.0" lazy_static = "1.4.0" thiserror = "1.0" -reqwest = "0.11.13" -zip = "0.6.3" +reqwest = { version = "0.12.5", features = ["stream", "json"] } +async_zip = { version = "0.0.17", features = ["full"] } semver = "1.0" chrono = { version = "0.4", features = ["serde"] } -bytes = "1.3.0" -rust-s3 = "0.33.0" +bytes = "1.6.0" +rust-s3 = "0.34.0" +dashmap = "5.5.3" +sha1_smol = { version = "1.0.0", features = ["std"] } +indexmap = { version = "2.2.6", features = ["serde"]} +itertools = "0.13.0" +tracing-error = "0.2.0" + +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-futures = { version = "0.2.5", features = ["futures", "tokio"] } \ No newline at end of file diff --git a/daedalus_client/library-patches.json b/daedalus_client/library-patches.json index ebde4ed0..268341a9 100644 --- a/daedalus_client/library-patches.json +++ b/daedalus_client/library-patches.json @@ -1,14 +1,23 @@ [ { - "_comment": "Only allow osx-arm64 for existing LWJGL 3.3.2", + "_comment": "Only allow osx-arm64 for existing LWJGL 3.3.2/3.3.3", "match": [ + "org.lwjgl:lwjgl-freetype-natives-macos-arm64:3.3.2", "org.lwjgl:lwjgl-glfw-natives-macos-arm64:3.3.2", "org.lwjgl:lwjgl-jemalloc-natives-macos-arm64:3.3.2", "org.lwjgl:lwjgl-openal-natives-macos-arm64:3.3.2", "org.lwjgl:lwjgl-opengl-natives-macos-arm64:3.3.2", "org.lwjgl:lwjgl-stb-natives-macos-arm64:3.3.2", "org.lwjgl:lwjgl-tinyfd-natives-macos-arm64:3.3.2", - "org.lwjgl:lwjgl-natives-macos-arm64:3.3.2" + "org.lwjgl:lwjgl-natives-macos-arm64:3.3.2", + "org.lwjgl:lwjgl-freetype-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-glfw-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-jemalloc-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-openal-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-opengl-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-stb-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-tinyfd-natives-macos-arm64:3.3.3", + "org.lwjgl:lwjgl-natives-macos-arm64:3.3.3" ], "override": { "rules": [ @@ -22,15 +31,24 @@ } }, { - "_comment": "Only allow windows-arm64 for existing LWJGL 3.3.2", + "_comment": "Only allow windows-arm64 for existing LWJGL 3.3.2/3.3.3", "match": [ + "org.lwjgl:lwjgl-freetype-natives-windows-arm64:3.3.2", "org.lwjgl:lwjgl-glfw-natives-windows-arm64:3.3.2", "org.lwjgl:lwjgl-jemalloc-natives-windows-arm64:3.3.2", "org.lwjgl:lwjgl-openal-natives-windows-arm64:3.3.2", "org.lwjgl:lwjgl-opengl-natives-windows-arm64:3.3.2", "org.lwjgl:lwjgl-stb-natives-windows-arm64:3.3.2", "org.lwjgl:lwjgl-tinyfd-natives-windows-arm64:3.3.2", - "org.lwjgl:lwjgl-natives-windows-arm64:3.3.2" + "org.lwjgl:lwjgl-natives-windows-arm64:3.3.2", + "org.lwjgl:lwjgl-freetype-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-glfw-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-jemalloc-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-openal-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-opengl-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-stb-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-tinyfd-natives-windows-arm64:3.3.3", + "org.lwjgl:lwjgl-natives-windows-arm64:3.3.3" ], "override": { "rules": [ @@ -165,7 +183,9 @@ ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -202,12 +222,13 @@ "org.lwjgl.lwjgl:lwjgl:2.9.1-nightly-20131120", "org.lwjgl.lwjgl:lwjgl:2.9.1-nightly-20131017", "org.lwjgl.lwjgl:lwjgl:2.9.1-nightly-20130708-debug3", - "org.lwjgl.lwjgl:lwjgl:2.9.1", - "org.lwjgl.lwjgl:lwjgl:2.9.0" + "org.lwjgl.lwjgl:lwjgl:2.9.1" ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -268,12 +289,13 @@ "org.lwjgl.lwjgl:lwjgl_util:2.9.1-nightly-20131120", "org.lwjgl.lwjgl:lwjgl_util:2.9.1-nightly-20131017", "org.lwjgl.lwjgl:lwjgl_util:2.9.1-nightly-20130708-debug3", - "org.lwjgl.lwjgl:lwjgl_util:2.9.1", - "org.lwjgl.lwjgl:lwjgl_util:2.9.0" + "org.lwjgl.lwjgl:lwjgl_util:2.9.1" ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -335,8 +357,7 @@ "org.lwjgl.lwjgl:lwjgl-platform:2.9.1-nightly-20131120", "org.lwjgl.lwjgl:lwjgl-platform:2.9.1-nightly-20131017", "org.lwjgl.lwjgl:lwjgl-platform:2.9.1-nightly-20130708-debug3", - "org.lwjgl.lwjgl:lwjgl-platform:2.9.1", - "org.lwjgl.lwjgl:lwjgl-platform:2.9.0" + "org.lwjgl.lwjgl:lwjgl-platform:2.9.1" ], "override": { "downloads": { @@ -375,7 +396,9 @@ ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -547,7 +570,9 @@ ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -719,7 +744,9 @@ ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -891,7 +918,9 @@ ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -1063,7 +1092,9 @@ ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -1235,7 +1266,9 @@ ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -1407,7 +1440,9 @@ ], "override": { "rules": [ - + { + "action": "allow" + }, { "action": "disallow", "os": { @@ -1993,6 +2028,32 @@ } ] }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-freetype:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "896e7d9b8f60d7273f3d491c69270afc67ece3ce", + "size": 1073374, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-freetype/lwjgl-freetype-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-freetype-natives-linux-arm64:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, { "_comment": "Add linux-arm64 support for LWJGL 3.3.2", "match": [ @@ -2175,6 +2236,32 @@ } ] }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.2", + "match": [ + "org.lwjgl:lwjgl-freetype:3.3.2" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "b7f77ceb951182659fd400437272aa7e96709968", + "size": 924657, + "url": "https://build.lwjgl.org/release/3.3.2/bin/lwjgl-freetype/lwjgl-freetype-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-freetype-natives-linux-arm32:3.3.2-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, { "_comment": "Add linux-arm32 support for LWJGL 3.3.2", "match": [ @@ -2357,6 +2444,422 @@ } ] }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-freetype:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "498965aac06c4a0d42df1fbef6bacd05bde7f974", + "size": 1093516, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-freetype/lwjgl-freetype-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-freetype-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-glfw:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "492a0f11f85b85899a6568f07511160c1b87cd38", + "size": 122159, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-glfw/lwjgl-glfw-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-jemalloc:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "eff8b86798191192fe2cba2dc2776109f30c239d", + "size": 209315, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-jemalloc/lwjgl-jemalloc-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-openal:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "ad8f302118a65bb8d615f8a2a680db58fb8f835e", + "size": 592963, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-openal/lwjgl-openal-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-opengl:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "2096f6b94b2d68745d858fbfe53aacf5f0c8074c", + "size": 58625, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-opengl/lwjgl-opengl-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-stb:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "ddc177afc2be1ee8d93684b11363b80589a13fe1", + "size": 207418, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-stb/lwjgl-stb-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-tinyfd:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "2823a8c955c758d0954d282888075019ef99cec7", + "size": 43864, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-tinyfd/lwjgl-tinyfd-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm64 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "f35d8b6ffe1ac1e3a5eb1d4e33de80f044ad5fd8", + "size": 91294, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl/lwjgl-natives-linux-arm64.jar" + } + }, + "name": "org.lwjgl:lwjgl-natives-linux-arm64:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm64" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-freetype:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "7dd3b1f751571adaf2c4dc882bc675a5d1e796e6", + "size": 942636, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-freetype/lwjgl-freetype-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-freetype-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-glfw:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "d9af485c32545b37dd5359b163161d42d7534dcf", + "size": 112560, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-glfw/lwjgl-glfw-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-glfw-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-jemalloc:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "109b6931880d02d4e65ced38928a16e41d19873e", + "size": 178324, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-jemalloc/lwjgl-jemalloc-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-jemalloc-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-openal:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "e1702aa09d20359d6cf5cb2999fa7685a785eca7", + "size": 505618, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-openal/lwjgl-openal-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-openal-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-opengl:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "dbba17fc5ac0985d14a57c11f9537617d67b9952", + "size": 59263, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-opengl/lwjgl-opengl-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-opengl-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-stb:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "1ae28ff044699ff29b0e980ffabd73fba8a664b3", + "size": 154931, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-stb/lwjgl-stb-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-stb-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl-tinyfd:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "c2a0a05c82c4b9f69ded0b6ad5f417addea78ce2", + "size": 49495, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl-tinyfd/lwjgl-tinyfd-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-tinyfd-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, + { + "_comment": "Add linux-arm32 support for LWJGL 3.3.3", + "match": [ + "org.lwjgl:lwjgl:3.3.3" + ], + "additionalLibraries": [ + { + "downloads": { + "artifact": { + "sha1": "2075c51a80f0ef0f22ba616ba54007ac2b0debd4", + "size": 89565, + "url": "https://build.lwjgl.org/release/3.3.3/bin/lwjgl/lwjgl-natives-linux-arm32.jar" + } + }, + "name": "org.lwjgl:lwjgl-natives-linux-arm32:3.3.3-lwjgl.1", + "rules": [ + { + "action": "allow", + "os": { + "name": "linux-arm32" + } + } + ] + } + ] + }, { "_comment": "Replace glfw from 3.3.1 with version from 3.3.2 to prevent stack smashing", "match": [ diff --git a/daedalus_client/src/error.rs b/daedalus_client/src/error.rs new file mode 100644 index 00000000..d4a0167c --- /dev/null +++ b/daedalus_client/src/error.rs @@ -0,0 +1,63 @@ +use tracing_error::InstrumentError; + +#[derive(thiserror::Error, Debug)] +pub enum ErrorKind { + #[error("Daedalus Error: {0}")] + Daedalus(#[from] daedalus::Error), + #[error("Invalid input: {0}")] + InvalidInput(String), + #[error("Error while managing asynchronous tasks")] + TaskError(#[from] tokio::task::JoinError), + #[error("Error while deserializing JSON: {0}")] + SerdeJSON(#[from] serde_json::Error), + #[error("Error while deserializing XML: {0}")] + SerdeXML(#[from] serde_xml_rs::Error), + #[error("Failed to validate file checksum at url {url} with hash {hash} after {tries} tries")] + ChecksumFailure { + hash: String, + url: String, + tries: u32, + }, + #[error("Unable to fetch {item}")] + Fetch { inner: reqwest::Error, item: String }, + #[error("Error while uploading file to S3: {file}")] + S3 { + inner: s3::error::S3Error, + file: String, + }, + #[error("Error acquiring semaphore: {0}")] + Acquire(#[from] tokio::sync::AcquireError), + #[error("Tracing error: {0}")] + Tracing(#[from] tracing::subscriber::SetGlobalDefaultError), + #[error("Zip error: {0}")] + Zip(#[from] async_zip::error::ZipError), +} + +#[derive(Debug)] +pub struct Error { + pub source: tracing_error::TracedError, +} + +impl std::fmt::Display for Error { + fn fmt(&self, fmt: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(fmt, "{}", self.source) + } +} + +impl> From for Error { + fn from(source: E) -> Self { + let error = Into::::into(source); + + Self { + source: error.in_current_span(), + } + } +} + +impl ErrorKind { + pub fn as_error(self) -> Error { + self.into() + } +} + +pub type Result = core::result::Result; diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 1a997da7..8d1d9431 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -1,372 +1,276 @@ -use crate::{download_file, format_url, upload_file_to_bucket, Error}; -use daedalus::minecraft::{Library, VersionManifest}; -use daedalus::modded::{ - LoaderVersion, Manifest, PartialVersionInfo, Version, DUMMY_REPLACE_STRING, -}; -use serde::{Deserialize, Serialize}; +use crate::util::{download_file, fetch_json, format_url}; +use crate::{insert_mirrored_artifact, Error, MirrorArtifact, UploadFile}; +use daedalus::modded::{Manifest, PartialVersionInfo, DUMMY_REPLACE_STRING}; +use dashmap::DashMap; +use serde::Deserialize; use std::sync::Arc; -use tokio::sync::{Mutex, RwLock, Semaphore}; +use tokio::sync::Semaphore; -pub async fn retrieve_data( - minecraft_versions: &VersionManifest, - uploaded_files: &mut Vec, +#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))] +pub async fn fetch_fabric( semaphore: Arc, + upload_files: &DashMap, + mirror_artifacts: &DashMap, ) -> Result<(), Error> { - let list = fetch_fabric_versions(None, semaphore.clone()).await?; - let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( - "fabric/v{}/manifest.json", + fetch( daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, - ))) + "fabric", + "https://meta.fabricmc.net/v2", + "https://maven.fabricmc.net/", + semaphore, + upload_files, + mirror_artifacts, + ) + .await +} + +#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))] +pub async fn fetch_quilt( + semaphore: Arc, + upload_files: &DashMap, + mirror_artifacts: &DashMap, +) -> Result<(), Error> { + fetch( + daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, + "quilt", + "https://meta.quiltmc.org/v3", + "https://meta.quiltmc.org/", + semaphore, + upload_files, + mirror_artifacts, + ) + .await +} + +#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))] +async fn fetch( + format_version: usize, + mod_loader: &str, + meta_url: &str, + maven_url: &str, + semaphore: Arc, + upload_files: &DashMap, + mirror_artifacts: &DashMap, +) -> Result<(), Error> { + let modrinth_manifest = fetch_json::( + &format_url(&format!("{mod_loader}/v{format_version}/manifest.json",)), + &semaphore, + ) .await .ok(); - - let mut versions = if let Some(old_manifest) = old_manifest { - old_manifest.game_versions - } else { - Vec::new() - }; - - let loaders_mutex = RwLock::new(Vec::new()); - - { - let mut loaders = loaders_mutex.write().await; - - for (index, loader) in list.loader.iter().enumerate() { - if versions.iter().any(|x| { - x.id == DUMMY_REPLACE_STRING - && x.loaders.iter().any(|x| x.id == loader.version) - }) { - if index == 0 { - loaders.push(( - Box::new(loader.stable), - loader.version.clone(), - Box::new(true), - )) - } - } else { - loaders.push(( - Box::new(loader.stable), - loader.version.clone(), - Box::new(false), - )) - } - } - } - - const DUMMY_GAME_VERSION: &str = "1.19.4-rc2"; - - let loader_version_mutex = Mutex::new(Vec::new()); - let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); - - let loader_versions = futures::future::try_join_all( - loaders_mutex.read().await.clone().into_iter().map( - |(stable, loader, skip_upload)| async { - let version = fetch_fabric_version( - DUMMY_GAME_VERSION, - &loader, - semaphore.clone(), - ) - .await?; - - Ok::<(Box, String, PartialVersionInfo, Box), Error>( - (stable, loader, version, skip_upload), - ) - }, - ), + let fabric_manifest = fetch_json::( + &format!("{meta_url}/versions"), + &semaphore, ) .await?; - let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); - futures::future::try_join_all(loader_versions.into_iter() - .map( - |(stable, loader, version, skip_upload)| async { - let libs = futures::future::try_join_all( - version.libraries.into_iter().map(|mut lib| async { - { - let mut visited_assets = - visited_artifacts_mutex.lock().await; + // We check Modrinth's fabric version manifest and compare if the fabric version exists in Modrinth's database + // We also check intermediary versions that are newly added to query + let (fetch_fabric_versions, fetch_intermediary_versions) = + if let Some(modrinth_manifest) = modrinth_manifest { + let (mut fetch_versions, mut fetch_intermediary_versions) = + (Vec::new(), Vec::new()); - if visited_assets.contains(&lib.name) { - lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); - lib.url = Some(format_url("maven/")); + for version in &fabric_manifest.loader { + if !modrinth_manifest + .game_versions + .iter() + .any(|x| x.loaders.iter().any(|x| x.id == version.version)) + { + fetch_versions.push(version); + } + } - return Ok(lib); - } else { - visited_assets.push(lib.name.clone()) - } + for version in &fabric_manifest.intermediary { + if !modrinth_manifest + .game_versions + .iter() + .any(|x| x.id == version.version) + && fabric_manifest + .game + .iter() + .any(|x| x.version == version.version) + { + fetch_intermediary_versions.push(version); + } + } + + (fetch_versions, fetch_intermediary_versions) + } else { + ( + fabric_manifest.loader.iter().collect(), + fabric_manifest.intermediary.iter().collect(), + ) + }; + + const DUMMY_GAME_VERSION: &str = "1.21"; + + if !fetch_intermediary_versions.is_empty() { + for x in &fetch_intermediary_versions { + insert_mirrored_artifact( + &x.maven, + maven_url.to_string(), + mirror_artifacts, + )?; + } + } + + if !fetch_fabric_versions.is_empty() { + let fabric_version_manifest_urls = fetch_fabric_versions + .iter() + .map(|x| { + format!( + "{}/versions/loader/{}/{}/profile/json", + meta_url, DUMMY_GAME_VERSION, x.version + ) + }) + .collect::>(); + let fabric_version_manifests = futures::future::try_join_all( + fabric_version_manifest_urls + .iter() + .map(|x| download_file(x, None, &semaphore)), + ) + .await? + .into_iter() + .map(|x| serde_json::from_slice(&x)) + .collect::, serde_json::Error>>()?; + + let patched_version_manifests = fabric_version_manifests + .into_iter() + .map(|mut version_info| { + for lib in &mut version_info.libraries { + let new_name = lib + .name + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); + // If a library is not intermediary, we add it to mirror artifacts to be mirrored + if lib.name == new_name { + insert_mirrored_artifact( + &new_name, + lib.url + .clone() + .unwrap_or_else(|| maven_url.to_string()), + mirror_artifacts, + )?; + } else { + lib.name = new_name; } - if lib.name.contains(DUMMY_GAME_VERSION) { - lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); - futures::future::try_join_all(list.game.clone().into_iter().map(|game_version| async { - let semaphore = semaphore.clone(); - let uploaded_files_mutex = uploaded_files_mutex.clone(); - let lib_name = lib.name.clone(); - let lib_url = lib.url.clone(); - - async move { - let artifact_path = - daedalus::get_path_from_artifact(&lib_name.replace(DUMMY_REPLACE_STRING, &game_version.version))?; - - let artifact = download_file( - &format!( - "{}{}", - lib_url.unwrap_or_else(|| { - "https://maven.fabricmc.net/".to_string() - }), - artifact_path - ), - None, - semaphore.clone(), - ) - .await?; - - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - &uploaded_files_mutex, - semaphore.clone(), - ) - .await?; - - Ok::<(), Error>(()) - }.await?; - - Ok::<(), Error>(()) - })).await?; - lib.url = Some(format_url("maven/")); - - return Ok(lib); - } - - let artifact_path = - daedalus::get_path_from_artifact(&lib.name)?; - - let artifact = download_file( - &format!( - "{}{}", - lib.url.unwrap_or_else(|| { - "https://maven.fabricmc.net/".to_string() - }), - artifact_path - ), - None, - semaphore.clone(), - ) - .await?; - lib.url = Some(format_url("maven/")); - - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - &uploaded_files_mutex, - semaphore.clone(), - ) - .await?; - - Ok::(lib) - }), - ) - .await?; - - if async move { - *skip_upload - }.await { - return Ok::<(), Error>(()) - } - - - let version_path = format!( - "fabric/v{}/versions/{}.json", - daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, - &loader - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&PartialVersionInfo { - arguments: version.arguments, - id: version - .id - .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), - main_class: version.main_class, - release_time: version.release_time, - time: version.time, - type_: version.type_, - inherits_from: version - .inherits_from - .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), - libraries: libs, - minecraft_arguments: version.minecraft_arguments, - processors: None, - data: None, - })?, - Some("application/json".to_string()), - &uploaded_files_mutex, - semaphore.clone(), - ) - .await?; - - { - let mut loader_version_map = loader_version_mutex.lock().await; - async move { - loader_version_map.push(LoaderVersion { - id: loader.to_string(), - url: format_url(&version_path), - stable: *stable, - }); } - .await; - } - Ok::<(), Error>(()) - }, - )) - .await?; + version_info.id = version_info + .id + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); + version_info.inherits_from = version_info + .inherits_from + .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); - let mut loader_version_mutex = loader_version_mutex.into_inner(); - if !loader_version_mutex.is_empty() { - if let Some(version) = - versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) - { - version.loaders.append(&mut loader_version_mutex); - } else { - versions.push(Version { - id: DUMMY_REPLACE_STRING.to_string(), - stable: true, - loaders: loader_version_mutex, - }); - } - } - - for version in &list.game { - if !versions.iter().any(|x| x.id == version.version) { - versions.push(Version { - id: version.version.clone(), - stable: version.stable, - loaders: vec![], - }); - } - } - - versions.sort_by(|x, y| { - minecraft_versions - .versions + Ok(version_info) + }) + .collect::, Error>>()?; + let serialized_version_manifests = patched_version_manifests .iter() - .position(|z| x.id == z.id) - .unwrap_or_default() - .cmp( - &minecraft_versions - .versions - .iter() - .position(|z| y.id == z.id) - .unwrap_or_default(), - ) - }); + .map(|x| serde_json::to_vec(x).map(bytes::Bytes::from)) + .collect::, serde_json::Error>>()?; - for version in &mut versions { - version.loaders.sort_by(|x, y| { - list.loader - .iter() - .position(|z| x.id == *z.version) - .unwrap_or_default() - .cmp( - &list - .loader - .iter() - .position(|z| y.id == z.version) - .unwrap_or_default(), - ) - }) + serialized_version_manifests + .into_iter() + .enumerate() + .for_each(|(index, bytes)| { + let loader = fetch_fabric_versions[index]; + + let version_path = format!( + "{mod_loader}/v{format_version}/versions/{}.json", + loader.version + ); + + upload_files.insert( + version_path, + UploadFile { + file: bytes, + content_type: Some("application/json".to_string()), + }, + ); + }); } - upload_file_to_bucket( - format!( - "fabric/v{}/manifest.json", - daedalus::modded::CURRENT_FABRIC_FORMAT_VERSION, - ), - serde_json::to_vec(&Manifest { - game_versions: versions, - })?, - Some("application/json".to_string()), - &uploaded_files_mutex, - semaphore, - ) - .await?; + if !fetch_fabric_versions.is_empty() + || !fetch_intermediary_versions.is_empty() + { + let fabric_manifest_path = + format!("{mod_loader}/v{format_version}/manifest.json",); - if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { - uploaded_files.extend(uploaded_files_mutex.into_inner()); + let loader_versions = daedalus::modded::Version { + id: DUMMY_REPLACE_STRING.to_string(), + stable: true, + loaders: fabric_manifest + .loader + .into_iter() + .map(|x| { + let version_path = format!( + "{mod_loader}/v{format_version}/versions/{}.json", + x.version, + ); + + daedalus::modded::LoaderVersion { + id: x.version, + url: format_url(&version_path), + stable: x.stable, + } + }) + .collect(), + }; + + let manifest = daedalus::modded::Manifest { + game_versions: std::iter::once(loader_versions) + .chain(fabric_manifest.game.into_iter().map(|x| { + daedalus::modded::Version { + id: x.version, + stable: x.stable, + loaders: vec![], + } + })) + .collect(), + }; + + upload_files.insert( + fabric_manifest_path, + UploadFile { + file: bytes::Bytes::from(serde_json::to_vec(&manifest)?), + content_type: Some("application/json".to_string()), + }, + ); } Ok(()) } -const FABRIC_META_URL: &str = "https://meta.fabricmc.net/v2"; - -async fn fetch_fabric_version( - version_number: &str, - loader_version: &str, - semaphore: Arc, -) -> Result { - Ok(serde_json::from_slice( - &download_file( - &format!( - "{}/versions/loader/{}/{}/profile/json", - FABRIC_META_URL, version_number, loader_version - ), - None, - semaphore, - ) - .await?, - )?) -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -/// Versions of fabric components +#[derive(Deserialize, Debug, Clone)] struct FabricVersions { - /// Versions of Minecraft that fabric supports - pub game: Vec, - /// Available versions of the fabric loader pub loader: Vec, + pub game: Vec, + #[serde(alias = "hashed")] + pub intermediary: Vec, } -#[derive(Serialize, Deserialize, Debug, Clone)] -/// A version of Minecraft that fabric supports -struct FabricGameVersion { - /// The version number of the game - pub version: String, - /// Whether the Minecraft version is stable or not - pub stable: bool, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -/// A version of the fabric loader +#[derive(Deserialize, Debug, Clone)] struct FabricLoaderVersion { - /// The separator to get the build number - pub separator: String, - /// The build number - pub build: u32, - /// The maven artifact - pub maven: String, - /// The version number of the fabric loader + // pub separator: String, + // pub build: u32, + // pub maven: String, pub version: String, - /// Whether the loader is stable or not + #[serde(default)] pub stable: bool, } -/// Fetches the list of fabric versions -async fn fetch_fabric_versions( - url: Option<&str>, - semaphore: Arc, -) -> Result { - Ok(serde_json::from_slice( - &download_file( - url.unwrap_or(&*format!("{}/versions", FABRIC_META_URL)), - None, - semaphore, - ) - .await?, - )?) + +#[derive(Deserialize, Debug, Clone)] +struct FabricIntermediaryVersion { + pub maven: String, + pub version: String, +} + +#[derive(Deserialize, Debug, Clone)] +struct FabricGameVersion { + pub version: String, + pub stable: bool, } diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index fc3bf2f7..99892f22 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -1,650 +1,766 @@ -use crate::{ - download_file, download_file_mirrors, format_url, upload_file_to_bucket, - Error, -}; +use crate::util::{download_file, fetch_json, fetch_xml, format_url}; +use crate::{insert_mirrored_artifact, Error, MirrorArtifact, UploadFile}; use chrono::{DateTime, Utc}; -use daedalus::minecraft::{ - Argument, ArgumentType, Library, VersionManifest, VersionType, -}; -use daedalus::modded::{ - LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry, -}; -use lazy_static::lazy_static; -use log::info; -use semver::{Version, VersionReq}; -use serde::{Deserialize, Serialize}; +use daedalus::get_path_from_artifact; +use daedalus::modded::PartialVersionInfo; +use dashmap::DashMap; +use futures::io::Cursor; +use indexmap::IndexMap; +use itertools::Itertools; +use serde::de::DeserializeOwned; +use serde::Deserialize; use std::collections::HashMap; -use std::io::Read; use std::sync::Arc; -use std::time::Instant; -use tokio::sync::{Mutex, Semaphore}; +use tokio::sync::Semaphore; -lazy_static! { - static ref FORGE_MANIFEST_V1_QUERY: VersionReq = - VersionReq::parse(">=8.0.684, <23.5.2851").unwrap(); - static ref FORGE_MANIFEST_V2_QUERY_P1: VersionReq = - VersionReq::parse(">=23.5.2851, <31.2.52").unwrap(); - static ref FORGE_MANIFEST_V2_QUERY_P2: VersionReq = - VersionReq::parse(">=32.0.1, <37.0.0").unwrap(); - static ref FORGE_MANIFEST_V3_QUERY: VersionReq = - VersionReq::parse(">=37.0.0").unwrap(); +#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))] +pub async fn fetch_forge( + semaphore: Arc, + upload_files: &DashMap, + mirror_artifacts: &DashMap, +) -> Result<(), Error> { + let forge_manifest = fetch_json::>>( + "https://files.minecraftforge.net/net/minecraftforge/forge/maven-metadata.json", + &semaphore, + ) + .await?; + + let mut format_version = 0; + + let forge_versions = forge_manifest.into_iter().flat_map(|(game_version, versions)| versions.into_iter().map(|loader_version| { + // Forge versions can be in these specific formats: + // 1.10.2-12.18.1.2016-failtests + // 1.9-12.16.0.1886 + // 1.9-12.16.0.1880-1.9 + // 1.14.4-28.1.30 + // This parses them to get the actual Forge version. Ex: 1.15.2-31.1.87 -> 31.1.87 + let version_split = loader_version.split('-').nth(1).unwrap_or(&loader_version).to_string(); + + // Forge has 3 installer formats: + // - Format 0 (Unsupported ATM): Forge Legacy (pre-1.5.2). Uses Binary Patch method to install + // To install: Download patch, download minecraft client JAR. Combine patch and client JAR and delete META-INF/. + // (pre-1.3-2) Client URL: https://maven.minecraftforge.net/net/minecraftforge/forge/{version}/forge-{version}-client.zip + // (pre-1.3-2) Server URL: https://maven.minecraftforge.net/net/minecraftforge/forge/{version}/forge-{version}-server.zip + // (1.3-2-onwards) Universal URL: https://maven.minecraftforge.net/net/minecraftforge/forge/{version}/forge-{version}-universal.zip + // - Format 1: Forge Installer Legacy (1.5.2-1.12.2ish) + // To install: Extract install_profile.json from archive. "versionInfo" is the profile's version info. Convert it to the modern format + // Extract forge library from archive. Path is at "install"."path". + // - Format 2: Forge Installer Modern + // To install: Extract install_profile.json from archive. Extract version.json from archive. Combine the two and extract all libraries + // which are embedded into the installer JAR. + // Then upload. The launcher will need to run processors! + if format_version != 1 && &*version_split == "7.8.0.684" { + format_version = 1; + } else if format_version != 2 && &*version_split == "14.23.5.2851" { + format_version = 2; + } + + ForgeVersion { + format_version, + installer_url: format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version), + raw: loader_version, + loader_version: version_split, + game_version: game_version.clone(), + } + }) + .collect::>()) + // TODO: support format version 0 (see above) + .filter(|x| x.format_version != 0) + .filter(|x| { + // These following Forge versions are broken and cannot be installed + const BLACKLIST : &[&str] = &[ + // Not supported due to `data` field being `[]` even though the type is a map + "1.12.2-14.23.5.2851", + // Malformed Archives + "1.6.1-8.9.0.749", + "1.6.1-8.9.0.751", + "1.6.4-9.11.1.960", + "1.6.4-9.11.1.961", + "1.6.4-9.11.1.963", + "1.6.4-9.11.1.964", + ]; + + !BLACKLIST.contains(&&*x.raw) + }) + .collect::>(); + + fetch( + daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, + "forge", + "https://maven.minecraftforge.net/", + forge_versions, + semaphore, + upload_files, + mirror_artifacts, + ) + .await } -pub async fn retrieve_data( - minecraft_versions: &VersionManifest, - uploaded_files: &mut Vec, +#[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))] +pub async fn fetch_neo( semaphore: Arc, + upload_files: &DashMap, + mirror_artifacts: &DashMap, ) -> Result<(), Error> { - let maven_metadata = fetch_maven_metadata(None, semaphore.clone()).await?; - let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( - "forge/v{}/manifest.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - ))) + #[derive(Debug, Deserialize)] + struct Metadata { + versioning: Versioning, + } + + #[derive(Debug, Deserialize)] + struct Versioning { + versions: Versions, + } + + #[derive(Debug, Deserialize)] + struct Versions { + version: Vec, + } + + let forge_versions = fetch_xml::( + "https://maven.neoforged.net/net/neoforged/forge/maven-metadata.xml", + &semaphore, + ) + .await?; + let neo_versions = fetch_xml::( + "https://maven.neoforged.net/net/neoforged/neoforge/maven-metadata.xml", + &semaphore, + ) + .await?; + + let parsed_versions = forge_versions.versioning.versions.version.into_iter().map(|loader_version| { + // NeoForge Forge versions can be in these specific formats: + // 1.20.1-47.1.74 + // 47.1.82 + // This parses them to get the actual Forge version. Ex: 1.20.1-47.1.74 -> 47.1.74 + let version_split = loader_version.split('-').nth(1).unwrap_or(&loader_version).to_string(); + + Ok(ForgeVersion { + format_version: 2, + installer_url: format!("https://maven.neoforged.net/net/neoforged/forge/{0}/forge-{0}-installer.jar", loader_version), + raw: loader_version, + loader_version: version_split, + game_version: "1.20.1".to_string(), // All NeoForge Forge versions are for 1.20.1 + }) + }).chain(neo_versions.versioning.versions.version.into_iter().map(|loader_version| { + let mut parts = loader_version.split('.'); + + // NeoForge Forge versions are in this format: 20.2.29-beta, 20.6.119 + // Where the first number is the major MC version, the second is the minor MC version, and the third is the NeoForge version + let major = parts.next().ok_or_else( + || crate::ErrorKind::InvalidInput(format!("Unable to find major game version for NeoForge {loader_version}")) + )?; + + let minor = parts.next().ok_or_else( + || crate::ErrorKind::InvalidInput(format!("Unable to find minor game version for NeoForge {loader_version}")) + )?; + + let game_version = if minor == "0" { + format!("1.{major}") + } else { + format!("1.{major}.{minor}") + }; + + Ok(ForgeVersion { + format_version: 2, + installer_url: format!("https://maven.neoforged.net/net/neoforged/neoforge/{0}/neoforge-{0}-installer.jar", loader_version), + loader_version: loader_version.clone(), + raw: loader_version, + game_version, + }) + })) + .collect::, Error>>()? + .into_iter() + .filter(|x| { + // These following Forge versions are broken and cannot be installed + const BLACKLIST : &[&str] = &[ + // Unreachable / 404 + "1.20.1-47.1.7", + "47.1.82", + ]; + + !BLACKLIST.contains(&&*x.raw) + }).collect(); + + fetch( + daedalus::modded::CURRENT_NEOFORGE_FORMAT_VERSION, + "neo", + "https://maven.neoforged.net/", + parsed_versions, + semaphore, + upload_files, + mirror_artifacts, + ) + .await +} + +#[tracing::instrument(skip( + forge_versions, + semaphore, + upload_files, + mirror_artifacts +))] +async fn fetch( + format_version: usize, + mod_loader: &str, + maven_url: &str, + forge_versions: Vec, + semaphore: Arc, + upload_files: &DashMap, + mirror_artifacts: &DashMap, +) -> Result<(), Error> { + let modrinth_manifest = fetch_json::( + &format_url(&format!("{mod_loader}/v{format_version}/manifest.json",)), + &semaphore, + ) .await .ok(); - let old_versions = - Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { - old_manifest.game_versions - } else { - Vec::new() - })); + let fetch_versions = if let Some(modrinth_manifest) = modrinth_manifest { + let mut fetch_versions = Vec::new(); - let versions = Arc::new(Mutex::new(Vec::new())); - - let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); - let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); - - let mut version_futures = Vec::new(); - - for (minecraft_version, loader_versions) in maven_metadata.clone() { - let mut loaders = Vec::new(); - - for loader_version_full in loader_versions { - let loader_version = loader_version_full.split('-').nth(1); - - if let Some(loader_version_raw) = loader_version { - // This is a dirty hack to get around Forge not complying with SemVer, but whatever - // Most of this is a hack anyways :( - // Works for all forge versions! - let split = - loader_version_raw.split('.').collect::>(); - let loader_version = if split.len() >= 4 { - if split[0].parse::().unwrap_or(0) < 6 { - format!("{}.{}.{}", split[0], split[1], split[3]) - } else { - format!("{}.{}.{}", split[1], split[2], split[3]) - } - } else { - loader_version_raw.to_string() - }; - - let version = Version::parse(&loader_version)?; - - if FORGE_MANIFEST_V1_QUERY.matches(&version) - || FORGE_MANIFEST_V2_QUERY_P1.matches(&version) - || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) - || FORGE_MANIFEST_V3_QUERY.matches(&version) - { - loaders.push((loader_version_full, version)) - } + for version in &forge_versions { + if !modrinth_manifest.game_versions.iter().any(|x| { + x.id == version.game_version + && x.loaders.iter().any(|x| x.id == version.loader_version) + }) { + fetch_versions.push(version); } } - if !loaders.is_empty() { - version_futures.push(async { - let mut loaders_versions = Vec::new(); - - { - let loaders_futures = loaders.into_iter().map(|(loader_version_full, version)| async { - let versions_mutex = Arc::clone(&old_versions); - let visited_assets = Arc::clone(&visited_assets_mutex); - let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); - let semaphore = Arc::clone(&semaphore); - let minecraft_version = minecraft_version.clone(); - - async move { - /// These forge versions are not worth supporting! - const WHITELIST : &[&str] = &[ - // Not supported due to `data` field being `[]` even though the type is a map - "1.12.2-14.23.5.2851", - // Malformed Archives - "1.6.1-8.9.0.749", - "1.6.1-8.9.0.751", - "1.6.4-9.11.1.960", - "1.6.4-9.11.1.961", - "1.6.4-9.11.1.963", - "1.6.4-9.11.1.964", - ]; - - if WHITELIST.contains(&&*loader_version_full) { - return Ok(None); - } - - { - let versions = versions_mutex.lock().await; - let version = versions.iter().find(|x| - x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full)); - - if let Some(version) = version { - return Ok::, Error>(Some(version.clone())); - } - } - - info!("Forge - Installer Start {}", loader_version_full.clone()); - let bytes = download_file(&format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None, semaphore.clone()).await?; - - let reader = std::io::Cursor::new(bytes); - - if let Ok(archive) = zip::ZipArchive::new(reader) { - if FORGE_MANIFEST_V1_QUERY.matches(&version) { - let mut archive_clone = archive.clone(); - let profile = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("install_profile.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&contents)?) - }).await??; - - let mut archive_clone = archive.clone(); - let file_path = profile.install.file_path.clone(); - let forge_universal_bytes = tokio::task::spawn_blocking(move || { - let mut forge_universal_file = archive_clone.by_name(&file_path)?; - let mut forge_universal = Vec::new(); - forge_universal_file.read_to_end(&mut forge_universal)?; - - - Ok::(bytes::Bytes::from(forge_universal)) - }).await??; - let forge_universal_path = profile.install.path.clone(); - - let now = Instant::now(); - let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async { - if let Some(url) = lib.url { - { - let mut visited_assets = visited_assets.lock().await; - - if visited_assets.contains(&lib.name) { - lib.url = Some(format_url("maven/")); - - return Ok::(lib); - } else { - visited_assets.push(lib.name.clone()) - } - } - - let artifact_path = - daedalus::get_path_from_artifact(&lib.name)?; - - let artifact = if lib.name == forge_universal_path { - forge_universal_bytes.clone() - } else { - let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"]; - - download_file_mirrors( - &artifact_path, - &mirrors, - None, - semaphore.clone(), - ) - .await? - }; - - lib.url = Some(format_url("maven/")); - - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref(), - semaphore.clone(), - ).await?; - } - - Ok::(lib) - })).await?; - - let elapsed = now.elapsed(); - info!("Elapsed lib DL: {:.2?}", elapsed); - - let new_profile = PartialVersionInfo { - id: profile.version_info.id, - inherits_from: profile.install.minecraft, - release_time: profile.version_info.release_time, - time: profile.version_info.time, - main_class: profile.version_info.main_class, - minecraft_arguments: profile.version_info.minecraft_arguments.clone(), - arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()), - libraries: libs, - type_: profile.version_info.type_, - data: None, - processors: None - }; - - let version_path = format!( - "forge/v{}/versions/{}.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - new_profile.id - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&new_profile)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - semaphore.clone(), - ).await?; - - return Ok(Some(LoaderVersion { - id: loader_version_full, - url: format_url(&version_path), - stable: false - })); - } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) { - let mut archive_clone = archive.clone(); - let mut profile = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("install_profile.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&contents)?) - }).await??; - - let mut archive_clone = archive.clone(); - let version_info = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("version.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&contents)?) - }).await??; - - - let mut libs : Vec = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library { - downloads: x.downloads, - extract: x.extract, - name: x.name, - url: x.url, - natives: x.natives, - rules: x.rules, - checksums: x.checksums, - include_in_classpath: false - })).collect(); - - let mut local_libs : HashMap = HashMap::new(); - - for lib in &libs { - if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) { - let mut archive_clone = archive.clone(); - let lib_name_clone = lib.name.clone(); - - let lib_bytes = tokio::task::spawn_blocking(move || { - let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?; - let mut lib_bytes = Vec::new(); - lib_file.read_to_end(&mut lib_bytes)?; - - Ok::(bytes::Bytes::from(lib_bytes)) - }).await??; - - local_libs.insert(lib.name.clone(), lib_bytes); - } - } - - let path = profile.path.clone(); - let version = profile.version.clone(); - - for entry in profile.data.values_mut() { - if entry.client.starts_with('/') || entry.server.starts_with('/') { - macro_rules! read_data { - ($value:expr) => { - let mut archive_clone = archive.clone(); - let value_clone = $value.clone(); - let lib_bytes = tokio::task::spawn_blocking(move || { - let mut lib_file = archive_clone.by_name(&value_clone[1..value_clone.len()])?; - let mut lib_bytes = Vec::new(); - lib_file.read_to_end(&mut lib_bytes)?; - - Ok::(bytes::Bytes::from(lib_bytes)) - }).await??; - - let split = $value.split('/').last(); - - if let Some(last) = split { - let mut file = last.split('.'); - - if let Some(file_name) = file.next() { - if let Some(ext) = file.next() { - let path = format!("{}:{}@{}", path.as_deref().unwrap_or(&*format!("net.minecraftforge:forge:{}", version)), file_name, ext); - $value = format!("[{}]", &path); - local_libs.insert(path.clone(), bytes::Bytes::from(lib_bytes)); - - libs.push(Library { - downloads: None, - extract: None, - name: path, - url: Some("".to_string()), - natives: None, - rules: None, - checksums: None, - include_in_classpath: false, - }); - } - } - } - } - } - - if entry.client.starts_with('/') { - read_data!(entry.client); - } - - if entry.server.starts_with('/') { - read_data!(entry.server); - } - } - } - - let now = Instant::now(); - let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { - let artifact_path = - daedalus::get_path_from_artifact(&lib.name)?; - - { - let mut visited_assets = visited_assets.lock().await; - - if visited_assets.contains(&lib.name) { - if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - artifact.url = format_url(&format!("maven/{}", artifact_path)); - } - } else if lib.url.is_some() { - lib.url = Some(format_url("maven/")); - } - - return Ok::(lib); - } else { - visited_assets.push(lib.name.clone()) - } - } - - let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - let res = if artifact.url.is_empty() { - local_libs.get(&lib.name).cloned() - } else { - Some(download_file( - &artifact.url, - Some(&*artifact.sha1), - semaphore.clone(), - ) - .await?) - }; - - if res.is_some() { - artifact.url = format_url(&format!("maven/{}", artifact_path)); - } - - res - } else { None } - } else if let Some(ref mut url) = lib.url { - let res = if url.is_empty() { - local_libs.get(&lib.name).cloned() - } else { - Some(download_file( - url, - None, - semaphore.clone(), - ) - .await?) - }; - - if res.is_some() { - lib.url = Some(format_url("maven/")); - } - - res - } else { None }; - - if let Some(bytes) = artifact_bytes { - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - bytes.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref(), - semaphore.clone(), - ).await?; - } - - Ok::(lib) - })).await?; - - let elapsed = now.elapsed(); - info!("Elapsed lib DL: {:.2?}", elapsed); - - let new_profile = PartialVersionInfo { - id: version_info.id, - inherits_from: version_info.inherits_from, - release_time: version_info.release_time, - time: version_info.time, - main_class: version_info.main_class, - minecraft_arguments: version_info.minecraft_arguments, - arguments: version_info.arguments, - libraries: libs, - type_: version_info.type_, - data: Some(profile.data), - processors: Some(profile.processors), - }; - - let version_path = format!( - "forge/v{}/versions/{}.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - new_profile.id - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&new_profile)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - semaphore.clone(), - ).await?; - - return Ok(Some(LoaderVersion { - id: loader_version_full, - url: format_url(&version_path), - stable: false - })); - } - } - - Ok(None) - }.await - }); - - { - let len = loaders_futures.len(); - let mut versions = loaders_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions.peek().is_some() { - let now = Instant::now(); - - let chunk: Vec<_> = versions.by_ref().take(1).collect(); - let res = futures::future::try_join_all(chunk).await?; - loaders_versions.extend(res.into_iter().flatten()); - - chunk_index += 1; - - let elapsed = now.elapsed(); - info!("Loader Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed); - } - } - //futures::future::try_join_all(loaders_futures).await?; - } - - versions.lock().await.push(daedalus::modded::Version { - id: minecraft_version, - stable: true, - loaders: loaders_versions - }); - - Ok::<(), Error>(()) - }); - } - } - - { - let len = version_futures.len(); - let mut versions = version_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions.peek().is_some() { - let now = Instant::now(); - - let chunk: Vec<_> = versions.by_ref().take(1).collect(); - futures::future::try_join_all(chunk).await?; - - chunk_index += 1; - - let elapsed = now.elapsed(); - info!("Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed); - } - } - //futures::future::try_join_all(version_futures).await?; - - if let Ok(versions) = Arc::try_unwrap(versions) { - let mut versions = versions.into_inner(); - - versions.sort_by(|x, y| { - minecraft_versions - .versions + fetch_versions + } else { + forge_versions.iter().collect() + }; + + if !fetch_versions.is_empty() { + let forge_installers = futures::future::try_join_all( + fetch_versions .iter() - .position(|z| { - x.id.replace("1.7.10_pre4", "1.7.10-pre4") == z.id - }) - .unwrap_or_default() - .cmp( - &minecraft_versions - .versions - .iter() - .position(|z| { - y.id.replace("1.7.10_pre4", "1.7.10-pre4") == z.id - }) - .unwrap_or_default(), - ) - }); - - for version in &mut versions { - let loader_versions = maven_metadata.get(&version.id); - if let Some(loader_versions) = loader_versions { - version.loaders.sort_by(|x, y| { - loader_versions - .iter() - .position(|z| &y.id == z) - .unwrap_or_default() - .cmp( - &loader_versions - .iter() - .position(|z| &x.id == z) - .unwrap_or_default(), - ) - }) - } - } - - upload_file_to_bucket( - format!( - "forge/v{}/manifest.json", - daedalus::modded::CURRENT_FORGE_FORMAT_VERSION, - ), - serde_json::to_vec(&Manifest { - game_versions: versions, - })?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - semaphore, + .map(|x| download_file(&x.installer_url, None, &semaphore)), ) .await?; - } - if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { - uploaded_files.extend(uploaded_files_mutex.into_inner()); + #[tracing::instrument(skip(raw, upload_files, mirror_artifacts))] + async fn read_forge_installer( + raw: bytes::Bytes, + loader: &ForgeVersion, + maven_url: &str, + upload_files: &DashMap, + mirror_artifacts: &DashMap, + ) -> Result { + tracing::trace!( + "Reading forge installer for {}", + loader.loader_version + ); + type ZipFileReader = async_zip::base::read::seek::ZipFileReader< + Cursor, + >; + + let cursor = Cursor::new(raw); + let mut zip = ZipFileReader::new(cursor).await?; + + #[tracing::instrument(skip(zip))] + async fn read_file( + zip: &mut ZipFileReader, + file_name: &str, + ) -> Result>, Error> { + let zip_index_option = + zip.file().entries().iter().position(|f| { + f.filename().as_str().unwrap_or_default() == file_name + }); + + if let Some(zip_index) = zip_index_option { + let mut buffer = Vec::new(); + let mut reader = zip.reader_with_entry(zip_index).await?; + reader.read_to_end_checked(&mut buffer).await?; + + Ok(Some(buffer)) + } else { + Ok(None) + } + } + + #[tracing::instrument(skip(zip))] + async fn read_json( + zip: &mut ZipFileReader, + file_name: &str, + ) -> Result, Error> { + if let Some(file) = read_file(zip, file_name).await? { + Ok(Some(serde_json::from_slice(&file)?)) + } else { + Ok(None) + } + } + + if loader.format_version == 1 { + #[derive(Deserialize, Debug)] + #[serde(rename_all = "camelCase")] + struct ForgeInstallerProfileInstallDataV1 { + // pub mirror_list: String, + // pub target: String, + /// Path to the Forge universal library + pub file_path: String, + // pub logo: String, + // pub welcome: String, + // pub version: String, + /// Maven coordinates of the Forge universal library + pub path: String, + // pub profile_name: String, + pub minecraft: String, + } + + #[derive(Deserialize, Debug)] + #[serde(rename_all = "camelCase")] + struct ForgeInstallerProfileManifestV1 { + pub id: String, + pub libraries: Vec, + pub main_class: Option, + pub minecraft_arguments: Option, + pub release_time: DateTime, + pub time: DateTime, + pub type_: daedalus::minecraft::VersionType, + // pub assets: Option, + // pub inherits_from: Option, + // pub jar: Option, + } + + #[derive(Deserialize, Debug)] + #[serde(rename_all = "camelCase")] + struct ForgeInstallerProfileV1 { + pub install: ForgeInstallerProfileInstallDataV1, + pub version_info: ForgeInstallerProfileManifestV1, + } + + let install_profile = read_json::( + &mut zip, + "install_profile.json", + ) + .await? + .ok_or_else(|| { + crate::ErrorKind::InvalidInput(format!( + "No install_profile.json present for loader {}", + loader.installer_url + )) + })?; + + let forge_library = + read_file(&mut zip, &install_profile.install.file_path) + .await? + .ok_or_else(|| { + crate::ErrorKind::InvalidInput(format!( + "No forge library present for loader {}", + loader.installer_url + )) + })?; + + upload_files.insert( + format!( + "maven/{}", + get_path_from_artifact(&install_profile.install.path)? + ), + UploadFile { + file: bytes::Bytes::from(forge_library), + content_type: None, + }, + ); + + Ok(PartialVersionInfo { + id: install_profile.version_info.id, + inherits_from: install_profile.install.minecraft, + release_time: install_profile.version_info.release_time, + time: install_profile.version_info.time, + main_class: install_profile.version_info.main_class, + minecraft_arguments: install_profile + .version_info + .minecraft_arguments + .clone(), + arguments: install_profile + .version_info + .minecraft_arguments + .map(|x| { + [( + daedalus::minecraft::ArgumentType::Game, + x.split(' ') + .map(|x| { + daedalus::minecraft::Argument::Normal( + x.to_string(), + ) + }) + .collect(), + )] + .iter() + .cloned() + .collect() + }), + libraries: install_profile + .version_info + .libraries + .into_iter() + .map(|mut lib| { + // For all libraries besides the forge lib extracted, we mirror them from maven servers + if lib.name != install_profile.install.path { + // TODO: add mirrors "https://maven.creeperhost.net/", "https://libraries.minecraft.net/" + insert_mirrored_artifact( + &lib.name, + lib.url.clone().unwrap_or_else(|| { + maven_url.to_string() + }), + mirror_artifacts, + )?; + } + + lib.url = Some(format_url("maven/")); + + Ok(lib) + }) + .collect::, Error>>()?, + type_: install_profile.version_info.type_, + data: None, + processors: None, + }) + } else if loader.format_version == 2 { + #[derive(Deserialize, Debug)] + #[serde(rename_all = "camelCase")] + struct ForgeInstallerProfileV2 { + // pub spec: i32, + // pub profile: String, + // pub version: String, + // pub json: String, + pub path: Option, + // pub minecraft: String, + pub data: HashMap, + pub libraries: Vec, + pub processors: Vec, + } + + let install_profile = read_json::( + &mut zip, + "install_profile.json", + ) + .await? + .ok_or_else(|| { + crate::ErrorKind::InvalidInput(format!( + "No install_profile.json present for loader {}", + loader.installer_url + )) + })?; + + let mut version_info = + read_json::(&mut zip, "version.json") + .await? + .ok_or_else(|| { + crate::ErrorKind::InvalidInput(format!( + "No version.json present for loader {}", + loader.installer_url + )) + })?; + + version_info.processors = Some(install_profile.processors); + version_info.libraries.extend( + install_profile.libraries.into_iter().map(|mut x| { + x.include_in_classpath = false; + + x + }), + ); + + async fn mirror_forge_library( + mut zip: ZipFileReader, + mut lib: daedalus::minecraft::Library, + upload_files: &DashMap, + mirror_artifacts: &DashMap, + ) -> Result + { + let artifact_path = get_path_from_artifact(&lib.name)?; + + if let Some(ref mut artifact) = + lib.downloads.as_mut().and_then(|x| x.artifact.as_mut()) + { + if !artifact.url.is_empty() { + insert_mirrored_artifact( + &lib.name, + artifact.url.clone(), + mirror_artifacts, + )?; + + artifact.url = + format_url(&format!("maven/{}", artifact_path)); + + return Ok(lib); + } + } else if let Some(url) = &lib.url { + if !url.is_empty() { + // TODO: add mirrors "https://maven.creeperhost.net/", "https://libraries.minecraft.net/" + insert_mirrored_artifact( + &lib.name, + url.clone(), + mirror_artifacts, + )?; + + lib.url = Some(format_url("maven/")); + + return Ok(lib); + } + } + + // Other libraries are generally available in the "maven" directory of the installer. If they are + // not present here, they will be generated by Forge processors. + let extract_path = format!("maven/{artifact_path}"); + if let Some(file) = + read_file(&mut zip, &extract_path).await? + { + upload_files.insert( + extract_path, + UploadFile { + file: bytes::Bytes::from(file), + content_type: None, + }, + ); + + lib.url = Some(format_url("maven/")); + } else { + lib.downloadable = false; + } + + Ok(lib) + } + + version_info.libraries = futures::future::try_join_all( + version_info.libraries.into_iter().map(|lib| { + mirror_forge_library( + zip.clone(), + lib, + upload_files, + mirror_artifacts, + ) + }), + ) + .await?; + + // In Minecraft Forge modern installers, processors are run during the install process. Some processors + // are extracted from the installer JAR. This function finds these files, extracts them, and uploads them + // and registers them as libraries instead. + // Ex: + // "BINPATCH": { + // "client": "/data/client.lzma", + // "server": "/data/server.lzma" + // }, + // Becomes: + // "BINPATCH": { + // "client": "[net.minecraftforge:forge:1.20.3-49.0.1:shim:client@lzma]", + // "server": "[net.minecraftforge:forge:1.20.3-49.0.1:shim:server@lzma]" + // }, + // And the resulting library is added to the profile's libraries + let mut new_data = HashMap::new(); + for (key, entry) in install_profile.data { + async fn extract_data( + zip: &mut ZipFileReader, + key: &str, + value: &str, + upload_files: &DashMap, + libs: &mut Vec, + install_profile_path: Option<&str>, + version: &ForgeVersion, + ) -> Result { + let extract_file = + read_file(zip, &value[1..value.len()]) + .await? + .ok_or_else(|| { + crate::ErrorKind::InvalidInput(format!( + "Unable reading data key {key} at path {value}", + )) + })?; + + let file_name = value.split('/').last() + .ok_or_else(|| { + crate::ErrorKind::InvalidInput(format!( + "Unable reading filename for data key {key} at path {value}", + + )) + })?; + + let mut file = file_name.split('.'); + let file_name = file.next() + .ok_or_else(|| { + crate::ErrorKind::InvalidInput(format!( + "Unable reading filename only for data key {key} at path {value}", + )) + })?; + let ext = file.next() + .ok_or_else(|| { + crate::ErrorKind::InvalidInput(format!( + "Unable reading extension only for data key {key} at path {value}", + )) + })?; + + let path = format!( + "{}:{}@{}", + install_profile_path.unwrap_or(&*format!( + "net.minecraftforge:forge:{}", + version.raw + )), + file_name, + ext + ); + + upload_files.insert( + format!("maven/{}", get_path_from_artifact(&path)?), + UploadFile { + file: bytes::Bytes::from(extract_file), + content_type: None, + }, + ); + + libs.push(daedalus::minecraft::Library { + downloads: None, + extract: None, + name: path.clone(), + url: Some(format_url("maven/")), + natives: None, + rules: None, + checksums: None, + include_in_classpath: false, + downloadable: true, + }); + + Ok(format!("[{path}]")) + } + + let client = if entry.client.starts_with('/') { + extract_data( + &mut zip, + &key, + &entry.client, + upload_files, + &mut version_info.libraries, + install_profile.path.as_deref(), + loader, + ) + .await? + } else { + entry.client.clone() + }; + + let server = if entry.server.starts_with('/') { + extract_data( + &mut zip, + &key, + &entry.server, + upload_files, + &mut version_info.libraries, + install_profile.path.as_deref(), + loader, + ) + .await? + } else { + entry.server.clone() + }; + + new_data.insert( + key.clone(), + daedalus::modded::SidedDataEntry { client, server }, + ); + } + + version_info.data = Some(new_data); + + Ok(version_info) + } else { + Err(crate::ErrorKind::InvalidInput(format!( + "Unknown format version {} for loader {}", + loader.format_version, loader.installer_url + )) + .into()) + } + } + + let forge_version_infos = futures::future::try_join_all( + forge_installers + .into_iter() + .enumerate() + .map(|(index, raw)| { + let loader = fetch_versions[index]; + + read_forge_installer( + raw, + loader, + maven_url, + upload_files, + mirror_artifacts, + ) + }), + ) + .await?; + + let serialized_version_manifests = forge_version_infos + .iter() + .map(|x| serde_json::to_vec(x).map(bytes::Bytes::from)) + .collect::, serde_json::Error>>()?; + + serialized_version_manifests + .into_iter() + .enumerate() + .for_each(|(index, bytes)| { + let loader = fetch_versions[index]; + + let version_path = format!( + "{mod_loader}/v{format_version}/versions/{}.json", + loader.loader_version + ); + + upload_files.insert( + version_path, + UploadFile { + file: bytes, + content_type: Some("application/json".to_string()), + }, + ); + }); + + let forge_manifest_path = + format!("{mod_loader}/v{format_version}/manifest.json",); + + let manifest = daedalus::modded::Manifest { + game_versions: forge_versions + .into_iter() + .rev() + .chunk_by(|x| x.game_version.clone()) + .into_iter() + .map(|(game_version, loaders)| daedalus::modded::Version { + id: game_version, + stable: true, + loaders: loaders + .map(|x| daedalus::modded::LoaderVersion { + url: format_url(&format!( + "{mod_loader}/v{format_version}/versions/{}.json", + x.loader_version + )), + id: x.loader_version, + stable: false, + }) + .collect(), + }) + .collect(), + }; + + upload_files.insert( + forge_manifest_path, + UploadFile { + file: bytes::Bytes::from(serde_json::to_vec(&manifest)?), + content_type: Some("application/json".to_string()), + }, + ); } Ok(()) } -const DEFAULT_MAVEN_METADATA_URL: &str = - "https://files.minecraftforge.net/net/minecraftforge/forge/maven-metadata.json"; - -/// Fetches the forge maven metadata from the specified URL. If no URL is specified, the default is used. -/// Returns a hashmap specifying the versions of the forge mod loader -/// The hashmap key is a Minecraft version, and the value is the loader versions that work on -/// the specified Minecraft version -pub async fn fetch_maven_metadata( - url: Option<&str>, - semaphore: Arc, -) -> Result>, Error> { - Ok(serde_json::from_slice( - &download_file( - url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), - None, - semaphore, - ) - .await?, - )?) -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -struct ForgeInstallerProfileInstallDataV1 { - pub mirror_list: String, - pub target: String, - /// Path to the Forge universal library - pub file_path: String, - pub logo: String, - pub welcome: String, - pub version: String, - /// Maven coordinates of the Forge universal library - pub path: String, - pub profile_name: String, - pub minecraft: String, -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -struct ForgeInstallerProfileManifestV1 { - pub id: String, - pub libraries: Vec, - pub main_class: Option, - pub minecraft_arguments: Option, - pub release_time: DateTime, - pub time: DateTime, - pub type_: VersionType, - pub assets: Option, - pub inherits_from: Option, - pub jar: Option, -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -struct ForgeInstallerProfileV1 { - pub install: ForgeInstallerProfileInstallDataV1, - pub version_info: ForgeInstallerProfileManifestV1, -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -struct ForgeInstallerProfileV2 { - pub spec: i32, - pub profile: String, - pub version: String, - pub json: String, - pub path: Option, - pub minecraft: String, - pub data: HashMap, - pub libraries: Vec, - pub processors: Vec, +#[derive(Debug)] +struct ForgeVersion { + pub format_version: usize, + pub raw: String, + pub loader_version: String, + pub game_version: String, + pub installer_url: String, } diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index af34ce3a..33df7090 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -1,118 +1,147 @@ -use log::{error, info, warn}; -use s3::creds::Credentials; -use s3::error::S3Error; -use s3::{Bucket, Region}; +use crate::util::{ + format_url, upload_file_to_bucket, upload_url_to_bucket_mirrors, + REQWEST_CLIENT, +}; +use daedalus::get_path_from_artifact; +use dashmap::{DashMap, DashSet}; use std::sync::Arc; -use std::time::Duration; use tokio::sync::Semaphore; +use tracing_error::ErrorLayer; +use tracing_subscriber::{fmt, prelude::*, EnvFilter}; +mod error; mod fabric; mod forge; mod minecraft; -mod neo; -mod quilt; +pub mod util; -#[derive(thiserror::Error, Debug)] -pub enum Error { - #[error("{0}")] - DaedalusError(#[from] daedalus::Error), - #[error("Error while deserializing JSON")] - SerdeError(#[from] serde_json::Error), - #[error("Error while deserializing XML")] - XMLError(#[from] serde_xml_rs::Error), - #[error("Unable to fetch {item}")] - FetchError { inner: reqwest::Error, item: String }, - #[error("Error while managing asynchronous tasks")] - TaskError(#[from] tokio::task::JoinError), - #[error("Error while uploading file to S3")] - S3Error { inner: S3Error, file: String }, - #[error("Error while parsing version as semver: {0}")] - SemVerError(#[from] semver::Error), - #[error("Error while reading zip file: {0}")] - ZipError(#[from] zip::result::ZipError), - #[error("Error while reading zip file: {0}")] - IoError(#[from] std::io::Error), - #[error("Error while obtaining strong reference to Arc")] - ArcError, - #[error("Error acquiring semaphore: {0}")] - AcquireError(#[from] tokio::sync::AcquireError), -} +pub use error::{Error, ErrorKind, Result}; #[tokio::main] -async fn main() { - env_logger::init(); +async fn main() -> Result<()> { + dotenvy::dotenv().ok(); + + let subscriber = tracing_subscriber::registry() + .with(fmt::layer()) + .with(EnvFilter::from_default_env()) + .with(ErrorLayer::default()); + + tracing::subscriber::set_global_default(subscriber)?; + + tracing::info!("Initialized tracing. Starting Daedalus!"); if check_env_vars() { - error!("Some environment variables are missing!"); + tracing::error!("Some environment variables are missing!"); - return; + return Ok(()); } - let mut timer = tokio::time::interval(Duration::from_secs(60 * 60)); - let semaphore = Arc::new(Semaphore::new(10)); + let semaphore = Arc::new(Semaphore::new( + dotenvy::var("CONCURRENCY_LIMIT") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(10), + )); - loop { - timer.tick().await; + // path, upload file + let upload_files: DashMap = DashMap::new(); + // path, mirror artifact + let mirror_artifacts: DashMap = DashMap::new(); - let mut uploaded_files = Vec::new(); + minecraft::fetch(semaphore.clone(), &upload_files, &mirror_artifacts) + .await?; + fabric::fetch_fabric(semaphore.clone(), &upload_files, &mirror_artifacts) + .await?; + fabric::fetch_quilt(semaphore.clone(), &upload_files, &mirror_artifacts) + .await?; + forge::fetch_neo(semaphore.clone(), &upload_files, &mirror_artifacts) + .await?; + forge::fetch_forge(semaphore.clone(), &upload_files, &mirror_artifacts) + .await?; - let versions = match minecraft::retrieve_data( - &mut uploaded_files, - semaphore.clone(), + futures::future::try_join_all(upload_files.iter().map(|x| { + upload_file_to_bucket( + x.key().clone(), + x.value().file.clone(), + x.value().content_type.clone(), + &semaphore, ) - .await - { - Ok(res) => Some(res), - Err(err) => { - error!("{:?}", err); + })) + .await?; - None + futures::future::try_join_all(mirror_artifacts.iter().map(|x| { + upload_url_to_bucket_mirrors( + format!("maven/{}", x.key()), + x.value().mirrors.iter().map(|x| x.key().clone()).collect(), + &semaphore, + ) + })) + .await?; + + if let Ok(token) = dotenvy::var("CLOUDFLARE_TOKEN") { + if let Ok(zone_id) = dotenvy::var("CLOUDFLARE_ZONE_ID") { + let cache_clears = upload_files + .into_iter() + .map(|x| format_url(&x.0)) + .chain( + mirror_artifacts + .into_iter() + .map(|x| format_url(&format!("maven/{}", x.0))), + ) + .collect::>(); + + // Cloudflare ratelimits cache clears to 500 files per request + for chunk in cache_clears.chunks(500) { + REQWEST_CLIENT.post(format!("https://api.cloudflare.com/client/v4/zones/{zone_id}/purge_cache")) + .bearer_auth(&token) + .json(&serde_json::json!({ + "files": chunk + })) + .send() + .await + .map_err(|err| { + ErrorKind::Fetch { + inner: err, + item: "cloudflare clear cache".to_string(), + } + })? + .error_for_status() + .map_err(|err| { + ErrorKind::Fetch { + inner: err, + item: "cloudflare clear cache".to_string(), + } + })?; } - }; - - if let Some(manifest) = versions { - match fabric::retrieve_data( - &manifest, - &mut uploaded_files, - semaphore.clone(), - ) - .await - { - Ok(..) => {} - Err(err) => error!("{:?}", err), - }; - match forge::retrieve_data( - &manifest, - &mut uploaded_files, - semaphore.clone(), - ) - .await - { - Ok(..) => {} - Err(err) => error!("{:?}", err), - }; - match quilt::retrieve_data( - &manifest, - &mut uploaded_files, - semaphore.clone(), - ) - .await - { - Ok(..) => {} - Err(err) => error!("{:?}", err), - }; - match neo::retrieve_data( - &manifest, - &mut uploaded_files, - semaphore.clone(), - ) - .await - { - Ok(..) => {} - Err(err) => error!("{:?}", err), - }; } } + + Ok(()) +} + +pub struct UploadFile { + file: bytes::Bytes, + content_type: Option, +} + +pub struct MirrorArtifact { + pub mirrors: DashSet, +} + +pub fn insert_mirrored_artifact( + artifact: &str, + mirror: String, + mirror_artifacts: &DashMap, +) -> Result<()> { + mirror_artifacts + .entry(get_path_from_artifact(artifact)?) + .or_insert(MirrorArtifact { + mirrors: DashSet::new(), + }) + .mirrors + .insert(mirror); + + Ok(()) } fn check_env_vars() -> bool { @@ -124,7 +153,7 @@ fn check_env_vars() -> bool { .and_then(|s| s.parse::().ok()) .is_none() { - warn!( + tracing::warn!( "Variable `{}` missing in dotenvy or not of type `{}`", var, std::any::type_name::() @@ -143,110 +172,14 @@ fn check_env_vars() -> bool { failed |= check_var::("S3_REGION"); failed |= check_var::("S3_BUCKET_NAME"); + if dotenvy::var("CLOUDFLARE_INTEGRATION") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(false) + { + failed |= check_var::("CLOUDFLARE_TOKEN"); + failed |= check_var::("CLOUDFLARE_ZONE_ID"); + } + failed } - -lazy_static::lazy_static! { - static ref CLIENT : Bucket = { - let region = dotenvy::var("S3_REGION").unwrap(); - let b = Bucket::new( - &dotenvy::var("S3_BUCKET_NAME").unwrap(), - if &*region == "r2" { - Region::R2 { - account_id: dotenvy::var("S3_URL").unwrap(), - } - } else { - Region::Custom { - region: region.clone(), - endpoint: dotenvy::var("S3_URL").unwrap(), - } - }, - Credentials::new( - Some(&*dotenvy::var("S3_ACCESS_TOKEN").unwrap()), - Some(&*dotenvy::var("S3_SECRET").unwrap()), - None, - None, - None, - ).unwrap(), - ).unwrap(); - - if region == "path-style" { - b.with_path_style() - } else { - b - } - }; -} - -pub async fn upload_file_to_bucket( - path: String, - bytes: Vec, - content_type: Option, - uploaded_files: &tokio::sync::Mutex>, - semaphore: Arc, -) -> Result<(), Error> { - let _permit = semaphore.acquire().await?; - info!("{} started uploading", path); - let key = path.clone(); - - for attempt in 1..=4 { - let result = if let Some(ref content_type) = content_type { - CLIENT - .put_object_with_content_type(key.clone(), &bytes, content_type) - .await - } else { - CLIENT.put_object(key.clone(), &bytes).await - } - .map_err(|err| Error::S3Error { - inner: err, - file: path.clone(), - }); - - match result { - Ok(_) => { - { - info!("{} done uploading", path); - let mut uploaded_files = uploaded_files.lock().await; - uploaded_files.push(key); - } - - return Ok(()); - } - Err(_) if attempt <= 3 => continue, - Err(_) => { - result?; - } - } - } - unreachable!() -} - -pub fn format_url(path: &str) -> String { - format!("{}/{}", &*dotenvy::var("BASE_URL").unwrap(), path) -} - -pub async fn download_file( - url: &str, - sha1: Option<&str>, - semaphore: Arc, -) -> Result { - let _permit = semaphore.acquire().await?; - info!("{} started downloading", url); - let val = daedalus::download_file(url, sha1).await?; - info!("{} finished downloading", url); - Ok(val) -} - -pub async fn download_file_mirrors( - base: &str, - mirrors: &[&str], - sha1: Option<&str>, - semaphore: Arc, -) -> Result { - let _permit = semaphore.acquire().await?; - info!("{} started downloading", base); - let val = daedalus::download_file_mirrors(base, mirrors, sha1).await?; - info!("{} finished downloading", base); - - Ok(val) -} diff --git a/daedalus_client/src/minecraft.rs b/daedalus_client/src/minecraft.rs index 1f0e5b7c..e98b1204 100644 --- a/daedalus_client/src/minecraft.rs +++ b/daedalus_client/src/minecraft.rs @@ -1,286 +1,181 @@ -use crate::download_file; -use crate::{format_url, upload_file_to_bucket, Error}; -use daedalus::get_hash; -use daedalus::minecraft::{ - merge_partial_library, Library, PartialLibrary, VersionManifest, +use crate::util::fetch_json; +use crate::{ + util::download_file, util::format_url, util::sha1_async, Error, + MirrorArtifact, UploadFile, }; -use log::info; +use daedalus::minecraft::{ + merge_partial_library, Library, PartialLibrary, VersionInfo, + VersionManifest, VERSION_MANIFEST_URL, +}; +use dashmap::DashMap; use serde::Deserialize; use std::sync::Arc; -use std::time::Instant; -use tokio::sync::{Mutex, Semaphore}; +use tokio::sync::Semaphore; -pub async fn retrieve_data( - uploaded_files: &mut Vec, +#[tracing::instrument(skip(semaphore, upload_files, _mirror_artifacts))] +pub async fn fetch( semaphore: Arc, -) -> Result { - let old_manifest = daedalus::minecraft::fetch_version_manifest(Some( - &*format_url(&format!( + upload_files: &DashMap, + _mirror_artifacts: &DashMap, +) -> Result<(), Error> { + let modrinth_manifest = fetch_json::( + &format_url(&format!( "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION )), - )) + &semaphore, + ) .await .ok(); + let mojang_manifest = + fetch_json::(VERSION_MANIFEST_URL, &semaphore).await?; - let mut manifest = - daedalus::minecraft::fetch_version_manifest(None).await?; - let cloned_manifest = - Arc::new(Mutex::new(old_manifest.clone().unwrap_or(manifest.clone()))); + // TODO: experimental snapshots: https://github.com/PrismLauncher/meta/blob/main/meta/common/mojang-minecraft-experiments.json + // TODO: old snapshots: https://github.com/PrismLauncher/meta/blob/main/meta/common/mojang-minecraft-old-snapshots.json - let patches = fetch_library_patches()?; - let cloned_patches = Arc::new(&patches); + // We check Modrinth's version manifest and compare if the version 1) exists in Modrinth's database and 2) is unchanged + // If they are not, we will fetch them + let (fetch_versions, existing_versions) = + if let Some(mut modrinth_manifest) = modrinth_manifest { + let (mut fetch_versions, mut existing_versions) = + (Vec::new(), Vec::new()); - let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); - let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); + for version in mojang_manifest.versions { + if let Some(index) = modrinth_manifest + .versions + .iter() + .position(|x| x.id == version.id) + { + let modrinth_version = + modrinth_manifest.versions.remove(index); - let now = Instant::now(); - - let mut version_futures = Vec::new(); - - for version in manifest.versions.iter_mut() { - version_futures.push(async { - let old_version = if let Some(old_manifest) = &old_manifest { - old_manifest.versions.iter().find(|x| x.id == version.id) - } else { - None - }; - - if old_version.is_some() { - return Ok(()); + if modrinth_version + .original_sha1 + .as_ref() + .map(|x| x == &version.sha1) + .unwrap_or(false) + { + existing_versions.push(modrinth_version); + } else { + fetch_versions.push(version); + } + } else { + fetch_versions.push(version); + } } - let visited_assets_mutex = Arc::clone(&visited_assets_mutex); - let cloned_manifest_mutex = Arc::clone(&cloned_manifest); - let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); - let semaphore = Arc::clone(&semaphore); - let patches = Arc::clone(&cloned_patches); + (fetch_versions, existing_versions) + } else { + (mojang_manifest.versions, Vec::new()) + }; - let assets_hash = - old_version.and_then(|x| x.assets_index_sha1.clone()); + if !fetch_versions.is_empty() { + let version_manifests = futures::future::try_join_all( + fetch_versions + .iter() + .map(|x| download_file(&x.url, Some(&x.sha1), &semaphore)), + ) + .await? + .into_iter() + .map(|x| serde_json::from_slice(&x)) + .collect::, serde_json::Error>>()?; - async move { - let mut upload_futures = Vec::new(); - - let mut version_info = - daedalus::minecraft::fetch_version_info(version).await?; - - fn patch_library( - patches: &Vec, - mut library: Library, - ) -> Vec { - let mut val = Vec::new(); - - let actual_patches = patches - .iter() - .filter(|x| x.match_.contains(&library.name)) - .collect::>(); - - if !actual_patches.is_empty() { - for patch in actual_patches { - if let Some(override_) = &patch.override_ { - library = merge_partial_library( - override_.clone(), - library, - ); - } - - if let Some(additional_libraries) = - &patch.additional_libraries - { - for additional_library in additional_libraries { - if patch - .patch_additional_libraries - .unwrap_or(false) - { - let mut libs = patch_library( - patches, - additional_library.clone(), - ); - val.append(&mut libs) - } else { - val.push(additional_library.clone()); - } - } - } - } - - val.push(library); - } else { - val.push(library); + // Patch libraries of Minecraft versions for M-series Mac Support, Better Linux Compatibility, etc + let library_patches = fetch_library_patches()?; + let patched_version_manifests = version_manifests + .into_iter() + .map(|mut x| { + if !library_patches.is_empty() { + let mut new_libraries = Vec::new(); + for library in x.libraries { + let mut libs = patch_library(&library_patches, library); + new_libraries.append(&mut libs) } - - val + x.libraries = new_libraries } - let mut new_libraries = Vec::new(); - for library in version_info.libraries.clone() { - let mut libs = patch_library(&patches, library); - new_libraries.append(&mut libs) - } - version_info.libraries = new_libraries; + x + }) + .collect::>(); - let version_info_hash = get_hash(bytes::Bytes::from( - serde_json::to_vec(&version_info)?, - )) - .await?; + // serialize + compute hashes + let serialized_version_manifests = patched_version_manifests + .iter() + .map(|x| serde_json::to_vec(x).map(bytes::Bytes::from)) + .collect::, serde_json::Error>>()?; + let hashes_version_manifests = futures::future::try_join_all( + serialized_version_manifests + .iter() + .map(|x| sha1_async(x.clone())), + ) + .await?; + // We upload the new version manifests and add them to the versions list + let mut new_versions = patched_version_manifests + .into_iter() + .zip(serialized_version_manifests.into_iter()) + .zip(hashes_version_manifests.into_iter()) + .map(|((version, bytes), hash)| { let version_path = format!( "minecraft/v{}/versions/{}.json", daedalus::minecraft::CURRENT_FORMAT_VERSION, version.id ); - let assets_path = format!( - "minecraft/v{}/assets/{}.json", - daedalus::minecraft::CURRENT_FORMAT_VERSION, - version_info.asset_index.id + + let url = format_url(&version_path); + upload_files.insert( + version_path, + UploadFile { + file: bytes, + content_type: Some("application/json".to_string()), + }, ); - let assets_index_url = version_info.asset_index.url.clone(); - { - let mut cloned_manifest = - cloned_manifest_mutex.lock().await; - - if let Some(position) = cloned_manifest - .versions + daedalus::minecraft::Version { + original_sha1: fetch_versions .iter() - .position(|x| version.id == x.id) - { - cloned_manifest.versions[position].url = - format_url(&version_path); - cloned_manifest.versions[position].assets_index_sha1 = - Some(version_info.asset_index.sha1.clone()); - cloned_manifest.versions[position].assets_index_url = - Some(format_url(&assets_path)); - cloned_manifest.versions[position].sha1 = - version_info_hash; - } else { - cloned_manifest.versions.insert( - 0, - daedalus::minecraft::Version { - id: version_info.id.clone(), - type_: version_info.type_.clone(), - url: format_url(&version_path), - time: version_info.time, - release_time: version_info.release_time, - sha1: version_info_hash, - compliance_level: 1, - assets_index_url: Some( - version_info.asset_index.sha1.clone(), - ), - assets_index_sha1: Some( - version_info.asset_index.sha1.clone(), - ), - }, - ) - } + .find(|x| x.id == version.id) + .map(|x| x.sha1.clone()), + id: version.id, + type_: version.type_, + url, + time: version.time, + release_time: version.release_time, + sha1: hash, + compliance_level: 1, } + }) + .chain(existing_versions.into_iter()) + .collect::>(); - let mut download_assets = false; + new_versions.sort_by(|a, b| b.release_time.cmp(&a.release_time)); - { - let mut visited_assets = visited_assets_mutex.lock().await; - - if !visited_assets.contains(&version_info.asset_index.id) { - if let Some(assets_hash) = assets_hash { - if version_info.asset_index.sha1 != assets_hash { - download_assets = true; - } - } else { - download_assets = true; - } - } - - if download_assets { - visited_assets - .push(version_info.asset_index.id.clone()); - } - } - - if download_assets { - let assets_index = download_file( - &assets_index_url, - Some(&version_info.asset_index.sha1), - semaphore.clone(), - ) - .await?; - - { - upload_futures.push(upload_file_to_bucket( - assets_path, - assets_index.to_vec(), - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - semaphore.clone(), - )); - } - } - - { - upload_futures.push(upload_file_to_bucket( - version_path, - serde_json::to_vec(&version_info)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - semaphore.clone(), - )); - } - - futures::future::try_join_all(upload_futures).await?; - - Ok::<(), Error>(()) - } - .await?; - - Ok::<(), Error>(()) - }) - } - - { - let mut versions = version_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions.peek().is_some() { - let now = Instant::now(); - - let chunk: Vec<_> = versions.by_ref().take(100).collect(); - futures::future::try_join_all(chunk).await?; - - chunk_index += 1; - - let elapsed = now.elapsed(); - info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed); - } - } - //futures::future::try_join_all(version_futures).await?; - - upload_file_to_bucket( - format!( + // create and upload the new manifest + let version_manifest_path = format!( "minecraft/v{}/manifest.json", daedalus::minecraft::CURRENT_FORMAT_VERSION - ), - serde_json::to_vec(&*cloned_manifest.lock().await)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - semaphore, - ) - .await?; + ); - if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { - uploaded_files.extend(uploaded_files_mutex.into_inner()); + let new_manifest = VersionManifest { + latest: mojang_manifest.latest, + versions: new_versions, + }; + + upload_files.insert( + version_manifest_path, + UploadFile { + file: bytes::Bytes::from(serde_json::to_vec(&new_manifest)?), + content_type: Some("application/json".to_string()), + }, + ); } - let elapsed = now.elapsed(); - info!("Elapsed: {:.2?}", elapsed); - - Ok(Arc::try_unwrap(cloned_manifest) - .map_err(|_| Error::ArcError)? - .into_inner()) + Ok(()) } #[derive(Deserialize, Debug)] #[serde(rename_all = "camelCase")] -/// A version of the fabric loader -struct LibraryPatch { +pub struct LibraryPatch { #[serde(rename = "_comment")] pub _comment: String, #[serde(rename = "match")] @@ -291,8 +186,45 @@ struct LibraryPatch { pub patch_additional_libraries: Option, } -/// Fetches the list of fabric versions fn fetch_library_patches() -> Result, Error> { let patches = include_bytes!("../library-patches.json"); Ok(serde_json::from_slice(patches)?) } + +pub fn patch_library( + patches: &Vec, + mut library: Library, +) -> Vec { + let mut val = Vec::new(); + + let actual_patches = patches + .iter() + .filter(|x| x.match_.contains(&library.name)) + .collect::>(); + + if !actual_patches.is_empty() { + for patch in actual_patches { + if let Some(override_) = &patch.override_ { + library = merge_partial_library(override_.clone(), library); + } + + if let Some(additional_libraries) = &patch.additional_libraries { + for additional_library in additional_libraries { + if patch.patch_additional_libraries.unwrap_or(false) { + let mut libs = + patch_library(patches, additional_library.clone()); + val.append(&mut libs) + } else { + val.push(additional_library.clone()); + } + } + } + } + + val.push(library); + } else { + val.push(library); + } + + val +} diff --git a/daedalus_client/src/neo.rs b/daedalus_client/src/neo.rs deleted file mode 100644 index ddc07564..00000000 --- a/daedalus_client/src/neo.rs +++ /dev/null @@ -1,495 +0,0 @@ -use crate::{download_file, format_url, upload_file_to_bucket, Error}; -use daedalus::minecraft::{Library, VersionManifest}; -use daedalus::modded::{ - LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry, -}; -use log::info; -use semver::Version; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::io::Read; -use std::sync::Arc; -use std::time::Instant; -use tokio::sync::{Mutex, Semaphore}; - -pub async fn retrieve_data( - minecraft_versions: &VersionManifest, - uploaded_files: &mut Vec, - semaphore: Arc, -) -> Result<(), Error> { - let maven_metadata = fetch_maven_metadata(semaphore.clone()).await?; - let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( - "neo/v{}/manifest.json", - daedalus::modded::CURRENT_NEOFORGE_FORMAT_VERSION, - ))) - .await - .ok(); - - let old_versions = - Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest { - old_manifest.game_versions - } else { - Vec::new() - })); - - let versions = Arc::new(Mutex::new(Vec::new())); - - let visited_assets_mutex = Arc::new(Mutex::new(Vec::new())); - let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); - - let mut version_futures = Vec::new(); - - for (minecraft_version, loader_versions) in maven_metadata.clone() { - let mut loaders = Vec::new(); - - for (full, loader_version, new_forge) in loader_versions { - let version = Version::parse(&loader_version)?; - - loaders.push((full, version, new_forge.to_string())) - } - - if !loaders.is_empty() { - version_futures.push(async { - let mut loaders_versions = Vec::new(); - - { - let loaders_futures = loaders.into_iter().map(|(loader_version_full, _, new_forge)| async { - let versions_mutex = Arc::clone(&old_versions); - let visited_assets = Arc::clone(&visited_assets_mutex); - let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex); - let semaphore = Arc::clone(&semaphore); - let minecraft_version = minecraft_version.clone(); - - async move { - { - let versions = versions_mutex.lock().await; - let version = versions.iter().find(|x| - x.id == minecraft_version).and_then(|x| x.loaders.iter().find(|x| x.id == loader_version_full)); - - if let Some(version) = version { - return Ok::, Error>(Some(version.clone())); - } - } - - info!("Forge - Installer Start {}", loader_version_full.clone()); - let bytes = download_file(&format!("https://maven.neoforged.net/net/neoforged/{1}/{0}/{1}-{0}-installer.jar", loader_version_full, if &*new_forge == "true" { "neoforge" } else { "forge" }), None, semaphore.clone()).await?; - - let reader = std::io::Cursor::new(bytes); - - if let Ok(archive) = zip::ZipArchive::new(reader) { - let mut archive_clone = archive.clone(); - let mut profile = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("install_profile.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&contents)?) - }).await??; - - let mut archive_clone = archive.clone(); - let version_info = tokio::task::spawn_blocking(move || { - let mut install_profile = archive_clone.by_name("version.json")?; - - let mut contents = String::new(); - install_profile.read_to_string(&mut contents)?; - - Ok::(serde_json::from_str::(&contents)?) - }).await??; - - - let mut libs : Vec = version_info.libraries.into_iter().chain(profile.libraries.into_iter().map(|x| Library { - downloads: x.downloads, - extract: x.extract, - name: x.name, - url: x.url, - natives: x.natives, - rules: x.rules, - checksums: x.checksums, - include_in_classpath: false - })).collect(); - - let mut local_libs : HashMap = HashMap::new(); - - for lib in &libs { - if lib.downloads.as_ref().and_then(|x| x.artifact.as_ref().map(|x| x.url.is_empty())).unwrap_or(false) { - let mut archive_clone = archive.clone(); - let lib_name_clone = lib.name.clone(); - - let lib_bytes = tokio::task::spawn_blocking(move || { - let mut lib_file = archive_clone.by_name(&format!("maven/{}", daedalus::get_path_from_artifact(&lib_name_clone)?))?; - let mut lib_bytes = Vec::new(); - lib_file.read_to_end(&mut lib_bytes)?; - - Ok::(bytes::Bytes::from(lib_bytes)) - }).await??; - - local_libs.insert(lib.name.clone(), lib_bytes); - } - } - - let path = profile.path.clone(); - let version = profile.version.clone(); - - for entry in profile.data.values_mut() { - if entry.client.starts_with('/') || entry.server.starts_with('/') { - macro_rules! read_data { - ($value:expr) => { - let mut archive_clone = archive.clone(); - let value_clone = $value.clone(); - let lib_bytes = tokio::task::spawn_blocking(move || { - let mut lib_file = archive_clone.by_name(&value_clone[1..value_clone.len()])?; - let mut lib_bytes = Vec::new(); - lib_file.read_to_end(&mut lib_bytes)?; - - Ok::(bytes::Bytes::from(lib_bytes)) - }).await??; - - let split = $value.split('/').last(); - - if let Some(last) = split { - let mut file = last.split('.'); - - if let Some(file_name) = file.next() { - if let Some(ext) = file.next() { - let path = format!("{}:{}@{}", path.as_deref().unwrap_or(&*format!("net.minecraftforge:forge:{}", version)), file_name, ext); - $value = format!("[{}]", &path); - local_libs.insert(path.clone(), bytes::Bytes::from(lib_bytes)); - - libs.push(Library { - downloads: None, - extract: None, - name: path, - url: Some("".to_string()), - natives: None, - rules: None, - checksums: None, - include_in_classpath: false, - }); - } - } - } - } - } - - if entry.client.starts_with('/') { - read_data!(entry.client); - } - - if entry.server.starts_with('/') { - read_data!(entry.server); - } - } - } - - let now = Instant::now(); - let libs = futures::future::try_join_all(libs.into_iter().map(|mut lib| async { - let artifact_path = - daedalus::get_path_from_artifact(&lib.name)?; - - { - let mut visited_assets = visited_assets.lock().await; - - if visited_assets.contains(&lib.name) { - if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - artifact.url = format_url(&format!("maven/{}", artifact_path)); - } - } else if lib.url.is_some() { - lib.url = Some(format_url("maven/")); - } - - return Ok::(lib); - } else { - visited_assets.push(lib.name.clone()) - } - } - - let artifact_bytes = if let Some(ref mut downloads) = lib.downloads { - if let Some(ref mut artifact) = downloads.artifact { - let res = if artifact.url.is_empty() { - local_libs.get(&lib.name).cloned() - } else { - Some(download_file( - &artifact.url, - Some(&*artifact.sha1), - semaphore.clone(), - ) - .await?) - }; - - if res.is_some() { - artifact.url = format_url(&format!("maven/{}", artifact_path)); - } - - res - } else { None } - } else if let Some(ref mut url) = lib.url { - let res = if url.is_empty() { - local_libs.get(&lib.name).cloned() - } else { - Some(download_file( - url, - None, - semaphore.clone(), - ) - .await?) - }; - - if res.is_some() { - lib.url = Some(format_url("maven/")); - } - - res - } else { None }; - - if let Some(bytes) = artifact_bytes { - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - bytes.to_vec(), - Some("application/java-archive".to_string()), - uploaded_files_mutex.as_ref(), - semaphore.clone(), - ).await?; - } - - Ok::(lib) - })).await?; - - let elapsed = now.elapsed(); - info!("Elapsed lib DL: {:.2?}", elapsed); - - let new_profile = PartialVersionInfo { - id: version_info.id, - inherits_from: version_info.inherits_from, - release_time: version_info.release_time, - time: version_info.time, - main_class: version_info.main_class, - minecraft_arguments: version_info.minecraft_arguments, - arguments: version_info.arguments, - libraries: libs, - type_: version_info.type_, - data: Some(profile.data), - processors: Some(profile.processors), - }; - - let version_path = format!( - "neo/v{}/versions/{}.json", - daedalus::modded::CURRENT_NEOFORGE_FORMAT_VERSION, - new_profile.id - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&new_profile)?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - semaphore.clone(), - ).await?; - - return Ok(Some(LoaderVersion { - id: loader_version_full, - url: format_url(&version_path), - stable: false - })); - } - - Ok(None) - }.await - }); - - { - let len = loaders_futures.len(); - let mut versions = loaders_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions.peek().is_some() { - let now = Instant::now(); - - let chunk: Vec<_> = versions.by_ref().take(1).collect(); - let res = futures::future::try_join_all(chunk).await?; - loaders_versions.extend(res.into_iter().flatten()); - - chunk_index += 1; - - let elapsed = now.elapsed(); - info!("Loader Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed); - } - } - } - - versions.lock().await.push(daedalus::modded::Version { - id: minecraft_version, - stable: true, - loaders: loaders_versions - }); - - Ok::<(), Error>(()) - }); - } - } - - { - let len = version_futures.len(); - let mut versions = version_futures.into_iter().peekable(); - let mut chunk_index = 0; - while versions.peek().is_some() { - let now = Instant::now(); - - let chunk: Vec<_> = versions.by_ref().take(1).collect(); - futures::future::try_join_all(chunk).await?; - - chunk_index += 1; - - let elapsed = now.elapsed(); - info!("Chunk {}/{len} Elapsed: {:.2?}", chunk_index, elapsed); - } - } - - if let Ok(versions) = Arc::try_unwrap(versions) { - let mut versions = versions.into_inner(); - - versions.sort_by(|x, y| { - minecraft_versions - .versions - .iter() - .position(|z| x.id == z.id) - .unwrap_or_default() - .cmp( - &minecraft_versions - .versions - .iter() - .position(|z| y.id == z.id) - .unwrap_or_default(), - ) - }); - - for version in &mut versions { - let loader_versions = maven_metadata.get(&version.id); - if let Some(loader_versions) = loader_versions { - version.loaders.sort_by(|x, y| { - loader_versions - .iter() - .position(|z| y.id == z.1) - .unwrap_or_default() - .cmp( - &loader_versions - .iter() - .position(|z| x.id == z.1) - .unwrap_or_default(), - ) - }); - version.loaders.reverse(); - } - } - - upload_file_to_bucket( - format!( - "neo/v{}/manifest.json", - daedalus::modded::CURRENT_NEOFORGE_FORMAT_VERSION, - ), - serde_json::to_vec(&Manifest { - game_versions: versions, - })?, - Some("application/json".to_string()), - uploaded_files_mutex.as_ref(), - semaphore, - ) - .await?; - } - - if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { - uploaded_files.extend(uploaded_files_mutex.into_inner()); - } - - Ok(()) -} - -const DEFAULT_MAVEN_METADATA_URL_1: &str = - "https://maven.neoforged.net/net/neoforged/forge/maven-metadata.xml"; -const DEFAULT_MAVEN_METADATA_URL_2: &str = - "https://maven.neoforged.net/net/neoforged/neoforge/maven-metadata.xml"; - -#[derive(Debug, Deserialize)] -struct Metadata { - versioning: Versioning, -} - -#[derive(Debug, Deserialize)] -struct Versioning { - versions: Versions, -} - -#[derive(Debug, Deserialize)] -struct Versions { - version: Vec, -} - -pub async fn fetch_maven_metadata( - semaphore: Arc, -) -> Result>, Error> { - async fn fetch_values( - url: &str, - semaphore: Arc, - ) -> Result { - Ok(serde_xml_rs::from_str( - &String::from_utf8( - download_file(url, None, semaphore).await?.to_vec(), - ) - .unwrap_or_default(), - )?) - } - - let forge_values = - fetch_values(DEFAULT_MAVEN_METADATA_URL_1, semaphore.clone()).await?; - let neo_values = - fetch_values(DEFAULT_MAVEN_METADATA_URL_2, semaphore).await?; - - let mut map: HashMap> = HashMap::new(); - - for value in forge_values.versioning.versions.version { - let original = value.clone(); - - let parts: Vec<&str> = value.split('-').collect(); - if parts.len() == 2 { - map.entry(parts[0].to_string()).or_default().push(( - original, - parts[1].to_string(), - false, - )); - } - } - - for value in neo_values.versioning.versions.version { - let original = value.clone(); - - let mut parts = value.split('.'); - - if let Some(major) = parts.next() { - if let Some(minor) = parts.next() { - let game_version = format!("1.{}.{}", major, minor); - - map.entry(game_version.clone()).or_default().push(( - original.clone(), - format!("{}-{}", game_version, original), - true, - )); - } - } - } - - Ok(map) -} - -#[derive(Serialize, Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -struct ForgeInstallerProfileV2 { - pub spec: i32, - pub profile: String, - pub version: String, - pub json: String, - pub path: Option, - pub minecraft: String, - pub data: HashMap, - pub libraries: Vec, - pub processors: Vec, -} diff --git a/daedalus_client/src/quilt.rs b/daedalus_client/src/quilt.rs deleted file mode 100644 index cd0df2da..00000000 --- a/daedalus_client/src/quilt.rs +++ /dev/null @@ -1,370 +0,0 @@ -use crate::{download_file, format_url, upload_file_to_bucket, Error}; -use daedalus::minecraft::{Library, VersionManifest}; -use daedalus::modded::{ - LoaderVersion, Manifest, PartialVersionInfo, Version, DUMMY_REPLACE_STRING, -}; -use serde::{Deserialize, Serialize}; -use std::sync::Arc; -use tokio::sync::{Mutex, RwLock, Semaphore}; - -pub async fn retrieve_data( - minecraft_versions: &VersionManifest, - uploaded_files: &mut Vec, - semaphore: Arc, -) -> Result<(), Error> { - let list = fetch_quilt_versions(None, semaphore.clone()).await?; - let old_manifest = daedalus::modded::fetch_manifest(&format_url(&format!( - "quilt/v{}/manifest.json", - daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, - ))) - .await - .ok(); - - let mut versions = if let Some(old_manifest) = old_manifest { - old_manifest.game_versions - } else { - Vec::new() - }; - - let loaders_mutex = RwLock::new(Vec::new()); - - { - let mut loaders = loaders_mutex.write().await; - - for (index, loader) in list.loader.iter().enumerate() { - if versions.iter().any(|x| { - x.id == DUMMY_REPLACE_STRING - && x.loaders.iter().any(|x| x.id == loader.version) - }) { - if index == 0 { - loaders.push(( - Box::new(false), - loader.version.clone(), - Box::new(true), - )) - } - } else { - loaders.push(( - Box::new(false), - loader.version.clone(), - Box::new(false), - )) - } - } - } - - const DUMMY_GAME_VERSION: &str = "1.19.4-rc2"; - - let loader_version_mutex = Mutex::new(Vec::new()); - let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new())); - - let loader_versions = futures::future::try_join_all( - loaders_mutex.read().await.clone().into_iter().map( - |(stable, loader, skip_upload)| async { - let version = fetch_quilt_version( - DUMMY_GAME_VERSION, - &loader, - semaphore.clone(), - ) - .await?; - - Ok::<(Box, String, PartialVersionInfo, Box), Error>( - (stable, loader, version, skip_upload), - ) - }, - ), - ) - .await?; - - let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new())); - futures::future::try_join_all(loader_versions.into_iter() - .map( - |(stable, loader, version, skip_upload)| async { - let libs = futures::future::try_join_all( - version.libraries.into_iter().map(|mut lib| async { - { - let mut visited_assets = - visited_artifacts_mutex.lock().await; - - if visited_assets.contains(&lib.name) { - lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); - lib.url = Some(format_url("maven/")); - - return Ok(lib); - } else { - visited_assets.push(lib.name.clone()) - } - } - - if lib.name.contains(DUMMY_GAME_VERSION) { - lib.name = lib.name.replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); - futures::future::try_join_all(list.game.clone().into_iter().map(|game_version| async { - let semaphore = semaphore.clone(); - let uploaded_files_mutex = uploaded_files_mutex.clone(); - let lib_name = lib.name.clone(); - let lib_url = lib.url.clone(); - - async move { - let artifact_path = - daedalus::get_path_from_artifact(&lib_name.replace(DUMMY_REPLACE_STRING, &game_version.version))?; - - let artifact = download_file( - &format!( - "{}{}", - lib_url.unwrap_or_else(|| { - "https://maven.quiltmc.org/".to_string() - }), - artifact_path - ), - None, - semaphore.clone(), - ) - .await?; - - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - &uploaded_files_mutex, - semaphore.clone(), - ) - .await?; - - Ok::<(), Error>(()) - }.await?; - - Ok::<(), Error>(()) - })).await?; - lib.url = Some(format_url("maven/")); - - return Ok(lib); - } - - let artifact_path = - daedalus::get_path_from_artifact(&lib.name)?; - - let artifact = download_file( - &format!( - "{}{}", - lib.url.unwrap_or_else(|| { - "https://maven.quiltmc.org/".to_string() - }), - artifact_path - ), - None, - semaphore.clone(), - ) - .await?; - - lib.url = Some(format_url("maven/")); - - upload_file_to_bucket( - format!("{}/{}", "maven", artifact_path), - artifact.to_vec(), - Some("application/java-archive".to_string()), - &uploaded_files_mutex, - semaphore.clone(), - ) - .await?; - - Ok::(lib) - }), - ) - .await?; - - if async move { - *skip_upload - }.await { - return Ok::<(), Error>(()) - } - - let version_path = format!( - "quilt/v{}/versions/{}.json", - daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, - &loader - ); - - upload_file_to_bucket( - version_path.clone(), - serde_json::to_vec(&PartialVersionInfo { - arguments: version.arguments, - id: version - .id - .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), - main_class: version.main_class, - release_time: version.release_time, - time: version.time, - type_: version.type_, - inherits_from: version - .inherits_from - .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING), - libraries: libs, - minecraft_arguments: version.minecraft_arguments, - processors: None, - data: None, - })?, - Some("application/json".to_string()), - &uploaded_files_mutex, - semaphore.clone(), - ) - .await?; - - { - let mut loader_version_map = loader_version_mutex.lock().await; - async move { - loader_version_map.push(LoaderVersion { - id: loader.to_string(), - url: format_url(&version_path), - stable: *stable, - }); - } - .await; - } - - Ok::<(), Error>(()) - }, - )) - .await?; - - let mut loader_version_mutex = loader_version_mutex.into_inner(); - if !loader_version_mutex.is_empty() { - if let Some(version) = - versions.iter_mut().find(|x| x.id == DUMMY_REPLACE_STRING) - { - version.loaders.append(&mut loader_version_mutex); - } else { - versions.push(Version { - id: DUMMY_REPLACE_STRING.to_string(), - stable: true, - loaders: loader_version_mutex, - }); - } - } - - for version in &list.game { - if !versions.iter().any(|x| x.id == version.version) { - versions.push(Version { - id: version.version.clone(), - stable: version.stable, - loaders: vec![], - }); - } - } - - versions.sort_by(|x, y| { - minecraft_versions - .versions - .iter() - .position(|z| x.id == z.id) - .unwrap_or_default() - .cmp( - &minecraft_versions - .versions - .iter() - .position(|z| y.id == z.id) - .unwrap_or_default(), - ) - }); - - for version in &mut versions { - version.loaders.sort_by(|x, y| { - list.loader - .iter() - .position(|z| x.id == *z.version) - .unwrap_or_default() - .cmp( - &list - .loader - .iter() - .position(|z| y.id == z.version) - .unwrap_or_default(), - ) - }) - } - - upload_file_to_bucket( - format!( - "quilt/v{}/manifest.json", - daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, - ), - serde_json::to_vec(&Manifest { - game_versions: versions, - })?, - Some("application/json".to_string()), - &uploaded_files_mutex, - semaphore, - ) - .await?; - - if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) { - uploaded_files.extend(uploaded_files_mutex.into_inner()); - } - - Ok(()) -} - -const QUILT_META_URL: &str = "https://meta.quiltmc.org/v3"; - -async fn fetch_quilt_version( - version_number: &str, - loader_version: &str, - semaphore: Arc, -) -> Result { - Ok(serde_json::from_slice( - &download_file( - &format!( - "{}/versions/loader/{}/{}/profile/json", - QUILT_META_URL, version_number, loader_version - ), - None, - semaphore, - ) - .await?, - )?) -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -/// Versions of quilt components -struct QuiltVersions { - /// Versions of Minecraft that quilt supports - pub game: Vec, - /// Available versions of the quilt loader - pub loader: Vec, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -/// A version of Minecraft that quilt supports -struct QuiltGameVersion { - /// The version number of the game - pub version: String, - /// Whether the Minecraft version is stable or not - pub stable: bool, -} - -#[derive(Serialize, Deserialize, Debug, Clone)] -/// A version of the quilt loader -struct QuiltLoaderVersion { - /// The separator to get the build number - pub separator: String, - /// The build number - pub build: u32, - /// The maven artifact - pub maven: String, - /// The version number of the quilt loader - pub version: String, -} - -/// Fetches the list of quilt versions -async fn fetch_quilt_versions( - url: Option<&str>, - semaphore: Arc, -) -> Result { - Ok(serde_json::from_slice( - &download_file( - url.unwrap_or(&*format!("{}/versions", QUILT_META_URL)), - None, - semaphore, - ) - .await?, - )?) -} diff --git a/daedalus_client/src/util.rs b/daedalus_client/src/util.rs new file mode 100644 index 00000000..0b571f0c --- /dev/null +++ b/daedalus_client/src/util.rs @@ -0,0 +1,369 @@ +use crate::{Error, ErrorKind}; +use bytes::{Bytes, BytesMut}; +use futures::StreamExt; +use s3::creds::Credentials; +use s3::{Bucket, Region}; +use serde::de::DeserializeOwned; +use std::sync::Arc; +use tokio::sync::Semaphore; + +lazy_static::lazy_static! { + static ref BUCKET : Bucket = { + let region = dotenvy::var("S3_REGION").unwrap(); + let b = Bucket::new( + &dotenvy::var("S3_BUCKET_NAME").unwrap(), + if &*region == "r2" { + Region::R2 { + account_id: dotenvy::var("S3_URL").unwrap(), + } + } else { + Region::Custom { + region: region.clone(), + endpoint: dotenvy::var("S3_URL").unwrap(), + } + }, + Credentials::new( + Some(&*dotenvy::var("S3_ACCESS_TOKEN").unwrap()), + Some(&*dotenvy::var("S3_SECRET").unwrap()), + None, + None, + None, + ).unwrap(), + ).unwrap(); + + if region == "path-style" { + b.with_path_style() + } else { + b + } + }; +} + +lazy_static::lazy_static! { + pub static ref REQWEST_CLIENT: reqwest::Client = { + let mut headers = reqwest::header::HeaderMap::new(); + if let Ok(header) = reqwest::header::HeaderValue::from_str(&format!( + "modrinth/daedalus/{} (support@modrinth.com)", + env!("CARGO_PKG_VERSION") + )) { + headers.insert(reqwest::header::USER_AGENT, header); + } + + reqwest::Client::builder() + .tcp_keepalive(Some(std::time::Duration::from_secs(10))) + .timeout(std::time::Duration::from_secs(15)) + .default_headers(headers) + .build() + .unwrap() + }; +} + +#[tracing::instrument(skip(bytes, semaphore))] +pub async fn upload_file_to_bucket( + path: String, + bytes: Bytes, + content_type: Option, + semaphore: &Arc, +) -> Result<(), Error> { + let _permit = semaphore.acquire().await?; + let key = path.clone(); + + const RETRIES: i32 = 3; + for attempt in 1..=(RETRIES + 1) { + tracing::trace!("Attempting file upload, attempt {attempt}"); + let result = if let Some(ref content_type) = content_type { + BUCKET + .put_object_with_content_type(key.clone(), &bytes, content_type) + .await + } else { + BUCKET.put_object(key.clone(), &bytes).await + } + .map_err(|err| ErrorKind::S3 { + inner: err, + file: path.clone(), + }); + + match result { + Ok(_) => return Ok(()), + Err(_) if attempt <= RETRIES => continue, + Err(_) => { + result?; + } + } + } + unreachable!() +} + +pub async fn upload_url_to_bucket_mirrors( + base: String, + mirrors: Vec, + semaphore: &Arc, +) -> Result<(), Error> { + if mirrors.is_empty() { + return Err(ErrorKind::InvalidInput( + "No mirrors provided!".to_string(), + ) + .into()); + } + + for (index, mirror) in mirrors.iter().enumerate() { + let result = upload_url_to_bucket( + &base, + &format!("{}{}", mirror, base), + semaphore, + ) + .await; + + if result.is_ok() || (result.is_err() && index == (mirrors.len() - 1)) { + return result; + } + } + + unreachable!() +} + +#[tracing::instrument(skip(semaphore))] +pub async fn upload_url_to_bucket( + path: &str, + url: &str, + semaphore: &Arc, +) -> Result<(), Error> { + let _permit = semaphore.acquire().await?; + + const RETRIES: i32 = 3; + for attempt in 1..=(RETRIES + 1) { + tracing::trace!("Attempting streaming file upload, attempt {attempt}"); + + let result: Result<(), Error> = { + let response = + REQWEST_CLIENT.get(url).send().await.map_err(|err| { + ErrorKind::Fetch { + inner: err, + item: url.to_string(), + } + })?; + + let content_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|ct| ct.to_str().ok()) + .unwrap_or("application/octet-stream") + .to_string(); + + let total_size = response.content_length().unwrap_or(0); + + const MIN_PART_SIZE: usize = 5 * 1024 * 1024; + + if total_size < MIN_PART_SIZE as u64 { + let data = + response.bytes().await.map_err(|err| ErrorKind::Fetch { + inner: err, + item: url.to_string(), + })?; + BUCKET.put_object(&path, &data).await.map_err(|err| { + ErrorKind::S3 { + inner: err, + file: path.to_string(), + } + })?; + } else { + let mut stream = response.bytes_stream(); + + let multipart = BUCKET + .initiate_multipart_upload(path, &content_type) + .await + .map_err(|err| ErrorKind::S3 { + inner: err, + file: path.to_string(), + })?; + + let mut parts = Vec::new(); + let mut buffer = BytesMut::new(); + + async fn upload_part( + parts: &mut Vec, + buffer: Vec, + path: &str, + upload_id: &str, + content_type: &str, + ) -> Result<(), Error> { + let part = BUCKET + .put_multipart_chunk( + buffer, + path, + (parts.len() + 1) as u32, + upload_id, + content_type, + ) + .await + .map_err(|err| ErrorKind::S3 { + inner: err, + file: path.to_string(), + })?; + + parts.push(part); + + Ok(()) + } + + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|err| ErrorKind::Fetch { + inner: err, + item: url.to_string(), + })?; + + buffer.extend_from_slice(&chunk); + + if buffer.len() >= MIN_PART_SIZE { + upload_part( + &mut parts, + buffer.to_vec(), + path, + &multipart.upload_id, + &content_type, + ) + .await?; + buffer.clear(); + } + } + + if !buffer.is_empty() { + let part = BUCKET + .put_multipart_chunk( + buffer.to_vec(), + path, + (parts.len() + 1) as u32, + &multipart.upload_id, + &content_type, + ) + .await + .map_err(|err| ErrorKind::S3 { + inner: err, + file: path.to_string(), + })?; + + parts.push(part); + } + + BUCKET + .complete_multipart_upload( + path, + &multipart.upload_id, + parts, + ) + .await + .map_err(|err| ErrorKind::S3 { + inner: err, + file: path.to_string(), + })?; + } + + Ok(()) + }; + + match result { + Ok(_) => return Ok(()), + Err(_) if attempt <= RETRIES => continue, + Err(_) => { + result?; + } + } + } + unreachable!() +} + +#[tracing::instrument(skip(bytes))] +pub async fn sha1_async(bytes: Bytes) -> Result { + let hash = tokio::task::spawn_blocking(move || { + sha1_smol::Sha1::from(bytes).hexdigest() + }) + .await?; + + Ok(hash) +} + +#[tracing::instrument(skip(semaphore))] +pub async fn download_file( + url: &str, + sha1: Option<&str>, + semaphore: &Arc, +) -> Result { + let _permit = semaphore.acquire().await?; + tracing::trace!("Starting file download"); + + const RETRIES: u32 = 10; + for attempt in 1..=(RETRIES + 1) { + let result = REQWEST_CLIENT + .get(url) + .send() + .await + .and_then(|x| x.error_for_status()); + + match result { + Ok(x) => { + let bytes = x.bytes().await; + + if let Ok(bytes) = bytes { + if let Some(sha1) = sha1 { + if &*sha1_async(bytes.clone()).await? != sha1 { + if attempt <= 3 { + continue; + } else { + return Err( + crate::ErrorKind::ChecksumFailure { + hash: sha1.to_string(), + url: url.to_string(), + tries: attempt, + } + .into(), + ); + } + } + } + + return Ok(bytes); + } else if attempt <= RETRIES { + continue; + } else if let Err(err) = bytes { + return Err(crate::ErrorKind::Fetch { + inner: err, + item: url.to_string(), + } + .into()); + } + } + Err(_) if attempt <= RETRIES => continue, + Err(err) => { + return Err(crate::ErrorKind::Fetch { + inner: err, + item: url.to_string(), + } + .into()) + } + } + } + + unreachable!() +} + +pub async fn fetch_json( + url: &str, + semaphore: &Arc, +) -> Result { + Ok(serde_json::from_slice( + &download_file(url, None, semaphore).await?, + )?) +} + +pub async fn fetch_xml( + url: &str, + semaphore: &Arc, +) -> Result { + Ok(serde_xml_rs::from_reader( + &*download_file(url, None, semaphore).await?, + )?) +} + +pub fn format_url(path: &str) -> String { + format!("{}/{}", &*dotenvy::var("BASE_URL").unwrap(), path) +} From 4274a8ed6840129fb0caf1e28f9c5da02d97d34b Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Fri, 28 Jun 2024 15:44:17 -0700 Subject: [PATCH 62/76] Fix forge install issues (#18) * Fix forge install issues * remove mac garb --- daedalus/Cargo.toml | 2 +- daedalus_client/Cargo.toml | 2 +- daedalus_client/src/fabric.rs | 34 +++++-- daedalus_client/src/forge.rs | 68 +++++++++----- daedalus_client/src/main.rs | 109 +++++++++++++++-------- daedalus_client/src/util.rs | 161 +++------------------------------- 6 files changed, 162 insertions(+), 214 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 1acc5253..82e16108 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.2.0" +version = "0.2.1" authors = ["Jai A "] edition = "2021" license = "MIT" diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index ce8204fa..c0bdbd2b 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.2.0" +version = "0.2.1" authors = ["Jai A "] edition = "2021" diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index 8d1d9431..a7e352b2 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -17,6 +17,7 @@ pub async fn fetch_fabric( "fabric", "https://meta.fabricmc.net/v2", "https://maven.fabricmc.net/", + &[], semaphore, upload_files, mirror_artifacts, @@ -34,7 +35,11 @@ pub async fn fetch_quilt( daedalus::modded::CURRENT_QUILT_FORMAT_VERSION, "quilt", "https://meta.quiltmc.org/v3", - "https://meta.quiltmc.org/", + "https://maven.quiltmc.org/repository/release/", + &[ + // This version is broken as it contains invalid library coordinates + "0.17.5-beta.4", + ], semaphore, upload_files, mirror_artifacts, @@ -48,6 +53,7 @@ async fn fetch( mod_loader: &str, meta_url: &str, maven_url: &str, + skip_versions: &[&str], semaphore: Arc, upload_files: &DashMap, mirror_artifacts: &DashMap, @@ -76,6 +82,7 @@ async fn fetch( .game_versions .iter() .any(|x| x.loaders.iter().any(|x| x.id == version.version)) + && !skip_versions.contains(&&*version.version) { fetch_versions.push(version); } @@ -98,7 +105,11 @@ async fn fetch( (fetch_versions, fetch_intermediary_versions) } else { ( - fabric_manifest.loader.iter().collect(), + fabric_manifest + .loader + .iter() + .filter(|x| !skip_versions.contains(&&*x.version)) + .collect(), fabric_manifest.intermediary.iter().collect(), ) }; @@ -109,7 +120,9 @@ async fn fetch( for x in &fetch_intermediary_versions { insert_mirrored_artifact( &x.maven, - maven_url.to_string(), + None, + vec![maven_url.to_string()], + false, mirror_artifacts, )?; } @@ -142,13 +155,24 @@ async fn fetch( let new_name = lib .name .replace(DUMMY_GAME_VERSION, DUMMY_REPLACE_STRING); + + // Hard-code: This library is not present on fabric's maven, so we fetch it from MC libraries + if &*lib.name == "net.minecraft:launchwrapper:1.12" { + lib.url = Some( + "https://libraries.minecraft.net/".to_string(), + ); + } + // If a library is not intermediary, we add it to mirror artifacts to be mirrored if lib.name == new_name { insert_mirrored_artifact( &new_name, - lib.url + None, + vec![lib + .url .clone() - .unwrap_or_else(|| maven_url.to_string()), + .unwrap_or_else(|| maven_url.to_string())], + false, mirror_artifacts, )?; } else { diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 99892f22..09c8e0aa 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -1,4 +1,6 @@ -use crate::util::{download_file, fetch_json, fetch_xml, format_url}; +use crate::util::{ + download_file, fetch_json, fetch_xml, format_url, sha1_async, +}; use crate::{insert_mirrored_artifact, Error, MirrorArtifact, UploadFile}; use chrono::{DateTime, Utc}; use daedalus::get_path_from_artifact; @@ -246,6 +248,7 @@ async fn fetch( raw: bytes::Bytes, loader: &ForgeVersion, maven_url: &str, + mod_loader: &str, upload_files: &DashMap, mirror_artifacts: &DashMap, ) -> Result { @@ -399,15 +402,27 @@ async fn fetch( .into_iter() .map(|mut lib| { // For all libraries besides the forge lib extracted, we mirror them from maven servers - if lib.name != install_profile.install.path { - // TODO: add mirrors "https://maven.creeperhost.net/", "https://libraries.minecraft.net/" - insert_mirrored_artifact( - &lib.name, - lib.url.clone().unwrap_or_else(|| { - maven_url.to_string() - }), - mirror_artifacts, - )?; + // unless the URL is empty/null or available on Minecraft's servers + if let Some(url) = lib.url { + if lib.name != install_profile.install.path + && !url.is_empty() + && !url.contains( + "https://libraries.minecraft.net/", + ) + { + insert_mirrored_artifact( + &lib.name, + None, + vec![ + url, + "https://maven.creeperhost.net/" + .to_string(), + maven_url.to_string(), + ], + false, + mirror_artifacts, + )?; + } } lib.url = Some(format_url("maven/")); @@ -468,6 +483,7 @@ async fn fetch( async fn mirror_forge_library( mut zip: ZipFileReader, mut lib: daedalus::minecraft::Library, + maven_url: &str, upload_files: &DashMap, mirror_artifacts: &DashMap, ) -> Result @@ -480,7 +496,9 @@ async fn fetch( if !artifact.url.is_empty() { insert_mirrored_artifact( &lib.name, - artifact.url.clone(), + Some(artifact.sha1.clone()), + vec![artifact.url.clone()], + true, mirror_artifacts, )?; @@ -491,10 +509,18 @@ async fn fetch( } } else if let Some(url) = &lib.url { if !url.is_empty() { - // TODO: add mirrors "https://maven.creeperhost.net/", "https://libraries.minecraft.net/" insert_mirrored_artifact( &lib.name, - url.clone(), + None, + vec![ + url.clone(), + "https://libraries.minecraft.net/" + .to_string(), + "https://maven.creeperhost.net/" + .to_string(), + maven_url.to_string(), + ], + false, mirror_artifacts, )?; @@ -531,6 +557,7 @@ async fn fetch( mirror_forge_library( zip.clone(), lib, + maven_url, upload_files, mirror_artifacts, ) @@ -560,7 +587,7 @@ async fn fetch( value: &str, upload_files: &DashMap, libs: &mut Vec, - install_profile_path: Option<&str>, + mod_loader: &str, version: &ForgeVersion, ) -> Result { let extract_file = @@ -595,11 +622,9 @@ async fn fetch( })?; let path = format!( - "{}:{}@{}", - install_profile_path.unwrap_or(&*format!( - "net.minecraftforge:forge:{}", - version.raw - )), + "com.modrinth.daedalus:{}-installer-extracts:{}:{}@{}", + mod_loader, + version.raw, file_name, ext ); @@ -634,7 +659,7 @@ async fn fetch( &entry.client, upload_files, &mut version_info.libraries, - install_profile.path.as_deref(), + mod_loader, loader, ) .await? @@ -649,7 +674,7 @@ async fn fetch( &entry.server, upload_files, &mut version_info.libraries, - install_profile.path.as_deref(), + mod_loader, loader, ) .await? @@ -686,6 +711,7 @@ async fn fetch( raw, loader, maven_url, + mod_loader, upload_files, mirror_artifacts, ) diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 33df7090..8e934cbd 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -72,46 +72,63 @@ async fn main() -> Result<()> { futures::future::try_join_all(mirror_artifacts.iter().map(|x| { upload_url_to_bucket_mirrors( format!("maven/{}", x.key()), - x.value().mirrors.iter().map(|x| x.key().clone()).collect(), + x.value() + .mirrors + .iter() + .map(|mirror| { + if mirror.entire_url { + mirror.path.clone() + } else { + format!("{}{}", mirror.path, x.key()) + } + }) + .collect(), + x.sha1.clone(), &semaphore, ) })) .await?; - if let Ok(token) = dotenvy::var("CLOUDFLARE_TOKEN") { - if let Ok(zone_id) = dotenvy::var("CLOUDFLARE_ZONE_ID") { - let cache_clears = upload_files - .into_iter() - .map(|x| format_url(&x.0)) - .chain( - mirror_artifacts - .into_iter() - .map(|x| format_url(&format!("maven/{}", x.0))), - ) - .collect::>(); + if dotenvy::var("CLOUDFLARE_INTEGRATION") + .ok() + .and_then(|x| x.parse::().ok()) + .unwrap_or(false) + { + if let Ok(token) = dotenvy::var("CLOUDFLARE_TOKEN") { + if let Ok(zone_id) = dotenvy::var("CLOUDFLARE_ZONE_ID") { + let cache_clears = upload_files + .into_iter() + .map(|x| format_url(&x.0)) + .chain( + mirror_artifacts + .into_iter() + .map(|x| format_url(&format!("maven/{}", x.0))), + ) + .collect::>(); - // Cloudflare ratelimits cache clears to 500 files per request - for chunk in cache_clears.chunks(500) { - REQWEST_CLIENT.post(format!("https://api.cloudflare.com/client/v4/zones/{zone_id}/purge_cache")) - .bearer_auth(&token) - .json(&serde_json::json!({ + // Cloudflare ratelimits cache clears to 500 files per request + for chunk in cache_clears.chunks(500) { + REQWEST_CLIENT.post(format!("https://api.cloudflare.com/client/v4/zones/{zone_id}/purge_cache")) + .bearer_auth(&token) + .json(&serde_json::json!({ "files": chunk })) - .send() - .await - .map_err(|err| { - ErrorKind::Fetch { - inner: err, - item: "cloudflare clear cache".to_string(), - } - })? - .error_for_status() - .map_err(|err| { - ErrorKind::Fetch { - inner: err, - item: "cloudflare clear cache".to_string(), - } - })?; + .send() + .await + .map_err(|err| { + ErrorKind::Fetch { + inner: err, + item: "cloudflare clear cache".to_string(), + } + })? + .error_for_status() + .map_err(|err| { + ErrorKind::Fetch { + inner: err, + item: "cloudflare clear cache".to_string(), + } + })?; + } } } } @@ -125,21 +142,37 @@ pub struct UploadFile { } pub struct MirrorArtifact { - pub mirrors: DashSet, + pub sha1: Option, + pub mirrors: DashSet, } +#[derive(Eq, PartialEq, Hash)] +pub struct Mirror { + path: String, + entire_url: bool, +} + +#[tracing::instrument(skip(mirror_artifacts))] pub fn insert_mirrored_artifact( artifact: &str, - mirror: String, + sha1: Option, + mirrors: Vec, + entire_url: bool, mirror_artifacts: &DashMap, ) -> Result<()> { - mirror_artifacts + let mut val = mirror_artifacts .entry(get_path_from_artifact(artifact)?) .or_insert(MirrorArtifact { + sha1, mirrors: DashSet::new(), - }) - .mirrors - .insert(mirror); + }); + + for mirror in mirrors { + val.mirrors.insert(Mirror { + path: mirror, + entire_url, + }); + } Ok(()) } diff --git a/daedalus_client/src/util.rs b/daedalus_client/src/util.rs index 0b571f0c..72ff2557 100644 --- a/daedalus_client/src/util.rs +++ b/daedalus_client/src/util.rs @@ -1,6 +1,5 @@ use crate::{Error, ErrorKind}; -use bytes::{Bytes, BytesMut}; -use futures::StreamExt; +use bytes::Bytes; use s3::creds::Credentials; use s3::{Bucket, Region}; use serde::de::DeserializeOwned; @@ -95,8 +94,9 @@ pub async fn upload_file_to_bucket( } pub async fn upload_url_to_bucket_mirrors( - base: String, + upload_path: String, mirrors: Vec, + sha1: Option, semaphore: &Arc, ) -> Result<(), Error> { if mirrors.is_empty() { @@ -108,8 +108,9 @@ pub async fn upload_url_to_bucket_mirrors( for (index, mirror) in mirrors.iter().enumerate() { let result = upload_url_to_bucket( - &base, - &format!("{}{}", mirror, base), + upload_path.clone(), + mirror.clone(), + sha1.clone(), semaphore, ) .await; @@ -124,152 +125,16 @@ pub async fn upload_url_to_bucket_mirrors( #[tracing::instrument(skip(semaphore))] pub async fn upload_url_to_bucket( - path: &str, - url: &str, + path: String, + url: String, + sha1: Option, semaphore: &Arc, ) -> Result<(), Error> { - let _permit = semaphore.acquire().await?; + let data = download_file(&url, sha1.as_deref(), semaphore).await?; - const RETRIES: i32 = 3; - for attempt in 1..=(RETRIES + 1) { - tracing::trace!("Attempting streaming file upload, attempt {attempt}"); + upload_file_to_bucket(path, data, None, semaphore).await?; - let result: Result<(), Error> = { - let response = - REQWEST_CLIENT.get(url).send().await.map_err(|err| { - ErrorKind::Fetch { - inner: err, - item: url.to_string(), - } - })?; - - let content_type = response - .headers() - .get(reqwest::header::CONTENT_TYPE) - .and_then(|ct| ct.to_str().ok()) - .unwrap_or("application/octet-stream") - .to_string(); - - let total_size = response.content_length().unwrap_or(0); - - const MIN_PART_SIZE: usize = 5 * 1024 * 1024; - - if total_size < MIN_PART_SIZE as u64 { - let data = - response.bytes().await.map_err(|err| ErrorKind::Fetch { - inner: err, - item: url.to_string(), - })?; - BUCKET.put_object(&path, &data).await.map_err(|err| { - ErrorKind::S3 { - inner: err, - file: path.to_string(), - } - })?; - } else { - let mut stream = response.bytes_stream(); - - let multipart = BUCKET - .initiate_multipart_upload(path, &content_type) - .await - .map_err(|err| ErrorKind::S3 { - inner: err, - file: path.to_string(), - })?; - - let mut parts = Vec::new(); - let mut buffer = BytesMut::new(); - - async fn upload_part( - parts: &mut Vec, - buffer: Vec, - path: &str, - upload_id: &str, - content_type: &str, - ) -> Result<(), Error> { - let part = BUCKET - .put_multipart_chunk( - buffer, - path, - (parts.len() + 1) as u32, - upload_id, - content_type, - ) - .await - .map_err(|err| ErrorKind::S3 { - inner: err, - file: path.to_string(), - })?; - - parts.push(part); - - Ok(()) - } - - while let Some(chunk) = stream.next().await { - let chunk = chunk.map_err(|err| ErrorKind::Fetch { - inner: err, - item: url.to_string(), - })?; - - buffer.extend_from_slice(&chunk); - - if buffer.len() >= MIN_PART_SIZE { - upload_part( - &mut parts, - buffer.to_vec(), - path, - &multipart.upload_id, - &content_type, - ) - .await?; - buffer.clear(); - } - } - - if !buffer.is_empty() { - let part = BUCKET - .put_multipart_chunk( - buffer.to_vec(), - path, - (parts.len() + 1) as u32, - &multipart.upload_id, - &content_type, - ) - .await - .map_err(|err| ErrorKind::S3 { - inner: err, - file: path.to_string(), - })?; - - parts.push(part); - } - - BUCKET - .complete_multipart_upload( - path, - &multipart.upload_id, - parts, - ) - .await - .map_err(|err| ErrorKind::S3 { - inner: err, - file: path.to_string(), - })?; - } - - Ok(()) - }; - - match result { - Ok(_) => return Ok(()), - Err(_) if attempt <= RETRIES => continue, - Err(_) => { - result?; - } - } - } - unreachable!() + Ok(()) } #[tracing::instrument(skip(bytes))] @@ -294,7 +159,7 @@ pub async fn download_file( const RETRIES: u32 = 10; for attempt in 1..=(RETRIES + 1) { let result = REQWEST_CLIENT - .get(url) + .get(&url.replace("http://", "https://")) .send() .await .and_then(|x| x.error_for_status()); From fc3056b0e011e65ef83a851fa655f26061e16dd4 Mon Sep 17 00:00:00 2001 From: Jai A Date: Fri, 28 Jun 2024 15:53:59 -0700 Subject: [PATCH 63/76] add rust log env --- .github/workflows/run.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index b97d2bdd..c5f0b827 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -37,6 +37,7 @@ jobs: run: | docker run -d \ --name daedalus \ + -e RUST_LOG=warn,daedalus_client=trace -e BASE_URL=$BASE_URL \ -e S3_ACCESS_TOKEN=$S3_ACCESS_TOKEN \ -e S3_SECRET=$S3_SECRET \ From 6e8e053b88c1370ca8862d7508fce1ffa1ef0164 Mon Sep 17 00:00:00 2001 From: Jai A Date: Fri, 28 Jun 2024 16:40:12 -0700 Subject: [PATCH 64/76] fix newline --- .github/workflows/run.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index c5f0b827..dbd8b847 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -37,7 +37,7 @@ jobs: run: | docker run -d \ --name daedalus \ - -e RUST_LOG=warn,daedalus_client=trace + -e RUST_LOG=warn,daedalus_client=trace \ -e BASE_URL=$BASE_URL \ -e S3_ACCESS_TOKEN=$S3_ACCESS_TOKEN \ -e S3_SECRET=$S3_SECRET \ From 02ebe59d2f30e6a568ba9246c451d2f3ce67acea Mon Sep 17 00:00:00 2001 From: Jai A Date: Fri, 28 Jun 2024 16:45:06 -0700 Subject: [PATCH 65/76] add dispatch support for meta --- .github/workflows/run.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index dbd8b847..e01d317a 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -3,6 +3,7 @@ name: Run Meta on: schedule: - cron: '*/5 * * * *' + workflow_dispatch: jobs: run-docker: From 8a0329b23da5e81492622183b3a8ebade21d5cbc Mon Sep 17 00:00:00 2001 From: Jai A Date: Fri, 28 Jun 2024 16:48:03 -0700 Subject: [PATCH 66/76] don't run in detached mode --- .github/workflows/run.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index e01d317a..4255e222 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -36,7 +36,7 @@ jobs: CLOUDFLARE_TOKEN: ${{ secrets.CLOUDFLARE_TOKEN }} CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }} run: | - docker run -d \ + docker run \ --name daedalus \ -e RUST_LOG=warn,daedalus_client=trace \ -e BASE_URL=$BASE_URL \ From 88db79188c1f773985e63ff7506eb8637e8a2a79 Mon Sep 17 00:00:00 2001 From: Jai A Date: Fri, 28 Jun 2024 16:50:47 -0700 Subject: [PATCH 67/76] use rustls --- .github/workflows/run.yml | 2 +- daedalus/Cargo.toml | 2 +- daedalus_client/Cargo.toml | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 4255e222..fa9e51a9 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -48,4 +48,4 @@ jobs: -e CLOUDFLARE_INTEGRATION=$CLOUDFLARE_INTEGRATION \ -e CLOUDFLARE_TOKEN=$CLOUDFLARE_TOKEN \ -e CLOUDFLARE_ZONE_ID=$CLOUDFLARE_ZONE_ID \ - ghcr.io/modrinth/daedalus:latest \ No newline at end of file + ghcr.io/modrinth/daedalus:latest diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 82e16108..1b28e72e 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.2.1" +version = "0.2.2" authors = ["Jai A "] edition = "2021" license = "MIT" diff --git a/daedalus_client/Cargo.toml b/daedalus_client/Cargo.toml index c0bdbd2b..de3b52c9 100644 --- a/daedalus_client/Cargo.toml +++ b/daedalus_client/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus_client" -version = "0.2.1" +version = "0.2.2" authors = ["Jai A "] edition = "2021" @@ -16,7 +16,7 @@ serde_json = "1.0" serde-xml-rs = "0.6.0" lazy_static = "1.4.0" thiserror = "1.0" -reqwest = { version = "0.12.5", features = ["stream", "json"] } +reqwest = { version = "0.12.5", features = ["stream", "json", "rustls-tls"] } async_zip = { version = "0.0.17", features = ["full"] } semver = "1.0" chrono = { version = "0.4", features = ["serde"] } From 7be02318e029c8e73d51535108549a6c00b0f98c Mon Sep 17 00:00:00 2001 From: Jai A Date: Fri, 28 Jun 2024 16:55:54 -0700 Subject: [PATCH 68/76] add openssl --- .github/workflows/run.yml | 2 +- Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index fa9e51a9..e50497a5 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -22,7 +22,7 @@ jobs: - name: Pull Docker image from GHCR - run: docker pull ghcr.io/modrinth/daedalus:latest + run: docker pull ghcr.io/modrinth/daedalus:master - name: Run Docker container env: diff --git a/Dockerfile b/Dockerfile index c015f674..a54439de 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ RUN cargo build --release FROM debian:bullseye-slim RUN apt-get update \ - && apt-get install -y --no-install-recommends ca-certificates \ + && apt-get install -y --no-install-recommends ca-certificates openssl \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* From 721365578ae72a4e479bfd7660877d51c27303ea Mon Sep 17 00:00:00 2001 From: Jai A Date: Fri, 28 Jun 2024 16:59:06 -0700 Subject: [PATCH 69/76] update action to use master branch --- .github/workflows/run.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index e50497a5..1409fb47 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -48,4 +48,4 @@ jobs: -e CLOUDFLARE_INTEGRATION=$CLOUDFLARE_INTEGRATION \ -e CLOUDFLARE_TOKEN=$CLOUDFLARE_TOKEN \ -e CLOUDFLARE_ZONE_ID=$CLOUDFLARE_ZONE_ID \ - ghcr.io/modrinth/daedalus:latest + ghcr.io/modrinth/daedalus:master From 34d931573cac307b3685d3dcc9f47cf88f347ff1 Mon Sep 17 00:00:00 2001 From: Jai A Date: Fri, 28 Jun 2024 17:02:30 -0700 Subject: [PATCH 70/76] try with bookworm --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a54439de..c926fa8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ COPY . . RUN cargo build --release -FROM debian:bullseye-slim +FROM debian:bookwormg-slim RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates openssl \ From 0ae1e40d79685a045360acf9a76ee38849392502 Mon Sep 17 00:00:00 2001 From: Jai A Date: Fri, 28 Jun 2024 17:03:54 -0700 Subject: [PATCH 71/76] fix typo --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c926fa8d..19c88feb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ COPY . . RUN cargo build --release -FROM debian:bookwormg-slim +FROM debian:bookworm-slim RUN apt-get update \ && apt-get install -y --no-install-recommends ca-certificates openssl \ From 9763a43943f3a97f52c572f6fb47a000f3eba227 Mon Sep 17 00:00:00 2001 From: Norbiros Date: Wed, 31 Jul 2024 22:14:30 +0200 Subject: [PATCH 72/76] fix: Correctly replace linux natives for LWJGL 3.3.1 (#20) --- daedalus_client/library-patches.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/daedalus_client/library-patches.json b/daedalus_client/library-patches.json index 268341a9..0acd32e0 100644 --- a/daedalus_client/library-patches.json +++ b/daedalus_client/library-patches.json @@ -2863,7 +2863,8 @@ { "_comment": "Replace glfw from 3.3.1 with version from 3.3.2 to prevent stack smashing", "match": [ - "org.lwjgl:lwjgl-glfw-natives-linux:3.3.1" + "org.lwjgl:lwjgl-glfw-natives-linux:3.3.1", + "org.lwjgl:lwjgl-glfw:3.3.1:natives-linux" ], "override": { "downloads": { @@ -2876,4 +2877,4 @@ "name": "org.lwjgl:lwjgl-glfw-natives-linux:3.3.2-lwjgl.1" } } -] \ No newline at end of file +] From 6de8d2684adc7d699daa4c9f3b944b84cb8c4528 Mon Sep 17 00:00:00 2001 From: Norbiros Date: Wed, 31 Jul 2024 22:14:39 +0200 Subject: [PATCH 73/76] fix: Disable `Run Meta` on forks (#19) * fix: Disable `Run Meta` on forks * fix: Also don't run `docker` action --- .github/workflows/docker.yml | 1 + .github/workflows/run.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fb101422..274085ff 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -9,6 +9,7 @@ on: pull_request: jobs: docker: + if: github.repository_owner == 'modrinth' runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/run.yml b/.github/workflows/run.yml index 1409fb47..635c6119 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/run.yml @@ -7,6 +7,7 @@ on: jobs: run-docker: + if: github.repository_owner == 'modrinth' runs-on: ubuntu-latest steps: From 93ae24e7075f992e0855bd7875f437507e51cd12 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sat, 17 Aug 2024 13:28:00 -0700 Subject: [PATCH 74/76] Fix forge format version 1 --- daedalus/src/minecraft.rs | 2 -- daedalus_client/src/forge.rs | 17 +++++++++-------- daedalus_client/src/main.rs | 2 +- daedalus_client/src/util.rs | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/daedalus/src/minecraft.rs b/daedalus/src/minecraft.rs index 021c91ab..e117052a 100644 --- a/daedalus/src/minecraft.rs +++ b/daedalus/src/minecraft.rs @@ -430,10 +430,8 @@ pub struct VersionInfo { /// The minimum version of the Minecraft Launcher that can run this version of the game pub minimum_launcher_version: u32, /// The time that the version was released - #[cfg_attr(feature = "bincode", bincode(with_serde))] pub release_time: DateTime, /// The latest time a file in this version was updated - #[cfg_attr(feature = "bincode", bincode(with_serde))] pub time: DateTime, #[serde(rename = "type")] /// The type of version diff --git a/daedalus_client/src/forge.rs b/daedalus_client/src/forge.rs index 09c8e0aa..a560b81a 100644 --- a/daedalus_client/src/forge.rs +++ b/daedalus_client/src/forge.rs @@ -1,5 +1,5 @@ use crate::util::{ - download_file, fetch_json, fetch_xml, format_url, sha1_async, + download_file, fetch_json, fetch_xml, format_url, }; use crate::{insert_mirrored_artifact, Error, MirrorArtifact, UploadFile}; use chrono::{DateTime, Utc}; @@ -403,9 +403,10 @@ async fn fetch( .map(|mut lib| { // For all libraries besides the forge lib extracted, we mirror them from maven servers // unless the URL is empty/null or available on Minecraft's servers - if let Some(url) = lib.url { - if lib.name != install_profile.install.path - && !url.is_empty() + if let Some(ref url) = lib.url { + if lib.name == install_profile.install.path { + lib.url = Some(format_url("maven/")); + } else if !url.is_empty() && !url.contains( "https://libraries.minecraft.net/", ) @@ -414,7 +415,7 @@ async fn fetch( &lib.name, None, vec![ - url, + url.clone(), "https://maven.creeperhost.net/" .to_string(), maven_url.to_string(), @@ -422,11 +423,11 @@ async fn fetch( false, mirror_artifacts, )?; + + lib.url = Some(format_url("maven/")); } } - lib.url = Some(format_url("maven/")); - Ok(lib) }) .collect::, Error>>()?, @@ -442,7 +443,7 @@ async fn fetch( // pub profile: String, // pub version: String, // pub json: String, - pub path: Option, + // pub path: Option, // pub minecraft: String, pub data: HashMap, pub libraries: Vec, diff --git a/daedalus_client/src/main.rs b/daedalus_client/src/main.rs index 8e934cbd..870a55ec 100644 --- a/daedalus_client/src/main.rs +++ b/daedalus_client/src/main.rs @@ -160,7 +160,7 @@ pub fn insert_mirrored_artifact( entire_url: bool, mirror_artifacts: &DashMap, ) -> Result<()> { - let mut val = mirror_artifacts + let val = mirror_artifacts .entry(get_path_from_artifact(artifact)?) .or_insert(MirrorArtifact { sha1, diff --git a/daedalus_client/src/util.rs b/daedalus_client/src/util.rs index 72ff2557..b5e4b360 100644 --- a/daedalus_client/src/util.rs +++ b/daedalus_client/src/util.rs @@ -159,7 +159,7 @@ pub async fn download_file( const RETRIES: u32 = 10; for attempt in 1..=(RETRIES + 1) { let result = REQWEST_CLIENT - .get(&url.replace("http://", "https://")) + .get(url.replace("http://", "https://")) .send() .await .and_then(|x| x.error_for_status()); From 679ffbcce75660410569e0538525b87b972a2fb1 Mon Sep 17 00:00:00 2001 From: Jai A Date: Wed, 21 Aug 2024 18:42:02 -0700 Subject: [PATCH 75/76] Fix Fabric Loader 0.16.0, old forge versions --- daedalus/Cargo.toml | 2 +- daedalus/src/modded.rs | 27 +++++++++++++++++++++++---- daedalus_client/src/fabric.rs | 1 + 3 files changed, 25 insertions(+), 5 deletions(-) diff --git a/daedalus/Cargo.toml b/daedalus/Cargo.toml index 1b28e72e..a57f4c61 100644 --- a/daedalus/Cargo.toml +++ b/daedalus/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "daedalus" -version = "0.2.2" +version = "0.2.3" authors = ["Jai A "] edition = "2021" license = "MIT" diff --git a/daedalus/src/modded.rs b/daedalus/src/modded.rs index c510fe78..e7043a4c 100644 --- a/daedalus/src/modded.rs +++ b/daedalus/src/modded.rs @@ -98,6 +98,27 @@ pub fn merge_partial_version( ) -> VersionInfo { let merge_id = merge.id.clone(); + let mut libraries = vec![]; + + // We skip duplicate libraries that exist already in the partial version + for mut lib in merge.libraries { + let lib_artifact = lib.name.rsplit_once(':').map(|x| x.0); + + if let Some(lib_artifact) = lib_artifact { + if !partial.libraries.iter().any(|x| { + let target_artifact = x.name.rsplit_once(':').map(|x| x.0); + + target_artifact == Some(lib_artifact) && x.include_in_classpath + }) { + libraries.push(lib); + } else { + lib.include_in_classpath = false; + } + } else { + libraries.push(lib); + } + } + VersionInfo { arguments: if let Some(partial_args) = partial.arguments { if let Some(merge_args) = merge.arguments { @@ -133,10 +154,8 @@ pub fn merge_partial_version( downloads: merge.downloads, id: partial.id.replace(DUMMY_REPLACE_STRING, &merge_id), java_version: merge.java_version, - libraries: partial - .libraries - .into_iter() - .chain(merge.libraries) + libraries: libraries.into_iter() + .chain(partial.libraries) .map(|mut x| { x.name = x.name.replace(DUMMY_REPLACE_STRING, &merge_id); diff --git a/daedalus_client/src/fabric.rs b/daedalus_client/src/fabric.rs index a7e352b2..2c394bc2 100644 --- a/daedalus_client/src/fabric.rs +++ b/daedalus_client/src/fabric.rs @@ -47,6 +47,7 @@ pub async fn fetch_quilt( .await } +#[allow(clippy::too_many_arguments)] #[tracing::instrument(skip(semaphore, upload_files, mirror_artifacts))] async fn fetch( format_version: usize, From f212fcf892d20a4de7a21542be6aff835a01e4c3 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sat, 19 Oct 2024 14:40:58 -0700 Subject: [PATCH 76/76] Start monorepo migration --- .../{docker.yml => daedalus-docker.yml} | 18 ++- .../workflows/{run.yml => daedalus-run.yml} | 2 +- .github/workflows/lint.yml | 30 ----- .github/workflows/publish.yml | 19 --- .github/workflows/rust.yml | 29 ----- .gitignore | 122 ------------------ .idea/daedalus.iml | 4 +- .idea/libraries/KotlinJavaRuntime.xml | 26 ++++ Cargo.toml | 6 - .env => apps/daedalus_client/.env | 0 .../daedalus_client}/Cargo.toml | 0 Dockerfile => apps/daedalus_client/Dockerfile | 0 LICENSE => apps/daedalus_client/LICENSE | 2 +- README.md => apps/daedalus_client/README.md | 0 .../daedalus_client/docker-compose.yml | 0 .../daedalus_client}/library-patches.json | 0 apps/daedalus_client/package.json | 10 ++ .../daedalus_client}/src/error.rs | 0 .../daedalus_client}/src/fabric.rs | 0 .../daedalus_client}/src/forge.rs | 0 .../daedalus_client}/src/main.rs | 0 .../daedalus_client}/src/minecraft.rs | 0 .../daedalus_client}/src/util.rs | 0 {daedalus => packages/daedalus}/Cargo.toml | 0 packages/daedalus/LICENSE | 7 + {daedalus => packages/daedalus}/README.md | 2 - packages/daedalus/package.json | 10 ++ {daedalus => packages/daedalus}/src/lib.rs | 0 .../daedalus}/src/minecraft.rs | 0 {daedalus => packages/daedalus}/src/modded.rs | 0 rustfmt.toml | 2 - 31 files changed, 69 insertions(+), 220 deletions(-) rename .github/workflows/{docker.yml => daedalus-docker.yml} (73%) rename .github/workflows/{run.yml => daedalus-run.yml} (96%) delete mode 100644 .github/workflows/lint.yml delete mode 100644 .github/workflows/publish.yml delete mode 100644 .github/workflows/rust.yml delete mode 100644 .gitignore create mode 100644 .idea/libraries/KotlinJavaRuntime.xml delete mode 100644 Cargo.toml rename .env => apps/daedalus_client/.env (100%) rename {daedalus_client => apps/daedalus_client}/Cargo.toml (100%) rename Dockerfile => apps/daedalus_client/Dockerfile (100%) rename LICENSE => apps/daedalus_client/LICENSE (97%) rename README.md => apps/daedalus_client/README.md (100%) rename docker-compose.yml => apps/daedalus_client/docker-compose.yml (100%) rename {daedalus_client => apps/daedalus_client}/library-patches.json (100%) create mode 100644 apps/daedalus_client/package.json rename {daedalus_client => apps/daedalus_client}/src/error.rs (100%) rename {daedalus_client => apps/daedalus_client}/src/fabric.rs (100%) rename {daedalus_client => apps/daedalus_client}/src/forge.rs (100%) rename {daedalus_client => apps/daedalus_client}/src/main.rs (100%) rename {daedalus_client => apps/daedalus_client}/src/minecraft.rs (100%) rename {daedalus_client => apps/daedalus_client}/src/util.rs (100%) rename {daedalus => packages/daedalus}/Cargo.toml (100%) create mode 100644 packages/daedalus/LICENSE rename {daedalus => packages/daedalus}/README.md (86%) create mode 100644 packages/daedalus/package.json rename {daedalus => packages/daedalus}/src/lib.rs (100%) rename {daedalus => packages/daedalus}/src/minecraft.rs (100%) rename {daedalus => packages/daedalus}/src/modded.rs (100%) delete mode 100644 rustfmt.toml diff --git a/.github/workflows/docker.yml b/.github/workflows/daedalus-docker.yml similarity index 73% rename from .github/workflows/docker.yml rename to .github/workflows/daedalus-docker.yml index 274085ff..f3e5a2d7 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/daedalus-docker.yml @@ -1,15 +1,21 @@ -name: docker-build +name: daedalus-docker-build on: push: - branches: - - '**' - tags: - - 'v*' + branches: [ "main" ] + paths: + - .github/workflows/daedalus-docker.yml + - 'apps/daedalus/**' pull_request: + types: [ opened, synchronize ] + paths: + - .github/workflows/daedalus-docker.yml + - 'apps/daedalus/**' + merge_group: + types: [ checks_requested ] + jobs: docker: - if: github.repository_owner == 'modrinth' runs-on: ubuntu-latest steps: - name: Checkout diff --git a/.github/workflows/run.yml b/.github/workflows/daedalus-run.yml similarity index 96% rename from .github/workflows/run.yml rename to .github/workflows/daedalus-run.yml index 635c6119..dcdbd049 100644 --- a/.github/workflows/run.yml +++ b/.github/workflows/daedalus-run.yml @@ -23,7 +23,7 @@ jobs: - name: Pull Docker image from GHCR - run: docker pull ghcr.io/modrinth/daedalus:master + run: docker pull ghcr.io/modrinth/daedalus:main - name: Run Docker container env: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml deleted file mode 100644 index 8bf19521..00000000 --- a/.github/workflows/lint.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Rust lint - -on: - push: - branches: [ master ] - pull_request: -env: - CARGO_TERM_COLOR: always -jobs: - lint: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Install toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - components: rustfmt, clippy - - name: Cache build artifacts - id: cache-build - uses: actions/cache@v2 - with: - path: target/** - key: ${{ runner.os }}-build-cache - - name: Annotate commit with clippy warnings - uses: actions-rs/clippy-check@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - args: --all-features diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml deleted file mode 100644 index 9e224e2a..00000000 --- a/.github/workflows/publish.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Publish to crates.io - -on: - push: - tags: - - 'v*' -env: - CARGO_TERM_COLOR: always -jobs: - publish: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v1 - - run: cargo login ${CRATES_IO_TOKEN} - working-directory: ./daedalus - env: - CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} - - run: cargo publish - working-directory: ./daedalus diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml deleted file mode 100644 index a55aaf6e..00000000 --- a/.github/workflows/rust.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: Rust building - -on: - push: - branches: [ master ] - pull_request: -env: - CARGO_TERM_COLOR: always -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Get build cache - id: cache-build - uses: actions/cache@v2 - with: - path: target/** - key: ${{ runner.os }}-build-cache - - name: Install toolchain - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - env: - SQLX_OFFLINE: true - - uses: actions-rs/cargo@v1 - name: Build program - with: - command: build diff --git a/.gitignore b/.gitignore deleted file mode 100644 index 06b153ac..00000000 --- a/.gitignore +++ /dev/null @@ -1,122 +0,0 @@ -### Intellij ### -# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider -# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 -caches/ - -# User-specific stuff -.idea/**/workspace.xml -.idea/**/tasks.xml -.idea/**/usage.statistics.xml -.idea/**/dictionaries -.idea/**/shelf - -# AWS User-specific -.idea/**/aws.xml - -# Generated files -.idea/**/contentModel.xml - -# Sensitive or high-churn files -.idea/**/dataSources/ -.idea/**/dataSources.ids -.idea/**/dataSources.local.xml -.idea/**/sqlDataSources.xml -.idea/**/dynamic.xml -.idea/**/uiDesigner.xml -.idea/**/dbnavigator.xml - -# Gradle -.idea/**/gradle.xml -.idea/**/libraries - -# Gradle and Maven with auto-import -# When using Gradle or Maven with auto-import, you should exclude module files, -# since they will be recreated, and may cause churn. Uncomment if using -# auto-import. -# .idea/artifacts -# .idea/compiler.xml -# .idea/jarRepositories.xml -# .idea/modules.xml -# .idea/*.iml -# .idea/modules -# *.iml -# *.ipr - -# CMake -cmake-build-*/ - -# Mongo Explorer plugin -.idea/**/mongoSettings.xml - -# File-based project format -*.iws - -# IntelliJ -out/ - -# mpeltonen/sbt-idea plugin -.idea_modules/ - -# JIRA plugin -atlassian-ide-plugin.xml - -# Cursive Clojure plugin -.idea/replstate.xml - -# Crashlytics plugin (for Android Studio and IntelliJ) -com_crashlytics_export_strings.xml -crashlytics.properties -crashlytics-build.properties -fabric.properties - -# Editor-based Rest Client -.idea/httpRequests - -# Android studio 3.1+ serialized cache file -.idea/caches/build_file_checksums.ser - -### Intellij Patch ### -# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 - -# *.iml -# modules.xml -# .idea/misc.xml -# *.ipr - -# Sonarlint plugin -# https://plugins.jetbrains.com/plugin/7973-sonarlint -.idea/**/sonarlint/ - -# SonarQube Plugin -# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin -.idea/**/sonarIssues.xml - -# Markdown Navigator plugin -# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced -.idea/**/markdown-navigator.xml -.idea/**/markdown-navigator-enh.xml -.idea/**/markdown-navigator/ - -# Cache file creation bug -# See https://youtrack.jetbrains.com/issue/JBR-2257 -.idea/$CACHE_FILE$ - -# CodeStream plugin -# https://plugins.jetbrains.com/plugin/12206-codestream -.idea/codestream.xml - -### Rust ### -# Generated by Cargo -# will have compiled files and executables -debug/ -target/ - -# Remove Cargo.lock from gitignore if creating an executable, leave it for libraries -# More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html -Cargo.lock - -# These are backup files generated by rustfmt -**/*.rs.bk - -# MSVC Windows builds of rustc generate these, which store debugging information -*.pdb diff --git a/.idea/daedalus.iml b/.idea/daedalus.iml index cc8620ac..3d715289 100644 --- a/.idea/daedalus.iml +++ b/.idea/daedalus.iml @@ -3,10 +3,10 @@ - - + + diff --git a/.idea/libraries/KotlinJavaRuntime.xml b/.idea/libraries/KotlinJavaRuntime.xml new file mode 100644 index 00000000..78d18789 --- /dev/null +++ b/.idea/libraries/KotlinJavaRuntime.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml deleted file mode 100644 index dc360c2e..00000000 --- a/Cargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[workspace] - -members = [ - "daedalus", - "daedalus_client" -] \ No newline at end of file diff --git a/.env b/apps/daedalus_client/.env similarity index 100% rename from .env rename to apps/daedalus_client/.env diff --git a/daedalus_client/Cargo.toml b/apps/daedalus_client/Cargo.toml similarity index 100% rename from daedalus_client/Cargo.toml rename to apps/daedalus_client/Cargo.toml diff --git a/Dockerfile b/apps/daedalus_client/Dockerfile similarity index 100% rename from Dockerfile rename to apps/daedalus_client/Dockerfile diff --git a/LICENSE b/apps/daedalus_client/LICENSE similarity index 97% rename from LICENSE rename to apps/daedalus_client/LICENSE index 75e4f779..bc179be0 100644 --- a/LICENSE +++ b/apps/daedalus_client/LICENSE @@ -1,4 +1,4 @@ -Copyright © 2023 Rinth, Inc. +Copyright © 2024 Rinth, Inc. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: diff --git a/README.md b/apps/daedalus_client/README.md similarity index 100% rename from README.md rename to apps/daedalus_client/README.md diff --git a/docker-compose.yml b/apps/daedalus_client/docker-compose.yml similarity index 100% rename from docker-compose.yml rename to apps/daedalus_client/docker-compose.yml diff --git a/daedalus_client/library-patches.json b/apps/daedalus_client/library-patches.json similarity index 100% rename from daedalus_client/library-patches.json rename to apps/daedalus_client/library-patches.json diff --git a/apps/daedalus_client/package.json b/apps/daedalus_client/package.json new file mode 100644 index 00000000..41c8b0ae --- /dev/null +++ b/apps/daedalus_client/package.json @@ -0,0 +1,10 @@ +{ + "name": "@modrinth/daedalus_client", + "scripts": { + "build": "cargo build --release", + "lint": "cargo fmt --check && cargo clippy --all-targets --all-features -- -D warnings", + "fix": "cargo fmt && cargo clippy --fix", + "dev": "cargo run", + "test": "cargo test" + } +} \ No newline at end of file diff --git a/daedalus_client/src/error.rs b/apps/daedalus_client/src/error.rs similarity index 100% rename from daedalus_client/src/error.rs rename to apps/daedalus_client/src/error.rs diff --git a/daedalus_client/src/fabric.rs b/apps/daedalus_client/src/fabric.rs similarity index 100% rename from daedalus_client/src/fabric.rs rename to apps/daedalus_client/src/fabric.rs diff --git a/daedalus_client/src/forge.rs b/apps/daedalus_client/src/forge.rs similarity index 100% rename from daedalus_client/src/forge.rs rename to apps/daedalus_client/src/forge.rs diff --git a/daedalus_client/src/main.rs b/apps/daedalus_client/src/main.rs similarity index 100% rename from daedalus_client/src/main.rs rename to apps/daedalus_client/src/main.rs diff --git a/daedalus_client/src/minecraft.rs b/apps/daedalus_client/src/minecraft.rs similarity index 100% rename from daedalus_client/src/minecraft.rs rename to apps/daedalus_client/src/minecraft.rs diff --git a/daedalus_client/src/util.rs b/apps/daedalus_client/src/util.rs similarity index 100% rename from daedalus_client/src/util.rs rename to apps/daedalus_client/src/util.rs diff --git a/daedalus/Cargo.toml b/packages/daedalus/Cargo.toml similarity index 100% rename from daedalus/Cargo.toml rename to packages/daedalus/Cargo.toml diff --git a/packages/daedalus/LICENSE b/packages/daedalus/LICENSE new file mode 100644 index 00000000..bc179be0 --- /dev/null +++ b/packages/daedalus/LICENSE @@ -0,0 +1,7 @@ +Copyright © 2024 Rinth, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/daedalus/README.md b/packages/daedalus/README.md similarity index 86% rename from daedalus/README.md rename to packages/daedalus/README.md index fb57d0f0..aa560259 100644 --- a/daedalus/README.md +++ b/packages/daedalus/README.md @@ -2,5 +2,3 @@ Daedalus (the rust library) is a library providing model structs and methods for requesting and parsing things from Minecraft and other mod loaders meta APIs. - -This is a work in progress! \ No newline at end of file diff --git a/packages/daedalus/package.json b/packages/daedalus/package.json new file mode 100644 index 00000000..41c8b0ae --- /dev/null +++ b/packages/daedalus/package.json @@ -0,0 +1,10 @@ +{ + "name": "@modrinth/daedalus_client", + "scripts": { + "build": "cargo build --release", + "lint": "cargo fmt --check && cargo clippy --all-targets --all-features -- -D warnings", + "fix": "cargo fmt && cargo clippy --fix", + "dev": "cargo run", + "test": "cargo test" + } +} \ No newline at end of file diff --git a/daedalus/src/lib.rs b/packages/daedalus/src/lib.rs similarity index 100% rename from daedalus/src/lib.rs rename to packages/daedalus/src/lib.rs diff --git a/daedalus/src/minecraft.rs b/packages/daedalus/src/minecraft.rs similarity index 100% rename from daedalus/src/minecraft.rs rename to packages/daedalus/src/minecraft.rs diff --git a/daedalus/src/modded.rs b/packages/daedalus/src/modded.rs similarity index 100% rename from daedalus/src/modded.rs rename to packages/daedalus/src/modded.rs diff --git a/rustfmt.toml b/rustfmt.toml deleted file mode 100644 index f5a8b867..00000000 --- a/rustfmt.toml +++ /dev/null @@ -1,2 +0,0 @@ -edition = "2018" -max_width = 80 \ No newline at end of file