From fe3581756ffdcb40693deb91c53233de83c5023e Mon Sep 17 00:00:00 2001 From: Daniel Hutzley Date: Wed, 1 Dec 2021 21:06:31 -0800 Subject: [PATCH 1/4] Complete refactor of modpacks, add modpack serialization --- Cargo.lock | 73 ++-- theseus/examples/download-pack.rs | 2 +- theseus/src/modpack/manifest.rs | 601 ++++++++++++++---------------- theseus/src/modpack/mod.rs | 36 +- theseus/src/modpack/pack.rs | 106 ++++++ 5 files changed, 446 insertions(+), 372 deletions(-) create mode 100644 theseus/src/modpack/pack.rs diff --git a/Cargo.lock b/Cargo.lock index ff5ce47f3..c1b51820a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -147,9 +147,9 @@ checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc" [[package]] name = "crc32fast" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +checksum = "3825b1e8580894917dc4468cb634a1b4e9745fddc854edad72d9c04644c0319f" dependencies = [ "cfg-if", ] @@ -230,9 +230,9 @@ checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" [[package]] name = "futures" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12aa0eb539080d55c3f2d45a67c3b58b6b0773c1a3ca2dfec66d58c97fd66ca" +checksum = "8cd0210d8c325c245ff06fd95a3b13689a1a276ac8cfa8e8720cb840bfb84b9e" dependencies = [ "futures-channel", "futures-core", @@ -245,9 +245,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5da6ba8c3bb3c165d3c7319fc1cc8304facf1fb8db99c5de877183c08a273888" +checksum = "7fc8cd39e3dbf865f7340dce6a2d401d24fd37c6fe6c4f0ee0de8bfca2252d27" dependencies = [ "futures-core", "futures-sink", @@ -255,15 +255,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88d1c26957f23603395cd326b0ffe64124b818f4449552f960d815cfba83a53d" +checksum = "629316e42fe7c2a0b9a65b47d159ceaa5453ab14e8f0a3c5eedbb8cd55b4a445" [[package]] name = "futures-executor" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45025be030969d763025784f7f355043dc6bc74093e4ecc5000ca4dc50d8745c" +checksum = "7b808bf53348a36cab739d7e04755909b9fcaaa69b7d7e588b37b6ec62704c97" dependencies = [ "futures-core", "futures-task", @@ -272,18 +272,16 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "522de2a0fe3e380f1bc577ba0474108faf3f6b18321dbf60b3b9c39a75073377" +checksum = "e481354db6b5c353246ccf6a728b0c5511d752c08da7260546fc0933869daa11" [[package]] name = "futures-macro" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18e4a4b95cea4b4ccbcf1c5675ca7c4ee4e9e75eb79944d07defde18068f79bb" +checksum = "a89f17b21645bc4ed773c69af9c9a0effd4a3f1a3876eadd453469f8854e7fdd" dependencies = [ - "autocfg", - "proc-macro-hack", "proc-macro2", "quote", "syn", @@ -291,23 +289,22 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36ea153c13024fe480590b3e3d4cad89a0cfacecc24577b68f86c6ced9c2bc11" +checksum = "996c6442437b62d21a32cd9906f9c41e7dc1e19a9579843fad948696769305af" [[package]] name = "futures-task" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d3d00f4eddb73e498a54394f228cd55853bdf059259e8e7bc6e69d408892e99" +checksum = "dabf1872aaab32c886832f2276d2f5399887e2bd613698a02359e4ea83f8de12" [[package]] name = "futures-util" -version = "0.3.17" +version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36568465210a3a6ee45e1f165136d68671471a501e632e9a98d96872222b5481" +checksum = "41d22213122356472061ac0f1ab2cee28d2bac8491410fd68c2af53d1cedb83e" dependencies = [ - "autocfg", "futures-channel", "futures-core", "futures-io", @@ -317,8 +314,6 @@ dependencies = [ "memchr", "pin-project-lite", "pin-utils", - "proc-macro-hack", - "proc-macro-nested", "slab", ] @@ -506,9 +501,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.107" +version = "0.2.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbe5e23404da5b4f555ef85ebed98fb4083e55a00c317800bc2a50ede9f3d219" +checksum = "8521a1b57e76b1ec69af7599e75e38e7b7fad6610f037db8c79b127201b5d119" [[package]] name = "lock_api" @@ -734,18 +729,6 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed0cfbc8191465bed66e1718596ee0b0b35d5ee1f41c5df2189d0fe8bde535ba" -[[package]] -name = "proc-macro-hack" -version = "0.5.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" - -[[package]] -name = "proc-macro-nested" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" - [[package]] name = "proc-macro2" version = "1.0.32" @@ -876,9 +859,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +checksum = "3c9613b5a66ab9ba26415184cfc41156594925a9cf3a2057e57f31ff145f6568" [[package]] name = "schannel" @@ -941,9 +924,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.71" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "063bf466a64011ac24040a49009724ee60a57da1b437617ceb32e53ad61bfb19" +checksum = "d0ffa0837f2dfa6fb90868c2b5468cad482e175f7dad97e7421951e663f2b527" dependencies = [ "itoa", "ryu", @@ -1001,9 +984,9 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.81" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2afee18b8beb5a596ecb4a2dce128c719b4ba399d34126b9e4396e3f9860966" +checksum = "8daf5dd0bb60cbd4137b1b587d2fc0ae729bc07cf01cd70b36a1ed5ade3b9d59" dependencies = [ "proc-macro2", "quote", diff --git a/theseus/examples/download-pack.rs b/theseus/examples/download-pack.rs index e41f15fe7..084bb44c8 100644 --- a/theseus/examples/download-pack.rs +++ b/theseus/examples/download-pack.rs @@ -1,7 +1,7 @@ use std::{path::PathBuf, time::Instant}; use argh::FromArgs; -use theseus::modpack::{fetch_modpack, manifest::ModpackSide}; +use theseus::modpack::{fetch_modpack, pack::ModpackSide}; #[derive(FromArgs)] /// Simple modpack download diff --git a/theseus/src/modpack/manifest.rs b/theseus/src/modpack/manifest.rs index 448311bfa..4e6973162 100644 --- a/theseus/src/modpack/manifest.rs +++ b/theseus/src/modpack/manifest.rs @@ -1,291 +1,266 @@ -use std::{ - convert::TryFrom, - path::{Path, PathBuf}, - str::FromStr, -}; +use std::path::{Path, PathBuf}; -use daedalus::download_file_mirrors; -use futures::future; -use tokio::fs; +use std::convert::TryFrom; use crate::launcher::ModLoader; -use super::ModpackError; +use super::pack::ModpackGame; +use super::{pack, ModpackError, ModpackResult}; +use daedalus::modded::LoaderType; +use serde::{Deserialize, Serialize}; -#[derive(Debug, Clone, PartialEq)] -pub struct Manifest { - format_version: u64, - game: ModpackGame, - version_id: String, +pub const DEFAULT_FORMAT_VERSION: u32 = 1; - name: String, - summary: Option, - - files: Vec, +#[derive(Debug, Deserialize, Serialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct Manifest<'a> { + pub format_version: u32, + pub game: &'a str, + pub version_id: &'a str, + pub name: &'a str, + #[serde(borrow)] + pub summary: Option<&'a str>, + pub files: Vec>, + pub dependencies: ManifestDeps<'a>, } -impl Manifest { - /// Download a modpack's files for a given side to a given destination - /// Assumes the destination exists and is a directory - pub async fn download_files(&self, dest: &Path, side: ModpackSide) -> Result<(), ModpackError> { - let handles = self.files.clone().into_iter().map(move |file| { - let (dest, side) = (dest.to_owned(), side); - tokio::spawn(async move { file.fetch(&dest, side).await }) - }); - future::try_join_all(handles) - .await? - .into_iter() - .collect::>()?; - - // TODO Integrate instance format to save other metadata - Ok(()) - } -} - -fn try_get<'r, F, T>( - manifest: &'r serde_json::Map, - field: &str, - caster: F, -) -> Result -where - F: Fn(&'r serde_json::Value) -> Option, -{ - manifest - .get(field) - .and_then(caster) - .ok_or(ModpackError::ManifestError(format!( - "Invalid or missing field: {}", - field - ))) -} - -impl TryFrom for Manifest { +impl TryFrom> for pack::Modpack { type Error = ModpackError; - fn try_from(value: serde_json::Value) -> Result { - use ModpackError::ManifestError; - - let value = value.as_object().ok_or(ManifestError(String::from( - "Manifest is not a JSON object!", - )))?; - - let game = ModpackGame::new( - try_get(value, "game", serde_json::Value::as_str)?, - try_get(value, "dependencies", serde_json::Value::as_object)?, - )?; - - let files = try_get(value, "files", serde_json::Value::as_array)? - .iter() - .map(|it| -> Result { - let file = it - .as_object() - .ok_or(ManifestError(String::from("Malformed file: not an object")))?; - - let path = Path::new(try_get(file, "path", serde_json::Value::as_str)?); - let hashes = ModpackFileHashes::try_from(try_get( - file, - "hashes", - serde_json::Value::as_object, - )?)?; - let downloads = try_get(file, "downloads", serde_json::Value::as_array)? - .iter() - .map(serde_json::Value::as_str) - .map(|it| it.map(String::from)) - .collect::>>() - .ok_or(ManifestError(format!( - "Invalid source for path {}", - path.to_str().unwrap_or("?") - )))?; - let env: Option<[ModpackEnv; 2]> = if let Some(env) = file.get("env") { - if !env.is_object() { - return Err(ManifestError(String::from( - "Env is provided, but is not an object!", - ))); - } - Some([ - ModpackEnv::from_str( - env.get("client") - .and_then(serde_json::Value::as_str) - .unwrap_or_default(), - )?, - ModpackEnv::from_str( - env.get("server") - .and_then(serde_json::Value::as_str) - .unwrap_or_default(), - )?, - ]) - } else { - None - }; - - ModpackFile::new(path, hashes, env, &downloads) - }) - .collect::, ModpackError>>()?; + fn try_from(manifest: Manifest<'_>) -> Result { + let files = manifest + .files + .into_iter() + .map(pack::ModpackFile::try_from) + .collect::>>()?; Ok(Self { - format_version: try_get(value, "formatVersion", serde_json::Value::as_u64)?, - game, - version_id: String::from(try_get(value, "versionId", serde_json::Value::as_str)?), - name: String::from(try_get(value, "name", serde_json::Value::as_str)?), - summary: value - .get("summary") - .and_then(serde_json::Value::as_str) - .map(String::from), + name: String::from(manifest.name), + version: String::from(manifest.version_id), + summary: manifest.summary.map(String::from), + game: ModpackGame::from(manifest.dependencies), files, }) } } -#[derive(Debug, Clone, PartialEq)] -pub enum ModpackGame { - // TODO: Currently, the launcher does not support specifying mod loader versions, so I just - // store the loader here. - Minecraft(String, ModLoader), +const MODRINTH_GAMEDATA_URL: &'static str = "https://staging-cdn.modrinth.com/gamedata"; +fn get_loader_version(loader: ModLoader, version: &str) -> ModpackResult { + let source = match loader { + ModLoader::Vanilla => Err(ModpackError::VersionError(String::from( + "Attempted to get mod loader version of Vanilla", + ))), + ModLoader::Forge => Ok(format!("{}/forge/v0/manifest.json", MODRINTH_GAMEDATA_URL)), + ModLoader::Fabric => Ok(format!("{}/fabric/v0/manifest.json", MODRINTH_GAMEDATA_URL)), + }?; + let manifest = futures::executor::block_on(daedalus::modded::fetch_manifest(&source))?; + + Ok(manifest + .game_versions + .iter() + .find(|&it| it.id == version) + .ok_or(ModpackError::VersionError(format!( + "No versions of {:?} exist for Minecraft {}", + loader, version + )))? + .loaders[&LoaderType::Latest] + .id + .clone()) } -impl ModpackGame { - pub fn new( - game: &str, - deps: &serde_json::Map, - ) -> Result { - match game { - "minecraft" => { - let game_version = String::from( - deps.get("minecraft") - .ok_or(ModpackError::ManifestError(String::from( - "No version of minecraft given", - )))? - .as_str() - .unwrap(), - ); - - // TODO: See comment in ModpackGame, this code was designed specifically to be - // easily adapted for versioned modloaders - let loader = if let Some(_) = deps.get("fabric-loader") { - ModLoader::Fabric - } else if let Some(_) = deps.get("forge") { - ModLoader::Forge - } else { - ModLoader::Vanilla - }; - - Ok(ModpackGame::Minecraft(game_version, loader)) - } - _ => Err(ModpackError::ManifestError(format!( - "Invalid game: {}", - game - ))), - } - } -} - -#[derive(Debug, Clone, PartialEq)] -pub struct ModpackFile { - path: PathBuf, - hashes: ModpackFileHashes, - envs: Option<[ModpackEnv; 2]>, - downloads: Vec, -} - -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum ModpackSide { - Client = 0, - Server = 1, -} - -impl ModpackFile { - pub fn new( - path: &Path, - hashes: ModpackFileHashes, - envs: Option<[ModpackEnv; 2]>, - downloads: &[String], - ) -> Result { - if path.is_dir() { - return Err(ModpackError::ManifestError(format!( - "Modpack file {} is a directory!", - path.to_str().unwrap_or("?") - ))); - } - - Ok(Self { - path: PathBuf::from(path), - hashes, - envs, - downloads: Vec::from(downloads), - }) - } - - pub async fn fetch(&self, dest: &Path, side: ModpackSide) -> Result<(), ModpackError> { - if let Some(envs) = &self.envs { - if envs[side as usize] == ModpackEnv::Unsupported - || envs[(side as usize + 1) % 2] == ModpackEnv::Required - { - return Ok(()); - } - } - - let output = dest.join(&self.path); - - // HACK: Since Daedalus appends a file name to all mirrors and the manifest supplies full - // URLs, I'm supplying it with an empty string to avoid reinventing the wheel. - let bytes = download_file_mirrors( - "", - &self - .downloads - .iter() - .map(|it| it.as_str()) - .collect::>() - .as_slice(), - Some(&self.hashes.sha1), - ) - .await?; - fs::create_dir_all(output.parent().unwrap()).await?; - fs::write(output, bytes).await?; - Ok(()) - } -} - -#[derive(Debug, Clone, PartialEq)] -pub struct ModpackFileHashes { - sha1: String, -} - -impl TryFrom<&serde_json::Map> for ModpackFileHashes { +impl<'a> TryFrom<&'a pack::Modpack> for Manifest<'a> { type Error = ModpackError; - fn try_from(value: &serde_json::Map) -> Result { - let sha1 = String::from(try_get(&value, "sha1", serde_json::Value::as_str)?); - Ok(Self { sha1 }) + fn try_from(pack: &'a pack::Modpack) -> Result { + let game_field: &'a str = match pack.game { + ModpackGame::Minecraft(..) => "minecraft", + }; + + let files = pack + .files + .iter() + .map(ManifestFile::from) + .collect::>(); + + Ok(Manifest { + format_version: DEFAULT_FORMAT_VERSION, + game: game_field, + version_id: &pack.version, + name: &pack.name, + summary: pack.summary.as_ref().map(String::as_str), + files, + dependencies: ManifestDeps::try_from(&pack.game)?, + }) } } -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum ModpackEnv { +#[derive(Debug, Deserialize, Serialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct ManifestFile<'a> { + #[serde(borrow)] + pub path: &'a Path, + pub hashes: ManifestHashes<'a>, + #[serde(default)] + pub env: ManifestEnvs, + #[serde(borrow)] + pub downloads: Vec<&'a str>, +} + +impl TryFrom> for pack::ModpackFile { + type Error = ModpackError; + + fn try_from(file: ManifestFile<'_>) -> Result { + Ok(Self { + path: PathBuf::from(file.path), + hashes: pack::ModpackFileHashes::from(file.hashes), + env: pack::ModpackEnv::try_from(file.env)?, + downloads: file.downloads.into_iter().map(ToOwned::to_owned).collect(), + }) + } +} + +impl<'a> From<&'a pack::ModpackFile> for ManifestFile<'a> { + fn from(file: &'a pack::ModpackFile) -> Self { + Self { + path: file.path.as_path(), + hashes: (&file.hashes).into(), + env: file.env.into(), + downloads: file + .downloads + .iter() + .map(String::as_str) + .collect::>(), + } + } +} + +#[derive(Debug, Deserialize, Serialize, PartialEq)] +pub struct ManifestHashes<'a> { + pub sha1: &'a str, +} + +impl From> for pack::ModpackFileHashes { + fn from(hashes: ManifestHashes<'_>) -> Self { + Self { + sha1: String::from(hashes.sha1), + } + } +} + +impl<'a> From<&'a pack::ModpackFileHashes> for ManifestHashes<'a> { + fn from(hashes: &'a pack::ModpackFileHashes) -> Self { + Self { sha1: &hashes.sha1 } + } +} + +#[derive(Debug, Deserialize, Serialize, PartialEq)] +pub struct ManifestEnvs { + pub client: ManifestEnv, + pub server: ManifestEnv, +} + +impl Default for ManifestEnvs { + fn default() -> Self { + Self { + client: ManifestEnv::Optional, + server: ManifestEnv::Optional, + } + } +} + +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] +#[serde(rename_all = "lowercase")] +pub enum ManifestEnv { Required, Optional, Unsupported, } -impl FromStr for ModpackEnv { - type Err = ModpackError; +impl TryFrom for pack::ModpackEnv { + type Error = ModpackError; - fn from_str(s: &str) -> Result { - use ModpackEnv::*; - match s { - "required" => Ok(Required), - "optional" => Ok(Optional), - "unsupported" => Ok(Unsupported), - _ => Err(ModpackError::ManifestError(format!( - "Invalid environment support: {}", - s + fn try_from(envs: ManifestEnvs) -> Result { + use ManifestEnv::*; + + match (envs.client, envs.server) { + (Required, Unsupported) => Ok(Self::ClientOnly), + (Unsupported, Required) => Ok(Self::ServerOnly), + (Optional, Optional) => Ok(Self::Both), + _ => Err(ModpackError::FormatError(format!( + "Invalid environment specification: {:?}", + envs ))), } } } -impl Default for ModpackEnv { - fn default() -> Self { - Self::Optional +impl From for ManifestEnvs { + fn from(envs: pack::ModpackEnv) -> Self { + use super::pack::ModpackEnv::*; + + let (client, server) = match envs { + ClientOnly => (ManifestEnv::Required, ManifestEnv::Unsupported), + ServerOnly => (ManifestEnv::Unsupported, ManifestEnv::Required), + Both => (ManifestEnv::Optional, ManifestEnv::Optional), + }; + + Self { client, server } + } +} + +#[derive(Debug, Deserialize, Serialize, PartialEq)] +#[serde(untagged)] +// HACK: I've tried for hours to get this working zero-copy, but I'm beat. If someone else wants to +// go through the # { + MinecraftFabric { + minecraft: &'a str, + #[serde(rename = "fabric-loader")] + fabric_loader: String, + }, + MinecraftForge { + minecraft: &'a str, + forge: String, + }, + MinecraftVanilla { + minecraft: &'a str, + }, +} + +impl From> for pack::ModpackGame { + fn from(deps: ManifestDeps<'_>) -> Self { + use ManifestDeps::*; + + match deps { + MinecraftVanilla { minecraft } => { + Self::Minecraft(String::from(minecraft), ModLoader::Vanilla) + } + MinecraftFabric { minecraft, .. } => { + Self::Minecraft(String::from(minecraft), ModLoader::Fabric) + } + MinecraftForge { minecraft, .. } => { + Self::Minecraft(String::from(minecraft), ModLoader::Forge) + } + } + } +} + +impl<'a> TryFrom<&'a pack::ModpackGame> for ManifestDeps<'a> { + type Error = ModpackError; + + fn try_from(game: &'a pack::ModpackGame) -> Result { + use super::pack::ModpackGame::*; + Ok(match game { + Minecraft(ref ver, ModLoader::Vanilla) => Self::MinecraftVanilla { minecraft: ver }, + Minecraft(ref ver, loader @ ModLoader::Fabric) => Self::MinecraftFabric { + minecraft: ver, + fabric_loader: get_loader_version(*loader, ver)?, + }, + Minecraft(ref ver, loader @ ModLoader::Forge) => Self::MinecraftForge { + minecraft: ver, + forge: get_loader_version(*loader, ver)?, + }, + }) } } @@ -294,7 +269,7 @@ mod tests { use super::*; #[test] - fn parse_simple() -> Result<(), ModpackError> { + fn parse_simple() -> ModpackResult<()> { const PACK_JSON: &'static str = r#" { "formatVersion": 1, @@ -309,22 +284,23 @@ mod tests { "#; let expected_manifest = Manifest { format_version: 1, - game: ModpackGame::Minecraft(String::from("1.17.1"), ModLoader::Vanilla), - version_id: String::from("deadbeef"), - name: String::from("Example Pack"), + game: "minecraft", + version_id: "deadbeef", + name: "Example Pack", summary: None, files: Vec::new(), + dependencies: ManifestDeps::MinecraftVanilla { + minecraft: "1.17.1", + }, }; - let manifest_json: serde_json::Value = - serde_json::from_str(PACK_JSON).expect("Error parsing pack JSON"); - let manifest = Manifest::try_from(manifest_json)?; + let manifest: Manifest = serde_json::from_str(PACK_JSON).expect("Error parsing pack JSON"); assert_eq!(expected_manifest, manifest); Ok(()) } #[test] - fn parse_forge() -> Result<(), ModpackError> { + fn parse_forge() -> ModpackResult<()> { const PACK_JSON: &'static str = r#" { "formatVersion": 1, @@ -348,33 +324,33 @@ mod tests { } } "#; - let expected_manifest = Manifest { format_version: 1, - game: ModpackGame::Minecraft(String::from("1.17.1"), ModLoader::Forge), - version_id: String::from("deadbeef"), - name: String::from("Example Pack"), + game: "minecraft", + version_id: "deadbeef", + name: "Example Pack", summary: None, - files: vec![ModpackFile::new( - Path::new("mods/testmod.jar"), - ModpackFileHashes { - sha1: String::from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + files: vec![ManifestFile { + path: Path::new("mods/testmod.jar"), + hashes: ManifestHashes { + sha1: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", }, - None, - &[String::from("https://example.com/testmod.jar")], - )?], + env: ManifestEnvs::default(), + downloads: vec!["https://example.com/testmod.jar"], + }], + dependencies: ManifestDeps::MinecraftForge { + minecraft: "1.17.1", + forge: String::from("37.0.110"), + }, }; - - let manifest_json: serde_json::Value = - serde_json::from_str(PACK_JSON).expect("Error parsing pack JSON"); - let manifest = Manifest::try_from(manifest_json)?; + let manifest: Manifest = serde_json::from_str(PACK_JSON).expect("Error parsing pack JSON"); assert_eq!(expected_manifest, manifest); Ok(()) } #[test] - fn parse_fabric() -> Result<(), ModpackError> { + fn parse_fabric() -> ModpackResult<()> { const PACK_JSON: &'static str = r#" { "formatVersion": 1, @@ -398,33 +374,33 @@ mod tests { } } "#; - let expected_manifest = Manifest { format_version: 1, - game: ModpackGame::Minecraft(String::from("1.17.1"), ModLoader::Fabric), - version_id: String::from("deadbeef"), - name: String::from("Example Pack"), + game: "minecraft", + version_id: "deadbeef", + name: "Example Pack", summary: None, - files: vec![ModpackFile::new( - Path::new("mods/testmod.jar"), - ModpackFileHashes { - sha1: String::from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + files: vec![ManifestFile { + path: Path::new("mods/testmod.jar"), + hashes: ManifestHashes { + sha1: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", }, - None, - &[String::from("https://example.com/testmod.jar")], - )?], + env: ManifestEnvs::default(), + downloads: vec!["https://example.com/testmod.jar"], + }], + dependencies: ManifestDeps::MinecraftFabric { + minecraft: "1.17.1", + fabric_loader: String::from("0.9.0"), + }, }; - - let manifest_json: serde_json::Value = - serde_json::from_str(PACK_JSON).expect("Error parsing pack JSON"); - let manifest = Manifest::try_from(manifest_json)?; + let manifest: Manifest = serde_json::from_str(PACK_JSON).expect("Error parsing pack JSON"); assert_eq!(expected_manifest, manifest); Ok(()) } #[test] - fn parse_complete() -> Result<(), ModpackError> { + fn parse_complete() -> ModpackResult<()> { const PACK_JSON: &'static str = r#" { "formatVersion": 1, @@ -453,28 +429,29 @@ mod tests { } } "#; - let expected_manifest = Manifest { format_version: 1, - game: ModpackGame::Minecraft(String::from("1.17.1"), ModLoader::Forge), - version_id: String::from("deadbeef"), - name: String::from("Example Pack"), - summary: Some(String::from("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.")), - files: vec![ - ModpackFile::new( - Path::new("mods/testmod.jar"), - ModpackFileHashes { - sha1: String::from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), - }, - Some([ ModpackEnv::Required, ModpackEnv::Unsupported ]), - &[ String::from("https://example.com/testmod.jar") ], - )? - ], + game: "minecraft", + version_id: "deadbeef", + name: "Example Pack", + summary: Some("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua."), + files: vec![ManifestFile { + path: Path::new("mods/testmod.jar"), + hashes: ManifestHashes { + sha1: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + }, + env: ManifestEnvs { + client: ManifestEnv::Required, + server: ManifestEnv::Unsupported, + }, + downloads: vec!["https://example.com/testmod.jar"], + }], + dependencies: ManifestDeps::MinecraftForge { + minecraft: "1.17.1", + forge: String::from("37.0.110"), + }, }; - - let manifest_json: serde_json::Value = - serde_json::from_str(PACK_JSON).expect("Error parsing pack JSON"); - let manifest = Manifest::try_from(manifest_json)?; + let manifest: Manifest = serde_json::from_str(PACK_JSON).expect("Error parsing pack JSON"); assert_eq!(expected_manifest, manifest); Ok(()) diff --git a/theseus/src/modpack/mod.rs b/theseus/src/modpack/mod.rs index f463fc921..3002737ac 100644 --- a/theseus/src/modpack/mod.rs +++ b/theseus/src/modpack/mod.rs @@ -2,14 +2,16 @@ use daedalus::download_file; use fs_extra::dir::CopyOptions; -use std::{borrow::Borrow, convert::TryFrom, env, io, path::Path}; +use serde::Deserialize; +use std::{convert::TryFrom, env, io, path::Path}; use tokio::fs; use uuid::Uuid; use zip::ZipArchive; -use self::manifest::Manifest; +use self::{manifest::Manifest, pack::Modpack}; -pub mod manifest; +pub mod pack; +mod manifest; pub const MANIFEST_PATH: &'static str = "index.json"; pub const OVERRIDES_PATH: &'static str = "overrides/"; @@ -42,15 +44,20 @@ pub enum ModpackError { #[error("Error joining futures: {0}")] JoinError(#[from] tokio::task::JoinError), + + #[error("Error fetching modloader version: {0}")] + VersionError(String), } +type ModpackResult = Result; + /// Realise a modpack from a given URL pub async fn fetch_modpack( url: &str, sha1: Option<&str>, dest: &Path, - side: manifest::ModpackSide, -) -> Result<(), ModpackError> { + side: pack::ModpackSide, +) -> ModpackResult<()> { let bytes = download_file(url, sha1).await?; let mut archive = ZipArchive::new(io::Cursor::new(&bytes as &[u8]))?; realise_modpack_zip(&mut archive, dest, side).await @@ -60,8 +67,8 @@ pub async fn fetch_modpack( pub async fn realise_modpack_zip( archive: &mut ZipArchive, dest: &Path, - side: manifest::ModpackSide, -) -> Result<(), ModpackError> { + side: pack::ModpackSide, +) -> ModpackResult<()> { let tmp = env::temp_dir().join(format!("theseus-{}/", Uuid::new_v4())); archive.extract(&tmp)?; realise_modpack(&tmp, dest, side).await @@ -71,8 +78,8 @@ pub async fn realise_modpack_zip( pub async fn realise_modpack( dir: &Path, dest: &Path, - side: manifest::ModpackSide, -) -> Result<(), ModpackError> { + side: pack::ModpackSide, +) -> ModpackResult<()> { if dest.is_file() { return Err(ModpackError::InvalidDirectory(String::from( "Output is not a directory", @@ -101,11 +108,12 @@ pub async fn realise_modpack( "Manifest missing or is not a file", )))?; let manifest_file = std::fs::File::open(manifest_path)?; - let manifest_json: serde_json::Value = - serde_json::from_reader(io::BufReader::new(manifest_file))?; - let manifest = Manifest::try_from(manifest_json)?; + let reader = io::BufReader::new(manifest_file); + let mut deserializer = serde_json::Deserializer::from_reader(reader); + let manifest = Manifest::deserialize(&mut deserializer)?; + let modpack = Modpack::try_from(manifest)?; - // Realise manifest - manifest.download_files(dest, side).await?; + // Realise modpack + modpack.download_files(dest, side).await?; Ok(()) } diff --git a/theseus/src/modpack/pack.rs b/theseus/src/modpack/pack.rs new file mode 100644 index 000000000..bd9a783e2 --- /dev/null +++ b/theseus/src/modpack/pack.rs @@ -0,0 +1,106 @@ +use std::path::{Path, PathBuf}; +use daedalus::download_file_mirrors; +use futures::future; +use tokio::fs; + +use crate::launcher::ModLoader; +use super::ModpackResult; + +#[derive(Debug, Clone, PartialEq)] +pub struct Modpack { + pub game: ModpackGame, + pub version: String, + pub name: String, + pub summary: Option, + pub files: Vec, +} + +impl Modpack { + /// Download a modpack's files for a given side to a given destination + /// Assumes the destination exists and is a directory + pub async fn download_files(&self, dest: &Path, side: ModpackSide) -> ModpackResult<()> { + let handles = self.files.iter().cloned().map(move |file| { + let (dest, side) = (dest.to_owned(), side); + tokio::spawn(async move { file.fetch(&dest, side).await }) + }); + future::try_join_all(handles) + .await? + .into_iter() + .collect::>()?; + + // TODO Integrate instance format to save other metadata + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq)] +pub enum ModpackGame { + // TODO: Currently, the launcher does not support specifying mod loader versions, so I just + // store the loader here. + Minecraft(String, ModLoader), +} + + +#[derive(Debug, Clone, PartialEq)] +pub struct ModpackFile { + pub path: PathBuf, + pub hashes: ModpackFileHashes, + pub env: ModpackEnv, + pub downloads: Vec, +} + +impl ModpackFile { + pub async fn fetch(&self, dest: &Path, side: ModpackSide) -> ModpackResult<()> { + if !self.env.supports(side) { + return Ok(()); + } + + let output = dest.join(&self.path); + + // HACK: Since Daedalus appends a file name to all mirrors and the manifest supplies full + // URLs, I'm supplying it with an empty string to avoid reinventing the wheel. + let bytes = download_file_mirrors( + "", + &self + .downloads + .iter() + .map(|it| it.as_str()) + .collect::>() + .as_slice(), + Some(&self.hashes.sha1), + ) + .await?; + fs::create_dir_all(output.parent().unwrap()).await?; + fs::write(output, bytes).await?; + Ok(()) + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum ModpackEnv { + ClientOnly, + ServerOnly, + Both, +} + +impl ModpackEnv { + pub fn supports(&self, side: ModpackSide) -> bool { + match self { + Self::ClientOnly => side == ModpackSide::Client, + Self::ServerOnly => side == ModpackSide::Server, + Self::Both => true, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum ModpackSide { + Client, Server, +} + + +#[derive(Debug, Clone, PartialEq)] +pub struct ModpackFileHashes { + pub sha1: String, +} + From 5ffddd6c8aa3a2c15925df635c7c9002be04f17f Mon Sep 17 00:00:00 2001 From: Daniel Hutzley Date: Sun, 5 Dec 2021 10:41:56 -0800 Subject: [PATCH 2/4] Added modpack creation and file adding --- Cargo.lock | 12 ++ theseus/Cargo.toml | 1 + theseus/src/modpack/manifest.rs | 15 +-- theseus/src/modpack/mod.rs | 8 +- theseus/src/modpack/modrinth_api.rs | 176 ++++++++++++++++++++++++++++ theseus/src/modpack/pack.rs | 152 +++++++++++++++++++++--- 6 files changed, 339 insertions(+), 25 deletions(-) create mode 100644 theseus/src/modpack/modrinth_api.rs diff --git a/Cargo.lock b/Cargo.lock index c1b51820a..87e05e039 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -46,6 +46,17 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38de00daab4eac7d753e97697066238d67ce9d7e2d823ab4f72fe14af29f3f33" +[[package]] +name = "async-trait" +version = "0.1.51" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44318e776df68115a881de9a8fd1b9e53368d7a4a5ce4cc48517da3393233a5e" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "autocfg" version = "1.0.1" @@ -1022,6 +1033,7 @@ name = "theseus" version = "0.1.0" dependencies = [ "argh", + "async-trait", "bytes", "chrono", "daedalus", diff --git a/theseus/Cargo.toml b/theseus/Cargo.toml index b6a71ea89..283d57262 100644 --- a/theseus/Cargo.toml +++ b/theseus/Cargo.toml @@ -8,6 +8,7 @@ edition = "2018" [dependencies] thiserror = "1.0" +async-trait = "0.1.51" daedalus = "0.1.6" diff --git a/theseus/src/modpack/manifest.rs b/theseus/src/modpack/manifest.rs index 4e6973162..dab367429 100644 --- a/theseus/src/modpack/manifest.rs +++ b/theseus/src/modpack/manifest.rs @@ -1,3 +1,4 @@ +use std::collections::HashSet; use std::path::{Path, PathBuf}; use std::convert::TryFrom; @@ -11,7 +12,7 @@ use serde::{Deserialize, Serialize}; pub const DEFAULT_FORMAT_VERSION: u32 = 1; -#[derive(Debug, Deserialize, Serialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct Manifest<'a> { pub format_version: u32, @@ -32,7 +33,7 @@ impl TryFrom> for pack::Modpack { .files .into_iter() .map(pack::ModpackFile::try_from) - .collect::>>()?; + .collect::>>()?; Ok(Self { name: String::from(manifest.name), @@ -60,7 +61,7 @@ fn get_loader_version(loader: ModLoader, version: &str) -> ModpackResult .iter() .find(|&it| it.id == version) .ok_or(ModpackError::VersionError(format!( - "No versions of {:?} exist for Minecraft {}", + "No versions of modloader {:?} exist for Minecraft {}", loader, version )))? .loaders[&LoaderType::Latest] @@ -94,7 +95,7 @@ impl<'a> TryFrom<&'a pack::Modpack> for Manifest<'a> { } } -#[derive(Debug, Deserialize, Serialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(rename_all = "camelCase")] pub struct ManifestFile<'a> { #[serde(borrow)] @@ -134,7 +135,7 @@ impl<'a> From<&'a pack::ModpackFile> for ManifestFile<'a> { } } -#[derive(Debug, Deserialize, Serialize, PartialEq)] +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] pub struct ManifestHashes<'a> { pub sha1: &'a str, } @@ -153,7 +154,7 @@ impl<'a> From<&'a pack::ModpackFileHashes> for ManifestHashes<'a> { } } -#[derive(Debug, Deserialize, Serialize, PartialEq)] +#[derive(Clone, Copy, Debug, Deserialize, Serialize, PartialEq)] pub struct ManifestEnvs { pub client: ManifestEnv, pub server: ManifestEnv, @@ -208,7 +209,7 @@ impl From for ManifestEnvs { } } -#[derive(Debug, Deserialize, Serialize, PartialEq)] +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq)] #[serde(untagged)] // HACK: I've tried for hours to get this working zero-copy, but I'm beat. If someone else wants to // go through the # = Result; diff --git a/theseus/src/modpack/modrinth_api.rs b/theseus/src/modpack/modrinth_api.rs new file mode 100644 index 000000000..bd663d9ee --- /dev/null +++ b/theseus/src/modpack/modrinth_api.rs @@ -0,0 +1,176 @@ +use std::{ + collections::HashSet, + convert::TryFrom, + path::{Path, PathBuf}, +}; + +use crate::launcher::ModLoader; + +use super::{ + manifest::{ManifestEnv, ManifestEnvs, ManifestHashes}, + pack::{ModpackEnv, ModpackFile, ModpackFileHashes, ModpackGame}, + ModpackError, ModpackResult, +}; +use async_trait::async_trait; +use bytes::Bytes; +use futures::future::try_join_all; +use serde::Deserialize; +use tokio::try_join; + +#[async_trait] +pub trait ModrinthAPI { + async fn get_latest_version( + &self, + project: &str, + channel: &str, + game: &ModpackGame, + ) -> ModpackResult>; + async fn get_version(&self, version: &str) -> ModpackResult>; +} + +#[derive(Debug)] +pub struct ModrinthV1(pub String); + +#[derive(Debug, Deserialize)] +struct ModrinthV1Project<'a> { + title: &'a str, + client_side: &'a str, + server_side: &'a str, +} + +#[derive(Debug, Deserialize)] +struct ModrinthV1ProjectVersion<'a> { + #[serde(borrow)] + dependencies: HashSet<&'a str>, + #[serde(borrow)] + game_versions: HashSet<&'a str>, + version_type: &'a str, + files: Vec>, + #[serde(borrow)] + loaders: HashSet<&'a str>, +} + +#[derive(Clone, Debug, Deserialize)] +struct ModrinthV1ProjectVersionFile<'a> { + hashes: ManifestHashes<'a>, + url: &'a str, + filename: &'a str, +} + +impl From> for ModpackFile { + fn from(file: ModrinthV1ProjectVersionFile<'_>) -> Self { + Self { + hashes: ModpackFileHashes::from(file.hashes), + downloads: { + let mut downloads: HashSet = HashSet::new(); + downloads.insert(String::from(file.url)); + downloads + }, + path: PathBuf::from(file.filename), + // WARNING: Since the sidedness of version 1 API requests is unknown, the environemnt is + // set here as both. + env: ModpackEnv::Both, + } + } +} + +#[async_trait] +impl ModrinthAPI for ModrinthV1 { + async fn get_latest_version( + &self, + project: &str, + channel: &str, + game: &ModpackGame, + ) -> ModpackResult> { + // Fetch metadata + let (project_json, versions_json): (Bytes, Bytes) = try_join!( + try_get_json(format!("{}/api/v1/mod/{}", self.0, project)), + try_get_json(format!("{}/api/v1/mod/{}/version", self.0, project)), + )?; + + let (mut project_deserializer, mut versions_deserializer) = ( + serde_json::Deserializer::from_slice(&project_json), + serde_json::Deserializer::from_slice(&versions_json), + ); + + let (project, versions) = ( + ModrinthV1Project::deserialize(&mut project_deserializer)?, + Vec::deserialize(&mut versions_deserializer)?, + ); + + let (game_version, loader) = match game { + ModpackGame::Minecraft(_, ModLoader::Vanilla) => Err(ModpackError::VersionError( + String::from("Modrinth V1 does not support vanilla projects"), + )), + ModpackGame::Minecraft(ref version, ref loader) => Ok((version, loader)), + _ => Err(ModpackError::VersionError(String::from( + "Attempted to use Modrinth API V1 to install a non-Minecraft project!", + ))), + }?; + + let version: ModrinthV1ProjectVersion = versions + .into_iter() + .find(|it: &ModrinthV1ProjectVersion| { + let loader_str = match loader { + ModLoader::Fabric => "fabric", + ModLoader::Forge => "forge", + ModLoader::Vanilla => unreachable!(), + }; + it.version_type == channel + && it.game_versions.contains(&game_version.as_str()) + && it.loaders.contains(&loader_str) + }) + .ok_or(ModpackError::VersionError(format!( + "Unable to find compatible version of mod {}", + project.title + )))?; + + // Project fields + let envs = ModpackEnv::try_from(ManifestEnvs { + client: serde_json::from_str(project.client_side)?, + server: serde_json::from_str(project.server_side)?, + })?; + + // Conversions + let files = version + .files + .iter() + .cloned() + .map(ModpackFile::from) + .collect::>(); + + let dep_futures = version.dependencies.iter().map(|it| self.get_version(&it)); + let deps = try_join_all(dep_futures) + .await? + .into_iter() + .flatten() + .collect::>(); + + Ok(files + .into_iter() + .chain(deps.into_iter()) + .map(|mut it| { + it.env = envs; + it + }) + .collect()) + } + + async fn get_version(&self, version: &str) -> ModpackResult> { + let version_json = try_get_json(format!("{}/api/v1/version/{}", self.0, version)).await?; + let mut version_deserializer = serde_json::Deserializer::from_slice(&version_json); + let version = ModrinthV1ProjectVersion::deserialize(&mut version_deserializer)?; + let base_path = PathBuf::from("mods/"); + + Ok(version + .files + .into_iter() + .map(ModpackFile::from) + .collect::>()) + } +} + +// Helpers +async fn try_get_json(url: String) -> ModpackResult { + Ok(reqwest::get(url).await?.error_for_status()?.bytes().await?) +} diff --git a/theseus/src/modpack/pack.rs b/theseus/src/modpack/pack.rs index bd9a783e2..367ea0a9e 100644 --- a/theseus/src/modpack/pack.rs +++ b/theseus/src/modpack/pack.rs @@ -1,18 +1,25 @@ -use std::path::{Path, PathBuf}; use daedalus::download_file_mirrors; use futures::future; +use std::{ + collections::HashSet, + hash::Hash, + path::{Path, PathBuf}, +}; use tokio::fs; +use super::{ + modrinth_api::{self, ModrinthV1}, + ModpackResult, +}; use crate::launcher::ModLoader; -use super::ModpackResult; -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct Modpack { pub game: ModpackGame, pub version: String, pub name: String, pub summary: Option, - pub files: Vec, + pub files: HashSet, } impl Modpack { @@ -31,22 +38,84 @@ impl Modpack { // TODO Integrate instance format to save other metadata Ok(()) } + + pub fn new(game: ModpackGame, version: &str, name: &str, summary: Option<&str>) -> Self { + Self { + game, + version: String::from(version), + name: String::from(name), + summary: summary.map(String::from), + files: HashSet::new(), + } + } + + pub async fn add_project( + &mut self, + project: &str, + base_path: &Path, + source: Option<&dyn modrinth_api::ModrinthAPI>, + channel: Option<&str>, + ) -> ModpackResult<()> { + let default_api = ModrinthV1(String::from("https://api.modrinth.com")); + let channel = channel.unwrap_or("release"); + let source = source.unwrap_or(&default_api); + + let files = source + .get_latest_version(project, channel, &self.game) + .await? + .into_iter() + .map(|mut it: ModpackFile| { + it.path = base_path.join(it.path); + it + }); + + self.files.extend(files); + Ok(()) + } + + pub async fn add_version( + &mut self, + version: &str, + base_path: &Path, + source: Option<&dyn modrinth_api::ModrinthAPI>, + ) -> ModpackResult<()> { + let default_api = ModrinthV1(String::from("https://api.modrinth.com")); + let source = source.unwrap_or(&default_api); + + let files = source + .get_version(version) + .await? + .into_iter() + .map(|mut it: ModpackFile| { + it.path = base_path.join(it.path); + it + }); + + self.files.extend(files); + Ok(()) + } } -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub enum ModpackGame { // TODO: Currently, the launcher does not support specifying mod loader versions, so I just // store the loader here. Minecraft(String, ModLoader), } - -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct ModpackFile { pub path: PathBuf, pub hashes: ModpackFileHashes, pub env: ModpackEnv, - pub downloads: Vec, + pub downloads: HashSet, +} + +impl Hash for ModpackFile { + fn hash(&self, state: &mut H) { + self.hashes.sha1.hash(state); + self.path.hash(state); + } } impl ModpackFile { @@ -76,13 +145,19 @@ impl ModpackFile { } } -#[derive(Debug, Clone, Copy, PartialEq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum ModpackEnv { ClientOnly, ServerOnly, Both, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ModpackSide { + Client, + Server, +} + impl ModpackEnv { pub fn supports(&self, side: ModpackSide) -> bool { match self { @@ -93,14 +168,59 @@ impl ModpackEnv { } } -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum ModpackSide { - Client, Server, -} - - -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Eq)] pub struct ModpackFileHashes { pub sha1: String, } +#[cfg(test)] +mod tests { + use std::{ + collections::HashSet, + path::{Path, PathBuf}, + }; + + use super::*; + use crate::launcher::ModLoader; + + #[tokio::test] + async fn add_version() -> ModpackResult<()> { + const TEST_VERSION: &'static str = "TpnSObJ7"; + let mut test_pack = Modpack::new( + ModpackGame::Minecraft(String::from("1.16.5"), ModLoader::Fabric), + "0.1.0", + "Example Modpack", + None, + ); + test_pack + .add_version(TEST_VERSION, Path::new("mods/"), None) + .await?; + + assert_eq!( + test_pack, + Modpack { + game: ModpackGame::Minecraft(String::from("1.16.5"), ModLoader::Fabric), + version: String::from("0.1.0"), + name: String::from("Example Modpack"), + summary: None, + files: { + let mut files = HashSet::new(); + files.insert(ModpackFile { + path: PathBuf::from("mods/gravestones-v1.9.jar"), + hashes: ModpackFileHashes { + sha1: String::from("3f0f6d523d218460310b345be03ab3f1d294e04d"), + }, + env: ModpackEnv::Both, + downloads: { + let mut downloads = HashSet::new(); + downloads.insert(String::from("https://cdn.modrinth.com/data/ssUbhMkL/versions/v1.9/gravestones-v1.9.jar")); + downloads + } + }); + files + }, + }, + ); + Ok(()) + } +} From fc076c2e54f3975f6787d171d95f7a1ffbe0be9a Mon Sep 17 00:00:00 2001 From: Daniel Hutzley Date: Sun, 5 Dec 2021 12:20:59 -0800 Subject: [PATCH 3/4] Add modpack creation and compilation --- Cargo.lock | 142 ++++++++++++++++++++++++++++++++++++ theseus/Cargo.toml | 1 + theseus/src/launcher/mod.rs | 4 +- theseus/src/modpack/mod.rs | 70 +++++++++++++++++- theseus/src/modpack/pack.rs | 12 +-- 5 files changed, 219 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 87e05e039..d6aaabb93 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -75,12 +75,39 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "block-buffer" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" +dependencies = [ + "block-padding", + "byte-tools", + "byteorder", + "generic-array", +] + +[[package]] +name = "block-padding" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" +dependencies = [ + "byte-tools", +] + [[package]] name = "bumpalo" version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f1e260c3a9040a7c19a12468758f4c16f31a81a1fe087482be9570ec864bb6c" +[[package]] +name = "byte-tools" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" + [[package]] name = "byteorder" version = "1.4.3" @@ -181,6 +208,15 @@ dependencies = [ "tokio", ] +[[package]] +name = "digest" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3d0c8c8752312f9713efd397ff63acb9f85585afbf179282e720e7704954dd5" +dependencies = [ + "generic-array", +] + [[package]] name = "encoding_rs" version = "0.8.29" @@ -190,6 +226,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "fake-simd" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" + [[package]] name = "flate2" version = "1.0.22" @@ -328,6 +370,15 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffdf9f34f1447443d37393cc6c2b8313aebddcd96906caf34e54c68d8e57d7bd" +dependencies = [ + "typenum", +] + [[package]] name = "getrandom" version = "0.2.3" @@ -504,6 +555,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json5" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b0db21af676c1ce64250b5f40f3ce2cf27e4e47cb91ed91eb6fe9350b430c1" +dependencies = [ + "pest", + "pest_derive", + "serde", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -534,6 +596,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "matches" version = "0.1.9" @@ -646,6 +714,12 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" +[[package]] +name = "opaque-debug" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2839e79665f131bdb5782e51f2c6c9599c133c6098982a54c794358bf432529c" + [[package]] name = "openssl" version = "0.10.38" @@ -716,6 +790,49 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + +[[package]] +name = "pest_derive" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "833d1ae558dc601e9a60366421196a8d94bc0ac980476d0b67e1d0988d72b2d0" +dependencies = [ + "pest", + "pest_generator", +] + +[[package]] +name = "pest_generator" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99b8db626e31e5b81787b9783425769681b347011cc59471e33ea46d2ea0cf55" +dependencies = [ + "pest", + "pest_meta", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pest_meta" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54be6e404f5317079812fc8f9f5279de376d8856929e21c184ecf6bbd692a11d" +dependencies = [ + "maplit", + "pest", + "sha-1", +] + [[package]] name = "pin-project-lite" version = "0.2.7" @@ -956,6 +1073,18 @@ dependencies = [ "serde", ] +[[package]] +name = "sha-1" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d94d0bede923b3cea61f3f1ff57ff8cdfd77b400fb8f9998949e0cf04163df" +dependencies = [ + "block-buffer", + "digest", + "fake-simd", + "opaque-debug", +] + [[package]] name = "sha1" version = "0.6.0" @@ -1039,6 +1168,7 @@ dependencies = [ "daedalus", "fs_extra", "futures", + "json5", "lazy_static", "path-clean", "regex", @@ -1194,6 +1324,18 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +[[package]] +name = "typenum" +version = "1.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63708a265f51345575b27fe43f9500ad611579e764c79edbc2037b1121959ec" + +[[package]] +name = "ucd-trie" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + [[package]] name = "unicode-bidi" version = "0.3.7" diff --git a/theseus/Cargo.toml b/theseus/Cargo.toml index 283d57262..28eb816e9 100644 --- a/theseus/Cargo.toml +++ b/theseus/Cargo.toml @@ -15,6 +15,7 @@ daedalus = "0.1.6" reqwest = { version = "0.11", features = ["json"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +json5 = "0.4.1" chrono = { version = "0.4", features = ["serde"] } uuid = { version = "0.8", features = ["serde", "v4"] } bytes = "1" diff --git a/theseus/src/launcher/mod.rs b/theseus/src/launcher/mod.rs index 8bf104060..649918754 100644 --- a/theseus/src/launcher/mod.rs +++ b/theseus/src/launcher/mod.rs @@ -1,4 +1,5 @@ use daedalus::minecraft::{ArgumentType, VersionInfo}; +use serde::{Deserialize, Serialize}; use std::path::Path; use std::process::{Command, Stdio}; use thiserror::Error; @@ -65,7 +66,8 @@ pub async fn fetch_metadata() -> Result< Ok((game?, forge?, fabric?)) } -#[derive(Debug, Eq, PartialEq, Clone, Copy)] +#[derive(Debug, Eq, PartialEq, Clone, Copy, Deserialize, Serialize)] +#[serde(rename_all = "lowercase")] pub enum ModLoader { Vanilla, Forge, diff --git a/theseus/src/modpack/mod.rs b/theseus/src/modpack/mod.rs index f9a2f4e2c..8c3ad45da 100644 --- a/theseus/src/modpack/mod.rs +++ b/theseus/src/modpack/mod.rs @@ -4,18 +4,23 @@ use daedalus::download_file; use fs_extra::dir::CopyOptions; use serde::Deserialize; use std::{convert::TryFrom, env, io, path::Path}; -use tokio::fs; +use tokio::{fs, try_join}; use uuid::Uuid; use zip::ZipArchive; -use self::{manifest::Manifest, pack::Modpack}; +use self::{ + manifest::Manifest, + pack::{Modpack, ModpackGame}, +}; -pub mod pack; pub mod manifest; pub mod modrinth_api; +pub mod pack; +pub const COMPILED_PATH: &'static str = "compiled/"; pub const MANIFEST_PATH: &'static str = "index.json"; pub const OVERRIDES_PATH: &'static str = "overrides/"; +pub const PACK_JSON5_PATH: &'static str = "modpack.json5"; #[derive(thiserror::Error, Debug)] pub enum ModpackError { @@ -43,6 +48,9 @@ pub enum ModpackError { #[error("Error parsing json: {0}")] JsonError(#[from] serde_json::Error), + #[error("Error parsing json5: {0}")] + Json5Error(#[from] json5::Error), + #[error("Error joining futures: {0}")] JoinError(#[from] tokio::task::JoinError), @@ -50,7 +58,7 @@ pub enum ModpackError { VersionError(String), #[error("Error downloading file: {0}")] - FetchError(#[from] reqwest::Error) + FetchError(#[from] reqwest::Error), } type ModpackResult = Result; @@ -121,3 +129,57 @@ pub async fn realise_modpack( modpack.download_files(dest, side).await?; Ok(()) } + +pub fn to_pack_json5(pack: &Modpack) -> ModpackResult { + let json5 = json5::to_string(pack)?; + Ok(format!("// This modpack is managed using Theseus. It can be edited using either a Theseus-compatible launcher or manually.\n{}", json5)) +} + +lazy_static::lazy_static! { + static ref PACK_GITIGNORE: String = format!(r#" + {0} + "#, COMPILED_PATH); +} + +pub async fn create_modpack( + name: &str, + game: ModpackGame, + summary: Option<&str>, +) -> ModpackResult<()> { + let output_dir = Path::new("./").join(name); + let pack = Modpack::new(game, "0.1.0", name, summary); + + try_join!( + fs::create_dir(&output_dir), + fs::create_dir(output_dir.join(OVERRIDES_PATH)), + fs::write(output_dir.join(".gitignore"), PACK_GITIGNORE.as_str()), + fs::write(output_dir.join(PACK_JSON5_PATH), to_pack_json5(&pack)?), + )?; + + Ok(()) +} + +pub async fn compile_modpack(dir: &Path) -> ModpackResult<()> { + let result_dir = dir.join(COMPILED_PATH); + let pack: Modpack = json5::from_str(&fs::read_to_string(dir.join(PACK_JSON5_PATH)).await?)?; + + if dir.join(OVERRIDES_PATH).exists() { + fs_extra::dir::copy( + dir.join(OVERRIDES_PATH), + result_dir.join(OVERRIDES_PATH), + &CopyOptions::new(), + )?; + } + let manifest = Manifest::try_from(&pack)?; + + try_join!( + fs::create_dir(&result_dir), + fs::write( + result_dir.join(MANIFEST_PATH), + serde_json::to_string(&manifest)? + ), + )?; + + Ok(()) +} + diff --git a/theseus/src/modpack/pack.rs b/theseus/src/modpack/pack.rs index 367ea0a9e..b8e726a7c 100644 --- a/theseus/src/modpack/pack.rs +++ b/theseus/src/modpack/pack.rs @@ -1,5 +1,6 @@ use daedalus::download_file_mirrors; use futures::future; +use serde::{Deserialize, Serialize}; use std::{ collections::HashSet, hash::Hash, @@ -13,7 +14,7 @@ use super::{ }; use crate::launcher::ModLoader; -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)] pub struct Modpack { pub game: ModpackGame, pub version: String, @@ -96,14 +97,14 @@ impl Modpack { } } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)] pub enum ModpackGame { // TODO: Currently, the launcher does not support specifying mod loader versions, so I just // store the loader here. Minecraft(String, ModLoader), } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)] pub struct ModpackFile { pub path: PathBuf, pub hashes: ModpackFileHashes, @@ -145,7 +146,8 @@ impl ModpackFile { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Deserialize, Serialize, Clone, Copy, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] pub enum ModpackEnv { ClientOnly, ServerOnly, @@ -168,7 +170,7 @@ impl ModpackEnv { } } -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)] pub struct ModpackFileHashes { pub sha1: String, } From c8c69b44250f22f0db5c129d81e4ac81db8fb942 Mon Sep 17 00:00:00 2001 From: Danielle Hutzley Date: Tue, 14 Dec 2021 08:00:06 -0800 Subject: [PATCH 4/4] Add mod URL download support --- theseus/src/modpack/mod.rs | 3 +++ theseus/src/modpack/pack.rs | 38 +++++++++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 2 deletions(-) diff --git a/theseus/src/modpack/mod.rs b/theseus/src/modpack/mod.rs index 8c3ad45da..ba5d86ed3 100644 --- a/theseus/src/modpack/mod.rs +++ b/theseus/src/modpack/mod.rs @@ -59,6 +59,9 @@ pub enum ModpackError { #[error("Error downloading file: {0}")] FetchError(#[from] reqwest::Error), + + #[error("Invalid modpack source: {0} (set the WHITELISTED_MODPACK_DOMAINS environment variable to override)")] + SourceWhitelistError(String), } type ModpackResult = Result; diff --git a/theseus/src/modpack/pack.rs b/theseus/src/modpack/pack.rs index b8e726a7c..09079183d 100644 --- a/theseus/src/modpack/pack.rs +++ b/theseus/src/modpack/pack.rs @@ -10,10 +10,18 @@ use tokio::fs; use super::{ modrinth_api::{self, ModrinthV1}, - ModpackResult, + ModpackResult, ModpackError, }; use crate::launcher::ModLoader; +pub const MODRINTH_DEFAULT_MODPACK_DOMAINS: &'static [&'static str] = &[ + "cdn.modrinth.com", + "edge.forgecdn.net", + "github.com", + "raw.githubusercontent.com", +]; +pub const MODRINTH_MODPACK_DOMAIN_WHITELIST_VAR: &'static str = "WHITELISTED_MODPACK_DOMAINS"; + #[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)] pub struct Modpack { pub game: ModpackGame, @@ -95,6 +103,32 @@ impl Modpack { self.files.extend(files); Ok(()) } + + pub async fn add_file(&mut self, source: reqwest::Url, dest: &Path, hashes: Option, env: Option) -> ModpackResult<()> { + let whitelisted_domains = std::env::var(MODRINTH_MODPACK_DOMAIN_WHITELIST_VAR) + .map(|it| serde_json::from_str::>(&it).ok().unwrap()) + .unwrap_or( + MODRINTH_DEFAULT_MODPACK_DOMAINS + .iter() + .cloned() + .map(String::from) + .collect::>(), + ); + + if (whitelisted_domains.iter().find(String::from(source.host_str().unwrap())).is_none()) { + return Err(ModpackError::SourceWhitelistError(String::from(source.host_str().unwrap()))); + } + + let file = ModpackFile { + path: dest, + hashes, + env: env.unwrap_or(ModpackEnv::Both), + downloads: HashSet::from([String::from(source)]) + }; + + self.files.insert(file); + Ok(()) + } } #[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)] @@ -107,7 +141,7 @@ pub enum ModpackGame { #[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq)] pub struct ModpackFile { pub path: PathBuf, - pub hashes: ModpackFileHashes, + pub hashes: Option, pub env: ModpackEnv, pub downloads: HashSet, }