From 52ed070b5b14b994f54103fb1348fd7b1b3ae313 Mon Sep 17 00:00:00 2001 From: Daniel Hutzley Date: Sat, 20 Nov 2021 11:56:44 -0800 Subject: [PATCH] Add tests and example for modpack support --- Cargo.lock | 45 ++++ theseus/Cargo.toml | 6 + theseus/examples/download-pack.rs | 53 +++++ theseus/src/modpack/manifest.rs | 358 +++++++++++++++++++++++++----- theseus/src/modpack/mod.rs | 57 ++++- 5 files changed, 459 insertions(+), 60 deletions(-) create mode 100644 theseus/examples/download-pack.rs diff --git a/Cargo.lock b/Cargo.lock index bb4915ee..ff5ce47f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,35 @@ dependencies = [ "memchr", ] +[[package]] +name = "argh" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f023c76cd7975f9969f8e29f0e461decbdc7f51048ce43427107a3d192f1c9bf" +dependencies = [ + "argh_derive", + "argh_shared", +] + +[[package]] +name = "argh_derive" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48ad219abc0c06ca788aface2e3a1970587e3413ab70acd20e54b6ec524c1f8f" +dependencies = [ + "argh_shared", + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "argh_shared" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38de00daab4eac7d753e97697066238d67ce9d7e2d823ab4f72fe14af29f3f33" + [[package]] name = "autocfg" version = "1.0.1" @@ -329,6 +358,15 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +[[package]] +name = "heck" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "hermit-abi" version = "0.1.19" @@ -1000,6 +1038,7 @@ dependencies = [ name = "theseus" version = "0.1.0" dependencies = [ + "argh", "bytes", "chrono", "daedalus", @@ -1175,6 +1214,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" + [[package]] name = "unicode-xid" version = "0.2.2" diff --git a/theseus/Cargo.toml b/theseus/Cargo.toml index d89703d5..b6a71ea8 100644 --- a/theseus/Cargo.toml +++ b/theseus/Cargo.toml @@ -29,3 +29,9 @@ tokio = { version = "1", features = ["full"] } futures = "0.3" sys-info = "0.9.0" + +[dev-dependencies] +argh = "0.1.6" + +[[example]] +name = "download-pack" diff --git a/theseus/examples/download-pack.rs b/theseus/examples/download-pack.rs new file mode 100644 index 00000000..e41f15fe --- /dev/null +++ b/theseus/examples/download-pack.rs @@ -0,0 +1,53 @@ +use std::{path::PathBuf, time::Instant}; + +use argh::FromArgs; +use theseus::modpack::{fetch_modpack, manifest::ModpackSide}; + +#[derive(FromArgs)] +/// Simple modpack download +struct ModpackDownloader { + /// where to download to + #[argh(positional)] + url: String, + + /// where to put the resulting pack + #[argh(option, short = 'o')] + output: Option, + + /// the sha1 hash, if you want it checked + #[argh(option, short = 'c')] + hash: Option, + + /// use verbose logging + #[argh(switch, short = 'v')] + verbose: bool, +} + +// Simple logging helper +fn debug(msg: &str, verbose: bool) { + if verbose { + println!("{}", msg); + } +} + +#[tokio::main] +pub async fn main() { + let args = argh::from_env::(); + let dest = args.output.unwrap_or(PathBuf::from("./pack-download/")); + + debug( + &format!( + "Downloading pack {} to {}", + args.url, + dest.to_str().unwrap_or("?") + ), + args.verbose, + ); + + let start = Instant::now(); + fetch_modpack(&args.url, args.hash.as_deref(), &dest, ModpackSide::Client).await; + let end = start.elapsed(); + + println!("Download completed in {} seconds", end.as_secs_f32()); + debug("Done!", args.verbose); +} diff --git a/theseus/src/modpack/manifest.rs b/theseus/src/modpack/manifest.rs index 0d7312fd..448311bf 100644 --- a/theseus/src/modpack/manifest.rs +++ b/theseus/src/modpack/manifest.rs @@ -1,13 +1,18 @@ -use std::{convert::TryFrom, path::{Path, PathBuf}, str::FromStr}; +use std::{ + convert::TryFrom, + path::{Path, PathBuf}, + str::FromStr, +}; use daedalus::download_file_mirrors; -use tokio::fs; use futures::future; +use tokio::fs; use crate::launcher::ModLoader; use super::ModpackError; +#[derive(Debug, Clone, PartialEq)] pub struct Manifest { format_version: u64, game: ModpackGame, @@ -22,27 +27,36 @@ pub struct Manifest { impl Manifest { /// Download a modpack's files for a given side to a given destination /// Assumes the destination exists and is a directory - pub async fn download_files(&self, dest: &Path, side: &ModpackSide) -> Result<(), ModpackError> { - let handles = self.files.clone().into_iter() - .map(move |file| { - let (dest, side) = (dest.to_owned(), *side); - tokio::spawn(async move { - file.fetch(&dest, &side).await - }) - }); - future::try_join_all(handles).await?.into_iter().collect::>()?; + pub async fn download_files(&self, dest: &Path, side: ModpackSide) -> Result<(), ModpackError> { + let handles = self.files.clone().into_iter().map(move |file| { + let (dest, side) = (dest.to_owned(), side); + tokio::spawn(async move { file.fetch(&dest, side).await }) + }); + future::try_join_all(handles) + .await? + .into_iter() + .collect::>()?; // TODO Integrate instance format to save other metadata Ok(()) } } -fn try_get<'r, F, T>(manifest: &'r serde_json::Map, field: &str, caster: F) -> Result - where - F: Fn(&'r serde_json::Value) -> Option { - manifest.get(field) +fn try_get<'r, F, T>( + manifest: &'r serde_json::Map, + field: &str, + caster: F, +) -> Result +where + F: Fn(&'r serde_json::Value) -> Option, +{ + manifest + .get(field) .and_then(caster) - .ok_or(ModpackError::ManifestError(format!("Invalid or missing field: {}", field))) + .ok_or(ModpackError::ManifestError(format!( + "Invalid or missing field: {}", + field + ))) } impl TryFrom for Manifest { @@ -51,8 +65,9 @@ impl TryFrom for Manifest { fn try_from(value: serde_json::Value) -> Result { use ModpackError::ManifestError; - let value = value.as_object() - .ok_or(ManifestError(String::from("Manifest is not a JSON object!")))?; + let value = value.as_object().ok_or(ManifestError(String::from( + "Manifest is not a JSON object!", + )))?; let game = ModpackGame::new( try_get(value, "game", serde_json::Value::as_str)?, @@ -60,42 +75,68 @@ impl TryFrom for Manifest { )?; let files = try_get(value, "files", serde_json::Value::as_array)? - .iter().map(|it| -> Result { - let file = it.as_object() + .iter() + .map(|it| -> Result { + let file = it + .as_object() .ok_or(ManifestError(String::from("Malformed file: not an object")))?; let path = Path::new(try_get(file, "path", serde_json::Value::as_str)?); - let hashes = ModpackFileHashes::try_from(try_get(file, "hashes", serde_json::Value::as_object)?)?; - let sources = try_get(file, "sources", serde_json::Value::as_array)?.iter() + let hashes = ModpackFileHashes::try_from(try_get( + file, + "hashes", + serde_json::Value::as_object, + )?)?; + let downloads = try_get(file, "downloads", serde_json::Value::as_array)? + .iter() .map(serde_json::Value::as_str) .map(|it| it.map(String::from)) .collect::>>() - .ok_or(ManifestError(format!("Invalid source for path {}", path.to_str().unwrap_or("?"))))?; - let env: Option<[ModpackEnv;2]> = if let Some(env) = file.get("env") { - if !env.is_object() { - return Err(ManifestError(String::from("Env is provided, but is not an object!"))); - } - Some([ModpackEnv::from_str(env.get("client").and_then(serde_json::Value::as_str).unwrap_or_default())?, - ModpackEnv::from_str(env.get("server").and_then(serde_json::Value::as_str).unwrap_or_default())?]) - } else { - None - }; + .ok_or(ManifestError(format!( + "Invalid source for path {}", + path.to_str().unwrap_or("?") + )))?; + let env: Option<[ModpackEnv; 2]> = if let Some(env) = file.get("env") { + if !env.is_object() { + return Err(ManifestError(String::from( + "Env is provided, but is not an object!", + ))); + } + Some([ + ModpackEnv::from_str( + env.get("client") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(), + )?, + ModpackEnv::from_str( + env.get("server") + .and_then(serde_json::Value::as_str) + .unwrap_or_default(), + )?, + ]) + } else { + None + }; - ModpackFile::new(path, hashes, env, sources.as_slice()) - } - ).collect::, ModpackError>>()?; + ModpackFile::new(path, hashes, env, &downloads) + }) + .collect::, ModpackError>>()?; Ok(Self { format_version: try_get(value, "formatVersion", serde_json::Value::as_u64)?, game, version_id: String::from(try_get(value, "versionId", serde_json::Value::as_str)?), name: String::from(try_get(value, "name", serde_json::Value::as_str)?), - summary: value.get("summary").and_then(serde_json::Value::as_str).map(String::from), - files + summary: value + .get("summary") + .and_then(serde_json::Value::as_str) + .map(String::from), + files, }) } } +#[derive(Debug, Clone, PartialEq)] pub enum ModpackGame { // TODO: Currently, the launcher does not support specifying mod loader versions, so I just // store the loader here. @@ -103,7 +144,10 @@ pub enum ModpackGame { } impl ModpackGame { - pub fn new(game: &str, deps: &serde_json::Map) -> Result { + pub fn new( + game: &str, + deps: &serde_json::Map, + ) -> Result { match game { "minecraft" => { let game_version = String::from( @@ -135,15 +179,15 @@ impl ModpackGame { } } -#[derive(Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct ModpackFile { path: PathBuf, hashes: ModpackFileHashes, envs: Option<[ModpackEnv; 2]>, - sources: Vec, + downloads: Vec, } -#[derive(Clone, Copy)] +#[derive(Debug, Clone, Copy, PartialEq)] pub enum ModpackSide { Client = 0, Server = 1, @@ -153,25 +197,28 @@ impl ModpackFile { pub fn new( path: &Path, hashes: ModpackFileHashes, - envs: Option<[ModpackEnv;2]>, - sources: &[String], + envs: Option<[ModpackEnv; 2]>, + downloads: &[String], ) -> Result { - if !path.is_dir() { - return Err(ModpackError::ManifestError(format!("Modpack file {} is a directory!", path.to_str().unwrap_or("?")))); + if path.is_dir() { + return Err(ModpackError::ManifestError(format!( + "Modpack file {} is a directory!", + path.to_str().unwrap_or("?") + ))); } Ok(Self { path: PathBuf::from(path), hashes, envs, - sources: Vec::from(sources), + downloads: Vec::from(downloads), }) } - pub async fn fetch(&self, dest: &Path, side: &ModpackSide) -> Result<(), ModpackError> { + pub async fn fetch(&self, dest: &Path, side: ModpackSide) -> Result<(), ModpackError> { if let Some(envs) = &self.envs { - if envs[*side as usize] == ModpackEnv::Unsupported - || envs[(*side as usize + 1) % 2] == ModpackEnv::Required + if envs[side as usize] == ModpackEnv::Unsupported + || envs[(side as usize + 1) % 2] == ModpackEnv::Required { return Ok(()); } @@ -181,14 +228,24 @@ impl ModpackFile { // HACK: Since Daedalus appends a file name to all mirrors and the manifest supplies full // URLs, I'm supplying it with an empty string to avoid reinventing the wheel. - let bytes = download_file_mirrors("", &self.sources.iter().map(|it| it.as_str()).collect::>().as_slice(), Some(&self.hashes.sha1)).await?; + let bytes = download_file_mirrors( + "", + &self + .downloads + .iter() + .map(|it| it.as_str()) + .collect::>() + .as_slice(), + Some(&self.hashes.sha1), + ) + .await?; fs::create_dir_all(output.parent().unwrap()).await?; fs::write(output, bytes).await?; Ok(()) } } -#[derive(Clone)] +#[derive(Debug, Clone, PartialEq)] pub struct ModpackFileHashes { sha1: String, } @@ -202,7 +259,7 @@ impl TryFrom<&serde_json::Map> for ModpackFileHashes } } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq)] pub enum ModpackEnv { Required, Optional, @@ -218,7 +275,10 @@ impl FromStr for ModpackEnv { "required" => Ok(Required), "optional" => Ok(Optional), "unsupported" => Ok(Unsupported), - _ => Err(ModpackError::ManifestError(format!("Invalid environment support: {}", s))), + _ => Err(ModpackError::ManifestError(format!( + "Invalid environment support: {}", + s + ))), } } } @@ -228,3 +288,195 @@ impl Default for ModpackEnv { Self::Optional } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_simple() -> Result<(), ModpackError> { + const PACK_JSON: &'static str = r#" + { + "formatVersion": 1, + "game": "minecraft", + "versionId": "deadbeef", + "name": "Example Pack", + "files": [], + "dependencies": { + "minecraft": "1.17.1" + } + } + "#; + let expected_manifest = Manifest { + format_version: 1, + game: ModpackGame::Minecraft(String::from("1.17.1"), ModLoader::Vanilla), + version_id: String::from("deadbeef"), + name: String::from("Example Pack"), + summary: None, + files: Vec::new(), + }; + let manifest_json: serde_json::Value = + serde_json::from_str(PACK_JSON).expect("Error parsing pack JSON"); + let manifest = Manifest::try_from(manifest_json)?; + + assert_eq!(expected_manifest, manifest); + Ok(()) + } + + #[test] + fn parse_forge() -> Result<(), ModpackError> { + const PACK_JSON: &'static str = r#" + { + "formatVersion": 1, + "game": "minecraft", + "versionId": "deadbeef", + "name": "Example Pack", + "files": [ + { + "path": "mods/testmod.jar", + "hashes": { + "sha1": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "downloads": [ + "https://example.com/testmod.jar" + ] + } + ], + "dependencies": { + "minecraft": "1.17.1", + "forge": "37.0.110" + } + } + "#; + + let expected_manifest = Manifest { + format_version: 1, + game: ModpackGame::Minecraft(String::from("1.17.1"), ModLoader::Forge), + version_id: String::from("deadbeef"), + name: String::from("Example Pack"), + summary: None, + files: vec![ModpackFile::new( + Path::new("mods/testmod.jar"), + ModpackFileHashes { + sha1: String::from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + }, + None, + &[String::from("https://example.com/testmod.jar")], + )?], + }; + + let manifest_json: serde_json::Value = + serde_json::from_str(PACK_JSON).expect("Error parsing pack JSON"); + let manifest = Manifest::try_from(manifest_json)?; + + assert_eq!(expected_manifest, manifest); + Ok(()) + } + + #[test] + fn parse_fabric() -> Result<(), ModpackError> { + const PACK_JSON: &'static str = r#" + { + "formatVersion": 1, + "game": "minecraft", + "versionId": "deadbeef", + "name": "Example Pack", + "files": [ + { + "path": "mods/testmod.jar", + "hashes": { + "sha1": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "downloads": [ + "https://example.com/testmod.jar" + ] + } + ], + "dependencies": { + "minecraft": "1.17.1", + "fabric-loader": "0.9.0" + } + } + "#; + + let expected_manifest = Manifest { + format_version: 1, + game: ModpackGame::Minecraft(String::from("1.17.1"), ModLoader::Fabric), + version_id: String::from("deadbeef"), + name: String::from("Example Pack"), + summary: None, + files: vec![ModpackFile::new( + Path::new("mods/testmod.jar"), + ModpackFileHashes { + sha1: String::from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + }, + None, + &[String::from("https://example.com/testmod.jar")], + )?], + }; + + let manifest_json: serde_json::Value = + serde_json::from_str(PACK_JSON).expect("Error parsing pack JSON"); + let manifest = Manifest::try_from(manifest_json)?; + + assert_eq!(expected_manifest, manifest); + Ok(()) + } + + #[test] + fn parse_complete() -> Result<(), ModpackError> { + const PACK_JSON: &'static str = r#" + { + "formatVersion": 1, + "game": "minecraft", + "versionId": "deadbeef", + "name": "Example Pack", + "summary": "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.", + "files": [ + { + "path": "mods/testmod.jar", + "hashes": { + "sha1": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + }, + "env": { + "client": "required", + "server": "unsupported" + }, + "downloads": [ + "https://example.com/testmod.jar" + ] + } + ], + "dependencies": { + "minecraft": "1.17.1", + "forge": "37.0.110" + } + } + "#; + + let expected_manifest = Manifest { + format_version: 1, + game: ModpackGame::Minecraft(String::from("1.17.1"), ModLoader::Forge), + version_id: String::from("deadbeef"), + name: String::from("Example Pack"), + summary: Some(String::from("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.")), + files: vec![ + ModpackFile::new( + Path::new("mods/testmod.jar"), + ModpackFileHashes { + sha1: String::from("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), + }, + Some([ ModpackEnv::Required, ModpackEnv::Unsupported ]), + &[ String::from("https://example.com/testmod.jar") ], + )? + ], + }; + + let manifest_json: serde_json::Value = + serde_json::from_str(PACK_JSON).expect("Error parsing pack JSON"); + let manifest = Manifest::try_from(manifest_json)?; + + assert_eq!(expected_manifest, manifest); + Ok(()) + } +} diff --git a/theseus/src/modpack/mod.rs b/theseus/src/modpack/mod.rs index a3188175..f463fc92 100644 --- a/theseus/src/modpack/mod.rs +++ b/theseus/src/modpack/mod.rs @@ -1,12 +1,15 @@ //! Provides utilties for downloading and parsing modpacks -use std::{convert::TryFrom, io, path::Path}; +use daedalus::download_file; use fs_extra::dir::CopyOptions; +use std::{borrow::Borrow, convert::TryFrom, env, io, path::Path}; use tokio::fs; +use uuid::Uuid; +use zip::ZipArchive; use self::manifest::Manifest; -mod manifest; +pub mod manifest; pub const MANIFEST_PATH: &'static str = "index.json"; pub const OVERRIDES_PATH: &'static str = "overrides/"; @@ -19,6 +22,12 @@ pub enum ModpackError { #[error("I/O error while reading modpack: {0}")] FSExtraError(#[from] fs_extra::error::Error), + #[error("Error extracting archive: {0}")] + ZipError(#[from] zip::result::ZipError), + + #[error("Invalid modpack format: {0}")] + FormatError(String), + #[error("Invalid output directory: {0}")] InvalidDirectory(String), @@ -35,13 +44,44 @@ pub enum ModpackError { JoinError(#[from] tokio::task::JoinError), } +/// Realise a modpack from a given URL +pub async fn fetch_modpack( + url: &str, + sha1: Option<&str>, + dest: &Path, + side: manifest::ModpackSide, +) -> Result<(), ModpackError> { + let bytes = download_file(url, sha1).await?; + let mut archive = ZipArchive::new(io::Cursor::new(&bytes as &[u8]))?; + realise_modpack_zip(&mut archive, dest, side).await +} + +/// Realise a given modpack from a zip archive +pub async fn realise_modpack_zip( + archive: &mut ZipArchive, + dest: &Path, + side: manifest::ModpackSide, +) -> Result<(), ModpackError> { + let tmp = env::temp_dir().join(format!("theseus-{}/", Uuid::new_v4())); + archive.extract(&tmp)?; + realise_modpack(&tmp, dest, side).await +} + /// Realise a given modpack into an instance -pub async fn realise_modpack(dir: &Path, dest: &Path, side: &manifest::ModpackSide) -> Result<(), ModpackError> { +pub async fn realise_modpack( + dir: &Path, + dest: &Path, + side: manifest::ModpackSide, +) -> Result<(), ModpackError> { if dest.is_file() { - return Err(ModpackError::InvalidDirectory(String::from("Output is not a directory"))); + return Err(ModpackError::InvalidDirectory(String::from( + "Output is not a directory", + ))); } if dest.exists() && std::fs::read_dir(dest).map_or(false, |it| it.count() != 0) { - return Err(ModpackError::InvalidDirectory(String::from("Output directory is non-empty"))); + return Err(ModpackError::InvalidDirectory(String::from( + "Output directory is non-empty", + ))); } if !dest.exists() { fs::create_dir_all(dest).await?; @@ -57,9 +97,12 @@ pub async fn realise_modpack(dir: &Path, dest: &Path, side: &manifest::ModpackSi // NOTE: I'm using standard files here, since Serde does not support async readers let manifest_path = Some(dir.join(MANIFEST_PATH)) .filter(|it| it.exists() && it.is_file()) - .ok_or(ModpackError::ManifestError(String::from("Manifest missing or is not a file")))?; + .ok_or(ModpackError::ManifestError(String::from( + "Manifest missing or is not a file", + )))?; let manifest_file = std::fs::File::open(manifest_path)?; - let manifest_json: serde_json::Value = serde_json::from_reader(io::BufReader::new(manifest_file))?; + let manifest_json: serde_json::Value = + serde_json::from_reader(io::BufReader::new(manifest_file))?; let manifest = Manifest::try_from(manifest_json)?; // Realise manifest