eightequal/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Legacy Forge Support (forgot to git add)

Browse Source
This commit is contained in:
Jai A
2021-10-17 23:23:27 -07:00
parent 16af479b83
commit 6528d3d7da
7 changed files with 376 additions and 134 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
LICENSE
View File

@@ -0,0 +1,7 @@
Copyright © 2021 Guavy LLC
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

12
README.md
View File

@@ -0,0 +1,12 @@
# Daedalus
Daedalus is a powerful tool which queries and generates metadata for the Minecraft (and other games in the future!) game
and mod loaders for:
- Performance (Serving static files can be easily cached and is extremely quick)
- Ease for Launcher Devs (Metadata is served in an easy to query and use format)
- Reliability (Provides a versioning system which ensures no breakage with updates)
Daedalus is currently a work in progress, but will support the original Minecraft data and reposting for all Forge and
Fabric artifacts.
Once Daedalus is done, Modrinth will provide full documentation for how to query from it and use it for your own launcher!

5
daedalus_client/Cargo.toml
View File

@@ -12,10 +12,15 @@ tokio = { version = "1", features = ["full"] }
futures = "0.3.17"
dotenv = "0.15.0"
log = "0.4.8"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
lazy_static = "1.4.0"
thiserror = "1.0"
reqwest = "0.11.4"
zip = "0.5.13"
semver = "1.0"
chrono = { version = "0.4", features = ["serde"] }
bytes = "1.1.0"
rusoto_core = "0.47.0"
rusoto_s3 = "0.47.0"

214
daedalus_client/src/fabric.rs
View File

@@ -2,21 +2,19 @@ use crate::{format_url, upload_file_to_bucket, Error};
use daedalus::fabric::PartialVersionInfo;
use daedalus::minecraft::Library;
use std::collections::HashMap;
use std::sync::{Arc, Mutex, RwLock};
use std::sync::{Arc};
use std::time::{Duration, Instant};
use futures::lock::Mutex;
pub async fn retrieve_data() -> Result<(), Error> {
let mut list = daedalus::fabric::fetch_fabric_versions(None).await?;
let loaders = RwLock::new(Vec::new());
let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new()));
if let Some(latest) = list.loader.get(0) {
let loaders_mutex = Arc::new(Mutex::new(Vec::new()));
let visited_artifacts_mutex = Arc::new(Mutex::new(Vec::new()));
{
let mut loaders = match loaders.write() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
let mut loaders = loaders_mutex.lock().await;
loaders.push(latest.version.clone());
@@ -33,131 +31,115 @@ pub async fn retrieve_data() -> Result<(), Error> {
.collect();
}
let mut versions = list
.game
.iter_mut()
.map(|game_version| {
let loaders = match loaders.read() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
let mut version_futures = Vec::new();
let visited_artifacts_mutex = Arc::clone(&visited_artifacts_mutex);
let game_version_mutex = Mutex::new(HashMap::new());
for game_version in list.game.iter_mut() {
let visited_artifacts_mutex = Arc::clone(&visited_artifacts_mutex);
let game_version_mutex = Mutex::new(HashMap::new());
let loaders_mutex = Arc::clone(&loaders_mutex);
version_futures.push(async move {
let versions = futures::future::try_join_all(loaders_mutex.lock().await.clone().into_iter().map(
|loader| async {
let version = daedalus::fabric::fetch_fabric_version(
&*game_version.version,
&*loader,
)
.await
.expect(&*format!("{}, {}", game_version.version, loader));
async move {
let versions = futures::future::try_join_all(loaders.clone().into_iter().map(
|loader| async {
let version = daedalus::fabric::fetch_fabric_version(
&*game_version.version,
&*loader,
)
.await
.expect(&*format!("{}, {}", game_version.version, loader));
Ok::<(String, PartialVersionInfo), Error>((loader, version))
},
))
.await?;
Ok::<(String, PartialVersionInfo), Error>((loader, version))
},
))
.await?;
futures::future::try_join_all(versions.into_iter().map(
|(loader, version)| async {
let libs = futures::future::try_join_all(
version.libraries.into_iter().map(|mut lib| async {
{
let mut visited_assets = visited_artifacts_mutex.lock().await;
futures::future::try_join_all(versions.into_iter().map(
|(loader, version)| async {
let libs = futures::future::try_join_all(
version.libraries.into_iter().map(|mut lib| async {
{
let mut visited_assets =
match visited_artifacts_mutex.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
if visited_assets.contains(&lib.name) {
lib.url = Some(format_url("maven/"));
if visited_assets.contains(&lib.name) {
lib.url = Some(format_url("maven/"));
return Ok(lib);
} else {
visited_assets.push(lib.name.clone())
}
return Ok(lib);
} else {
visited_assets.push(lib.name.clone())
}
}
let artifact_path =
daedalus::get_path_from_artifact(&*lib.name)?;
let artifact_path =
daedalus::get_path_from_artifact(&*lib.name)?;
let artifact = daedalus::download_file(
&*format!(
"{}{}",
lib.url.unwrap_or_else(|| {
"https://maven.fabricmc.net/".to_string()
}),
artifact_path
),
None,
)
.await?;
let artifact = daedalus::download_file(
&*format!(
"{}{}",
lib.url.unwrap_or_else(|| {
"https://maven.fabricmc.net/".to_string()
}),
artifact_path
),
None,
)
.await?;
lib.url = Some(format_url("maven/"));
lib.url = Some(format_url("maven/"));
upload_file_to_bucket(
format!("{}/{}", "maven", artifact_path),
artifact.to_vec(),
Some("application/java-archive".to_string()),
)
.await?;
upload_file_to_bucket(
format!("{}/{}", "maven", artifact_path),
artifact.to_vec(),
Some("application/java-archive".to_string()),
)
.await?;
Ok::<Library, Error>(lib)
}),
)
.await?;
Ok::<Library, Error>(lib)
}),
)
.await?;
let version_path = format!(
"fabric/v{}/versions/{}-{}.json",
daedalus::fabric::CURRENT_FORMAT_VERSION,
version.inherits_from,
loader
);
let version_path = format!(
"fabric/v{}/versions/{}-{}.json",
daedalus::fabric::CURRENT_FORMAT_VERSION,
version.inherits_from,
loader
);
upload_file_to_bucket(
version_path.clone(),
serde_json::to_vec(&PartialVersionInfo {
arguments: version.arguments,
id: version.id,
main_class: version.main_class,
release_time: version.release_time,
time: version.time,
type_: version.type_,
inherits_from: version.inherits_from,
libraries: libs,
})?,
Some("application/json".to_string()),
)
.await?;
upload_file_to_bucket(
version_path.clone(),
serde_json::to_vec(&PartialVersionInfo {
arguments: version.arguments,
id: version.id,
main_class: version.main_class,
release_time: version.release_time,
time: version.time,
type_: version.type_,
inherits_from: version.inherits_from,
libraries: libs,
})?,
Some("application/json".to_string()),
)
.await?;
{
let mut game_version_map = match game_version_mutex.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
game_version_map.insert(loader, format_url(&*version_path));
}
Ok::<(), Error>(())
},
))
.await?;
game_version.urls = Some(
match game_version_mutex.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
{
let mut game_version_map = game_version_mutex.lock().await;
game_version_map.insert(loader, format_url(&*version_path));
}
.clone(),
);
Ok::<(), Error>(())
}
})
.peekable();
Ok::<(), Error>(())
},
))
.await?;
game_version.urls = Some(
game_version_mutex.lock().await
.clone(),
);
Ok::<(), Error>(())
});
}
let mut versions = version_futures.into_iter().peekable();
let mut chunk_index = 0;
while versions.peek().is_some() {
let now = Instant::now();

210
daedalus_client/src/forge.rs
View File

@@ -0,0 +1,210 @@
use crate::{format_url, upload_file_to_bucket, Error};
use semver::{VersionReq, Version};
use lazy_static::lazy_static;
use daedalus::download_file;
use std::io::Read;
use tokio::sync::{Mutex};
use std::sync::{Arc};
use daedalus::minecraft::{Library, VersionType, ArgumentType, Argument};
use chrono::{DateTime, Utc};
use serde::{Serialize, Deserialize};
use daedalus::fabric::PartialVersionInfo;
use std::time::{Instant, Duration};
#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct ForgeInstallerProfileInstallDataV1 {
pub mirror_list: String,
pub target: String,
/// Path to the Forge universal library
pub file_path: String,
pub logo: String,
pub welcome: String,
pub version: String,
/// Maven coordinates of the Forge universal library
pub path: String,
pub profile_name: String,
pub minecraft: String,
}
#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct ForgeInstallerProfileManifestV1 {
pub id: String,
pub libraries: Vec<Library>,
pub main_class: Option<String>,
pub minecraft_arguments: Option<String>,
pub release_time: DateTime<Utc>,
pub time: DateTime<Utc>,
pub type_: VersionType,
pub assets: Option<String>,
pub inherits_from: Option<String>,
pub jar: Option<String>,
}
#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct ForgeInstallerProfileV1 {
pub install: ForgeInstallerProfileInstallDataV1,
pub version_info: ForgeInstallerProfileManifestV1,
}
lazy_static! {
static ref FORGE_MANIFEST_V1_QUERY: VersionReq = VersionReq::parse(">=8.0.684, <23.5.2851").unwrap();
}
pub async fn retrieve_data() -> Result<(), Error> {
let maven_metadata = daedalus::forge::fetch_maven_metadata(None).await?;
let visited_assets_mutex = Arc::new(Mutex::new(Vec::new()));
let mut version_futures = Vec::new();
for (minecraft_version, loader_versions) in maven_metadata {
if let Some(loader_version_full) = loader_versions.into_iter().last() {
let loader_version = loader_version_full.split('-').into_iter().nth(1);
if let Some(loader_version_raw) = loader_version {
// This is a dirty hack to get around Forge not complying with SemVer, but whatever
// Most of this is a hack anyways :(
// Works for all forge versions!
let split = loader_version_raw.split('.').collect::<Vec<&str>>();
let loader_version =if split.len() >= 4 {
if split[0].parse::<i32>().unwrap() < 6 {
format!("{}.{}.{}", split[0], split[1], split[3])
} else {
format!("{}.{}.{}", split[1], split[2], split[3])
}
} else {
loader_version_raw.to_string()
};
if FORGE_MANIFEST_V1_QUERY.matches(&Version::parse(&*loader_version).unwrap()) {
version_futures.push(async {
let visited_assets = Arc::clone(&visited_assets_mutex);
async move {
println!("installer start {}", loader_version_full.clone());
let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await.unwrap();
let reader = std::io::Cursor::new(&*bytes);
if let Ok(mut archive) = zip::ZipArchive::new(reader) {
let install_profile = {
let mut install_profile = archive.by_name("install_profile.json").unwrap();
let mut contents = String::new();
install_profile.read_to_string(&mut contents).unwrap();
contents
};
let profile = serde_json::from_str::<ForgeInstallerProfileV1>(&*install_profile).unwrap();
let forge_universal_bytes = {
let mut forge_universal_file = archive.by_name(&*profile.install.file_path).unwrap();
let mut forge_universal = Vec::new();
forge_universal_file.read_to_end(&mut forge_universal).unwrap();
bytes::Bytes::from(forge_universal)
};
let forge_universal_path = profile.install.file_path.clone();
let now = Instant::now();
let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async {
if let Some(url) = lib.url {
{
let mut visited_assets = visited_assets.lock().await;
if visited_assets.contains(&lib.name) {
lib.url = Some(format_url("maven/"));
return Ok::<Library, Error>(lib);
} else {
visited_assets.push(lib.name.clone())
}
}
let artifact_path =
daedalus::get_path_from_artifact(&*lib.name)?;
let artifact = if lib.name == forge_universal_path {
forge_universal_bytes.clone()
} else {
let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"];
daedalus::download_file_mirrors(
&*artifact_path,
&mirrors,
None,
)
.await?
};
lib.url = Some(format_url("maven/"));
upload_file_to_bucket(
format!("{}/{}", "maven", artifact_path),
artifact.to_vec(),
Some("application/java-archive".to_string()),
).await?;
}
Ok::<Library, Error>(lib)
})).await?;
let elapsed = now.elapsed();
println!("Elapsed lib DL: {:.2?}", elapsed);
let new_profile = PartialVersionInfo {
id: profile.version_info.id,
inherits_from: profile.install.minecraft,
release_time: profile.version_info.release_time,
time: profile.version_info.time,
main_class: profile.version_info.main_class,
arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()),
libraries: libs,
type_: profile.version_info.type_,
};
let version_path = format!(
"forge/v{}/versions/{}.json",
daedalus::forge::CURRENT_FORMAT_VERSION,
new_profile.id
);
upload_file_to_bucket(
version_path.clone(),
serde_json::to_vec(&new_profile)?,
Some("application/json".to_string()),
).await?;
}
Ok::<(), Error>(())
}.await?;
Ok::<(), Error>(())
});
}
}
}
}
let mut versions = version_futures.into_iter().peekable();
let mut chunk_index = 0;
while versions.peek().is_some() {
let now = Instant::now();
let chunk: Vec<_> = versions.by_ref().take(100).collect();
futures::future::try_join_all(chunk).await?;
std::thread::sleep(Duration::from_secs(1));
chunk_index += 1;
let elapsed = now.elapsed();
println!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
}
Ok(())
}

32
daedalus_client/src/main.rs
View File

@@ -3,9 +3,11 @@ use rusoto_core::credential::StaticProvider;
use rusoto_core::{HttpClient, Region, RusotoError};
use rusoto_s3::{PutObjectError, S3Client};
use rusoto_s3::{PutObjectRequest, S3};
use std::time::Duration;
mod fabric;
mod minecraft;
mod forge;
#[derive(thiserror::Error, Debug)]
pub enum Error {
@@ -32,7 +34,35 @@ async fn main() {
return;
}
fabric::retrieve_data().await.unwrap();
let mut timer = tokio::time::interval(Duration::from_secs(10 * 60));
loop {
timer.tick().await;
tokio::spawn(
async {
match fabric::retrieve_data().await {
Ok(..) => {}
Err(err) => error!("{:?}", err)
};
}
);
tokio::spawn(
async {
match minecraft::retrieve_data().await {
Ok(..) => {}
Err(err) => error!("{:?}", err)
};
}
);
tokio::spawn(
async {
match forge::retrieve_data().await {
Ok(..) => {}
Err(err) => error!("{:?}", err)
};
}
);
}
}
fn check_env_vars() -> bool {

30
daedalus_client/src/minecraft.rs
View File

@@ -1,7 +1,8 @@
use crate::{format_url, upload_file_to_bucket, Error};
use daedalus::download_file;
use std::sync::{Arc, Mutex};
use std::sync::{Arc};
use std::time::{Duration, Instant};
use futures::lock::Mutex;
pub async fn retrieve_data() -> Result<(), Error> {
let old_manifest =
@@ -19,10 +20,13 @@ pub async fn retrieve_data() -> Result<(), Error> {
let now = Instant::now();
let mut versions = manifest
let mut version_futures = Vec::new();
for version in manifest
.versions
.iter_mut()
.map(|version| async {
{
version_futures.push(async {
let old_version = if let Some(old_manifest) = &old_manifest {
old_manifest.versions.iter().find(|x| x.id == version.id)
} else {
@@ -58,10 +62,7 @@ pub async fn retrieve_data() -> Result<(), Error> {
let assets_index_url = version_info.asset_index.url.clone();
{
let mut cloned_manifest = match cloned_manifest_mutex.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
let mut cloned_manifest = cloned_manifest_mutex.lock().await;
let position = cloned_manifest
.versions
@@ -79,10 +80,7 @@ pub async fn retrieve_data() -> Result<(), Error> {
let mut download_assets = false;
{
let mut visited_assets = match visited_assets_mutex.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
let mut visited_assets = visited_assets_mutex.lock().await;
if !visited_assets.contains(&version_info.asset_index.id) {
if let Some(assets_hash) = assets_hash {
@@ -128,12 +126,13 @@ pub async fn retrieve_data() -> Result<(), Error> {
Ok::<(), Error>(())
}
.await?;
.await?;
Ok::<(), Error>(())
})
.peekable();
}
let mut versions = version_futures.into_iter().peekable();
let mut chunk_index = 0;
while versions.peek().is_some() {
let now = Instant::now();
@@ -154,10 +153,7 @@ pub async fn retrieve_data() -> Result<(), Error> {
"minecraft/v{}/manifest.json",
daedalus::minecraft::CURRENT_FORMAT_VERSION
),
serde_json::to_vec(&*match cloned_manifest.lock() {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
})?,
serde_json::to_vec(&*cloned_manifest.lock().await)?,
Some("application/json".to_string()),
)
.await?;