AstralRinth/daedalus_client/src/forge.rs

use crate::{format_url, upload_file_to_bucket, Error};
use chrono::{DateTime, Utc};
use daedalus::download_file;
use daedalus::minecraft::{Argument, ArgumentType, Library, VersionType};
use daedalus::modded::{
    LoaderType, LoaderVersion, Manifest, PartialVersionInfo, Processor, SidedDataEntry,
};
use lazy_static::lazy_static;
use semver::{Version, VersionReq};
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::io::Read;
use std::sync::Arc;
use std::time::{Duration, Instant};
use tokio::sync::Mutex;
use log::info;

lazy_static! {
    static ref FORGE_MANIFEST_V1_QUERY: VersionReq =
        VersionReq::parse(">=8.0.684, <23.5.2851").unwrap();
    static ref FORGE_MANIFEST_V2_QUERY_P1: VersionReq =
        VersionReq::parse(">=23.5.2851, <31.2.52").unwrap();
    static ref FORGE_MANIFEST_V2_QUERY_P2: VersionReq =
        VersionReq::parse(">=32.0.1, <37.0.0").unwrap();
    static ref FORGE_MANIFEST_V3_QUERY: VersionReq = VersionReq::parse(">=37.0.0").unwrap();
}

pub async fn retrieve_data(uploaded_files: &mut Vec<String>) -> Result<(), Error> {
    let maven_metadata = fetch_maven_metadata(None).await?;
    let old_manifest = daedalus::modded::fetch_manifest(&*format_url(&*format!(
        "forge/v{}/manifest.json",
        daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
    )))
    .await
    .ok();

    let versions = Arc::new(Mutex::new(if let Some(old_manifest) = old_manifest {
        old_manifest.game_versions
    } else {
        Vec::new()
    }));

    let visited_assets_mutex = Arc::new(Mutex::new(Vec::new()));
    let uploaded_files_mutex = Arc::new(Mutex::new(Vec::new()));

    let mut version_futures = Vec::new();

    for (minecraft_version, loader_versions) in maven_metadata {
        let mut loaders = Vec::new();

        for loader_version_full in loader_versions {
            let loader_version = loader_version_full.split('-').into_iter().nth(1);

            if let Some(loader_version_raw) = loader_version {
                // This is a dirty hack to get around Forge not complying with SemVer, but whatever
                // Most of this is a hack anyways :(
                // Works for all forge versions!
                let split = loader_version_raw.split('.').collect::<Vec<&str>>();
                let loader_version = if split.len() >= 4 {
                    if split[0].parse::<i32>().unwrap_or(0) < 6 {
                        format!("{}.{}.{}", split[0], split[1], split[3])
                    } else {
                        format!("{}.{}.{}", split[1], split[2], split[3])
                    }
                } else {
                    loader_version_raw.to_string()
                };

                let version = Version::parse(&*loader_version)?;

                if FORGE_MANIFEST_V1_QUERY.matches(&version)
                    || FORGE_MANIFEST_V2_QUERY_P1.matches(&version)
                    || FORGE_MANIFEST_V2_QUERY_P2.matches(&version)
                    || FORGE_MANIFEST_V3_QUERY.matches(&version)
                {
                    loaders.push((loader_version_full, version))
                }
            }
        }

        if let Some((loader_version_full, version)) = loaders.into_iter().last() {
            version_futures.push(async {
                let versions_mutex = Arc::clone(&versions);
                let visited_assets = Arc::clone(&visited_assets_mutex);
                let uploaded_files_mutex = Arc::clone(&uploaded_files_mutex);
                async move {
                    {
                        if versions_mutex.lock().await.iter().any(|x| {
                            x.id == minecraft_version
                                && x.loaders
                                .get(&LoaderType::Latest)
                                .map(|x| x.id == loader_version_full)
                                .unwrap_or(false)
                        }) {
                            return Ok(());
                        }
                    }

                    info!("Forge - Installer Start {}", loader_version_full.clone());
                    let bytes = download_file(&*format!("https://maven.minecraftforge.net/net/minecraftforge/forge/{0}/forge-{0}-installer.jar", loader_version_full), None).await?;

                    let reader = std::io::Cursor::new(&*bytes);

                    if let Ok(mut archive) = zip::ZipArchive::new(reader) {
                        if FORGE_MANIFEST_V1_QUERY.matches(&version) {
                            let profile = {
                                let mut install_profile = archive.by_name("install_profile.json")?;

                                let mut contents = String::new();
                                install_profile.read_to_string(&mut contents)?;

                                serde_json::from_str::<ForgeInstallerProfileV1>(&*contents)?
                            };

                            let forge_universal_bytes = {
                                let mut forge_universal_file = archive.by_name(&*profile.install.file_path)?;
                                let mut forge_universal =  Vec::new();
                                forge_universal_file.read_to_end(&mut forge_universal)?;

                                bytes::Bytes::from(forge_universal)
                            };
                            let forge_universal_path = profile.install.path.clone();

                            let now = Instant::now();
                            let libs = futures::future::try_join_all(profile.version_info.libraries.into_iter().map(|mut lib| async {
                                if let Some(url) = lib.url {
                                    {
                                        let mut visited_assets = visited_assets.lock().await;

                                        if visited_assets.contains(&lib.name) {
                                            lib.url = Some(format_url("maven/"));

                                            return Ok::<Library, Error>(lib);
                                        } else {
                                            visited_assets.push(lib.name.clone())
                                        }
                                    }

                                    let artifact_path =
                                        daedalus::get_path_from_artifact(&*lib.name)?;

                                    let artifact = if lib.name == forge_universal_path {
                                        forge_universal_bytes.clone()
                                    } else {
                                        let mirrors = vec![&*url, "https://maven.creeperhost.net/", "https://libraries.minecraft.net/"];

                                        daedalus::download_file_mirrors(
                                            &*artifact_path,
                                            &mirrors,
                                            None,
                                        )
                                            .await?
                                    };

                                    lib.url = Some(format_url("maven/"));

                                    upload_file_to_bucket(
                                        format!("{}/{}", "maven", artifact_path),
                                        artifact.to_vec(),
                                        Some("application/java-archive".to_string()),
                                        uploaded_files_mutex.as_ref(),
                                    ).await?;
                                }

                                Ok::<Library, Error>(lib)
                            })).await?;

                            let elapsed = now.elapsed();
                            info!("Elapsed lib DL: {:.2?}", elapsed);

                            let new_profile = PartialVersionInfo {
                                id: profile.version_info.id,
                                inherits_from: profile.install.minecraft,
                                release_time: profile.version_info.release_time,
                                time: profile.version_info.time,
                                main_class: profile.version_info.main_class,
                                minecraft_arguments: profile.version_info.minecraft_arguments.clone(),
                                arguments: profile.version_info.minecraft_arguments.map(|x| [(ArgumentType::Game, x.split(' ').map(|x| Argument::Normal(x.to_string())).collect())].iter().cloned().collect()),
                                libraries: libs,
                                type_: profile.version_info.type_,
                                data: None,
                                processors: None
                            };

                            let version_path = format!(
                                "forge/v{}/versions/{}.json",
                                daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
                                new_profile.id
                            );

                            upload_file_to_bucket(
                                version_path.clone(),
                                serde_json::to_vec(&new_profile)?,
                                Some("application/json".to_string()),
                                uploaded_files_mutex.as_ref()
                            ).await?;

                            let mut map = HashMap::new();
                            map.insert(LoaderType::Latest, LoaderVersion {
                                id: loader_version_full,
                                url: format_url(&*version_path)
                            });
                            versions_mutex.lock().await.push(daedalus::modded::Version {
                                id: minecraft_version,
                                loaders: map
                            })
                        } else if FORGE_MANIFEST_V2_QUERY_P1.matches(&version) || FORGE_MANIFEST_V2_QUERY_P2.matches(&version) || FORGE_MANIFEST_V3_QUERY.matches(&version) {
                            let profile = {
                                let mut install_profile = archive.by_name("install_profile.json")?;

                                let mut contents = String::new();
                                install_profile.read_to_string(&mut contents)?;

                                serde_json::from_str::<ForgeInstallerProfileV2>(&*contents)?
                            };

                            let version_info = {
                                let mut install_profile = archive.by_name("version.json")?;

                                let mut contents = String::new();
                                install_profile.read_to_string(&mut contents)?;
                                serde_json::from_str::<PartialVersionInfo>(&*contents)?
                            };

                            let forge_universal_bytes = {
                                if let Some(path) = &profile.path {
                                    let mut forge_universal_file = archive.by_name(&*format!("maven/{}", daedalus::get_path_from_artifact(&*path)?))?;
                                    let mut forge_universal =  Vec::new();
                                    forge_universal_file.read_to_end(&mut forge_universal)?;

                                    Some(bytes::Bytes::from(forge_universal))
                                } else {
                                    None
                                }
                            };

                            let now = Instant::now();
                            let libs = futures::future::try_join_all(profile.libraries.into_iter().chain(version_info.libraries).map(|mut lib| async {
                                if let Some(ref mut downloads) = lib.downloads {
                                    if let Some(ref mut artifact) = downloads.artifact {
                                        {
                                            let mut visited_assets = visited_assets.lock().await;

                                            if visited_assets.contains(&lib.name) {
                                                artifact.url = format_url(&*format!("maven/{}", artifact.path));

                                                return Ok::<Library, Error>(lib);
                                            } else {
                                                visited_assets.push(lib.name.clone())
                                            }
                                        }

                                        let artifact_path =
                                            daedalus::get_path_from_artifact(&*lib.name)?;

                                        let artifact_bytes = if artifact.url.is_empty() {
                                            forge_universal_bytes.clone().unwrap_or_default()
                                        } else {
                                            daedalus::download_file(
                                                &*artifact.url,
                                                Some(&*artifact.sha1),
                                            )
                                                .await?
                                        };

                                        artifact.url = format_url(&*format!("maven/{}", artifact.path));

                                        upload_file_to_bucket(
                                            format!("{}/{}", "maven", artifact_path),
                                            artifact_bytes.to_vec(),
                                            Some("application/java-archive".to_string()),
                                            uploaded_files_mutex.as_ref()
                                        ).await?;
                                    }
                                }

                                Ok::<Library, Error>(lib)
                            })).await?;

                            let elapsed = now.elapsed();
                            info!("Elapsed lib DL: {:.2?}", elapsed);

                            let new_profile = PartialVersionInfo {
                                id: version_info.id,
                                inherits_from: version_info.inherits_from,
                                release_time: version_info.release_time,
                                time: version_info.time,
                                main_class: version_info.main_class,
                                minecraft_arguments: version_info.minecraft_arguments,
                                arguments: version_info.arguments,
                                libraries: libs,
                                type_: version_info.type_,
                                data: Some(profile.data),
                                processors: Some(profile.processors),
                            };

                            let version_path = format!(
                                "forge/v{}/versions/{}.json",
                                daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
                                new_profile.id
                            );

                            upload_file_to_bucket(
                                version_path.clone(),
                                serde_json::to_vec(&new_profile)?,
                                Some("application/json".to_string()),
                                uploaded_files_mutex.as_ref()
                            ).await?;

                            let mut map = HashMap::new();
                            map.insert(LoaderType::Latest, LoaderVersion {
                                id: loader_version_full,
                                url: format_url(&*version_path)
                            });
                            versions_mutex.lock().await.push(daedalus::modded::Version {
                                id: minecraft_version,
                                loaders: map
                            })
                        }
                    }

                    Ok::<(), Error>(())
                }.await?;

                Ok::<(), Error>(())
            });
        }
    }

    {
        let mut versions_peek = version_futures.into_iter().peekable();
        let mut chunk_index = 0;
        while versions_peek.peek().is_some() {
            info!("Chunk {} Start", chunk_index);
            let now = Instant::now();

            let chunk: Vec<_> = versions_peek.by_ref().take(100).collect();
            futures::future::try_join_all(chunk).await?;

            tokio::time::sleep(Duration::from_secs(1)).await;

            chunk_index += 1;

            let elapsed = now.elapsed();
            info!("Chunk {} Elapsed: {:.2?}", chunk_index, elapsed);
        }
    }

    if let Ok(versions) = Arc::try_unwrap(versions) {
        upload_file_to_bucket(
            format!(
                "forge/v{}/manifest.json",
                daedalus::modded::CURRENT_FORGE_FORMAT_VERSION,
            ),
            serde_json::to_vec(&Manifest {
                game_versions: versions.into_inner(),
            })?,
            Some("application/json".to_string()),
            uploaded_files_mutex.as_ref()
        )
        .await?;
    }

    if let Ok(uploaded_files_mutex) = Arc::try_unwrap(uploaded_files_mutex) {
        uploaded_files.extend(uploaded_files_mutex.into_inner());
    }

    Ok(())
}

const DEFAULT_MAVEN_METADATA_URL: &str =
    "https://files.minecraftforge.net/net/minecraftforge/forge/maven-metadata.json";

/// Fetches the forge maven metadata from the specified URL. If no URL is specified, the default is used.
/// Returns a hashmap specifying the versions of the forge mod loader
/// The hashmap key is a Minecraft version, and the value is the loader versions that work on
/// the specified Minecraft version
pub async fn fetch_maven_metadata(
    url: Option<&str>,
) -> Result<HashMap<String, Vec<String>>, Error> {
    Ok(serde_json::from_slice(
        &download_file(url.unwrap_or(DEFAULT_MAVEN_METADATA_URL), None).await?,
    )?)
}

#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct ForgeInstallerProfileInstallDataV1 {
    pub mirror_list: String,
    pub target: String,
    /// Path to the Forge universal library
    pub file_path: String,
    pub logo: String,
    pub welcome: String,
    pub version: String,
    /// Maven coordinates of the Forge universal library
    pub path: String,
    pub profile_name: String,
    pub minecraft: String,
}

#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct ForgeInstallerProfileManifestV1 {
    pub id: String,
    pub libraries: Vec<Library>,
    pub main_class: Option<String>,
    pub minecraft_arguments: Option<String>,
    pub release_time: DateTime<Utc>,
    pub time: DateTime<Utc>,
    pub type_: VersionType,
    pub assets: Option<String>,
    pub inherits_from: Option<String>,
    pub jar: Option<String>,
}

#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct ForgeInstallerProfileV1 {
    pub install: ForgeInstallerProfileInstallDataV1,
    pub version_info: ForgeInstallerProfileManifestV1,
}

#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct ForgeInstallerProfileV2 {
    pub spec: i32,
    pub profile: String,
    pub version: String,
    pub json: String,
    pub path: Option<String>,
    pub minecraft: String,
    pub data: HashMap<String, SidedDataEntry>,
    pub libraries: Vec<Library>,
    pub processors: Vec<Processor>,
}