didirus/AstralRinth
5
11
Fork 0
Code Issues 2 Pull Requests Packages Projects Releases 15 Wiki Activity

Merge tag 'v0.14.7' into beta

Browse Source 
v0.14.7
This commit is contained in:
didirus
2026-06-18 03:27:02 +03:00
parent f130a7e64b 8591bc8ebc
commit af99cb8109
65 changed files with 2223 additions and 355 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/app-lib/src/api/jre.rs
+2
View File
@@ -78,6 +78,7 @@ pub async fn auto_install_java(java_version: u32) -> crate::Result<PathBuf> {
),
None,
None,
None,
&state.fetch_semaphore,
&state.pool,
).await?;
@@ -92,6 +93,7 @@ pub async fn auto_install_java(java_version: u32) -> crate::Result<PathBuf> {
None,
None,
Some((&loading_bar, 80.0)),
None,
&state.fetch_semaphore,
&state.pool,
)
packages/app-lib/src/api/pack/import/curseforge.rs
+1
View File
@@ -82,6 +82,7 @@ pub async fn import_curseforge(
&thumbnail_url,
None,
None,
None,
&state.fetch_semaphore,
&state.pool,
)
packages/app-lib/src/api/pack/install_from.rs
+2
View File
@@ -326,6 +326,7 @@ pub async fn generate_pack_from_version_id(
None,
Some(&download_meta),
Some((&loading_bar, 70.0)),
None,
&state.fetch_semaphore,
&state.pool,
)
@@ -356,6 +357,7 @@ pub async fn generate_pack_from_version_id(
&icon_url,
None,
None,
None,
&state.fetch_semaphore,
&state.pool,
)
packages/app-lib/src/api/pack/install_mrpack.rs
+3
View File
@@ -441,6 +441,7 @@ pub async fn install_zipped_mrpack_files(
.collect::<Vec<&str>>(),
project.hashes.get(&PackFileHash::Sha1).map(|x| &**x),
Some(&download_meta),
None,
&state.fetch_semaphore,
&state.pool,
)
@@ -456,6 +457,7 @@ pub async fn install_zipped_mrpack_files(
project.path.as_str(),
project.hashes.get(&PackFileHash::Sha1).map(|x| &**x),
ProjectType::get_from_parent_folder(&path),
None,
&state.pool,
)
.await?;
@@ -514,6 +516,7 @@ pub async fn install_zipped_mrpack_files(
ProjectType::get_from_parent_folder(
relative_override_file_path.as_str(),
),
None,
&state.pool,
)
.await?;
packages/app-lib/src/api/profile/create.rs
+1
View File
@@ -113,6 +113,7 @@ pub async fn profile_create(
icon,
None,
None,
None,
&state.fetch_semaphore,
&state.pool,
)
packages/app-lib/src/api/profile/mod.rs
+37 -6
View File
@@ -547,6 +547,7 @@ pub async fn add_project_from_path(
bytes::Bytes::from(file),
None,
project_type,
None,
&state.io_semaphore,
&state.pool,
)
@@ -666,7 +667,7 @@ pub async fn export_mrpack(
}
// File is not in the config file, add it to the .mrpack zip
if path.is_file() {
if path.is_file() && is_path_exportable(&relative_path) {
let mut file = File::open(&path)
.await
.map_err(|e| IOError::with_path(e, &path))?;
@@ -695,6 +696,30 @@ pub async fn export_mrpack(
Ok(())
}
fn is_path_exportable(relative_path: &SafeRelativeUtf8UnixPathBuf) -> bool {
if relative_path.ends_with(".DS_Store") {
return false;
}
if relative_path.starts_with("mods/.connector/")
|| relative_path.starts_with(".sable/natives/")
|| relative_path.starts_with("local/crash_assistant/")
|| relative_path.starts_with("mods/mcef-libraries/")
|| relative_path.starts_with("mods/mcef-cache/")
|| relative_path.starts_with("config/super_resolution/libraries/")
|| relative_path.starts_with("config/Veinminer/update/")
|| relative_path.starts_with("config/epicfight/native/")
|| relative_path.starts_with("essential/")
|| relative_path.starts_with(".mixin.out/")
|| relative_path.starts_with(".fabric/")
|| relative_path.starts_with("__MACOSX/")
{
return false;
}
true
}
// Given a folder path, populate a Vec of all the subfolders and files, at most 2 layers deep
// profile
// -- folder1
@@ -726,14 +751,20 @@ pub async fn get_pack_export_candidates(
.await
.map_err(|e| IOError::with_path(e, &profile_base_dir))?
{
path_list.push(pack_get_relative_path(
&profile_base_dir,
&entry.path(),
)?);
let relative =
pack_get_relative_path(&profile_base_dir, &entry.path())?;
if !is_path_exportable(&relative) {
continue;
}
path_list.push(relative);
}
} else {
// One layer of files/folders if its a file
path_list.push(pack_get_relative_path(&profile_base_dir, &path)?);
let relative = pack_get_relative_path(&profile_base_dir, &path)?;
if !is_path_exportable(&relative) {
continue;
}
path_list.push(relative);
}
}
Ok(path_list)
packages/app-lib/src/error.rs
+2 -2
View File
@@ -68,8 +68,8 @@ pub enum ErrorKind {
#[error("Error fetching URL: {0}")]
FetchError(#[from] reqwest::Error),
#[error("Too many API errors; temporarily blocked")]
ApiIsDownError,
#[error("Too many API errors, try again in {0} minutes")]
ApiIsDownError(u32),
#[error("{0}")]
LabrinthError(LabrinthError),
packages/app-lib/src/launcher/download.rs
+18 -4
View File
@@ -88,6 +88,7 @@ pub async fn download_version_info(
&version.url,
None,
None,
None,
&st.api_semaphore,
&st.pool,
)
@@ -99,6 +100,7 @@ pub async fn download_version_info(
&loader.url,
None,
None,
None,
&st.api_semaphore,
&st.pool,
)
@@ -149,6 +151,7 @@ pub async fn download_client(
&client_download.url,
Some(&client_download.sha1),
None,
None,
&st.fetch_semaphore,
&st.pool,
)
@@ -189,6 +192,7 @@ pub async fn download_assets_index(
&version.asset_index.url,
None,
None,
None,
&st.fetch_semaphore,
&st.pool,
)
@@ -239,7 +243,7 @@ pub async fn download_assets(
async {
if !resource_path.exists() || force {
let resource = fetch_cell
.get_or_try_init(|| fetch(&url, Some(hash), None, &st.fetch_semaphore, &st.pool))
.get_or_try_init(|| fetch(&url, Some(hash), None, None, &st.fetch_semaphore, &st.pool))
.await?;
write(&resource_path, resource, &st.io_semaphore).await?;
tracing::trace!("Fetched asset with hash {hash}");
@@ -253,7 +257,7 @@ pub async fn download_assets(
if with_legacy && !resource_path.exists() || force {
let resource = fetch_cell
.get_or_try_init(|| fetch(&url, Some(hash), None, &st.fetch_semaphore, &st.pool))
.get_or_try_init(|| fetch(&url, Some(hash), None, None, &st.fetch_semaphore, &st.pool))
.await?;
write(&resource_path, resource, &st.io_semaphore).await?;
tracing::trace!("Fetched legacy asset with hash {hash}");
@@ -328,6 +332,7 @@ pub async fn download_libraries(
&native.url,
Some(&native.sha1),
None,
None,
&st.fetch_semaphore,
&st.pool,
)
@@ -373,6 +378,7 @@ pub async fn download_libraries(
&artifact.url,
Some(&artifact.sha1),
None,
None,
&st.fetch_semaphore,
&st.pool,
)
@@ -409,8 +415,15 @@ pub async fn download_libraries(
// failed download here is not a fatal condition.
//
// See DEV-479.
match fetch(&url, None, None, &st.fetch_semaphore, &st.pool)
.await
match fetch(
&url,
None,
None,
None,
&st.fetch_semaphore,
&st.pool,
)
.await
{
Ok(bytes) => {
write(&path, &bytes, &st.io_semaphore).await?;
@@ -470,6 +483,7 @@ pub async fn download_log_config(
&log_download.url,
Some(&log_download.sha1),
None,
None,
&st.fetch_semaphore,
&st.pool,
)
packages/app-lib/src/state/cache.rs
+78 -2
View File
@@ -372,6 +372,16 @@ pub struct CachedFileHash {
pub size: u64,
pub hash: String,
pub project_type: Option<ProjectType>,
#[serde(default)]
pub project_id: Option<String>,
#[serde(default)]
pub version_id: Option<String>,
}
#[derive(Clone, Copy, Debug)]
pub struct KnownModrinthFile<'a> {
pub project_id: &'a str,
pub version_id: &'a str,
}
#[derive(Serialize, Deserialize, Clone, Debug)]
@@ -1080,6 +1090,7 @@ impl CachedEntry {
method: Method,
api_url: &str,
url: &str,
uri_path: Option<&'static str>,
keys: &DashSet<impl Display + Eq + Hash + Serialize>,
fetch_semaphore: &FetchSemaphore,
pool: &SqlitePool,
@@ -1102,6 +1113,7 @@ impl CachedEntry {
url,
None,
None,
uri_path,
fetch_semaphore,
pool,
)
@@ -1112,11 +1124,12 @@ impl CachedEntry {
}
macro_rules! fetch_original_values {
($type:ident, $api_url:expr, $url_suffix:expr, $cache_variant:path) => {{
($type:ident, $api_url:expr, $url_suffix:expr, $uri_path:expr, $cache_variant:path) => {{
let mut results = fetch_many_batched(
Method::GET,
$api_url,
&format!("{}?ids=", $url_suffix),
$uri_path,
&keys,
&fetch_semaphore,
&pool,
@@ -1172,7 +1185,7 @@ impl CachedEntry {
}
macro_rules! fetch_original_value {
($type:ident, $api_url:expr, $url_suffix:expr, $cache_variant:path) => {{
($type:ident, $api_url:expr, $url_suffix:expr, $uri_path:expr, $cache_variant:path) => {{
vec![(
$cache_variant(
fetch_json(
@@ -1180,6 +1193,7 @@ impl CachedEntry {
&*format!("{}{}", $api_url, $url_suffix),
None,
None,
$uri_path,
&fetch_semaphore,
pool,
)
@@ -1197,6 +1211,7 @@ impl CachedEntry {
Project,
env!("MODRINTH_API_URL"),
"projects",
Some("/v2/projects"),
CacheValue::Project
)
}
@@ -1205,6 +1220,7 @@ impl CachedEntry {
ProjectV3,
env!("MODRINTH_API_URL_V3"),
"projects",
Some("/v3/projects"),
CacheValue::ProjectV3
)
}
@@ -1213,6 +1229,7 @@ impl CachedEntry {
Version,
env!("MODRINTH_API_URL"),
"versions",
Some("/v2/versions"),
CacheValue::Version
)
}
@@ -1221,6 +1238,7 @@ impl CachedEntry {
User,
env!("MODRINTH_API_URL"),
"users",
Some("/v2/users"),
CacheValue::User
)
}
@@ -1229,6 +1247,7 @@ impl CachedEntry {
Method::GET,
env!("MODRINTH_API_URL_V3"),
"teams?ids=",
Some("/v3/teams"),
&keys,
fetch_semaphore,
pool,
@@ -1268,6 +1287,7 @@ impl CachedEntry {
Method::GET,
env!("MODRINTH_API_URL_V3"),
"organizations?ids=",
Some("/v3/organizations"),
&keys,
fetch_semaphore,
pool,
@@ -1327,6 +1347,7 @@ impl CachedEntry {
"algorithm": "sha1",
"hashes": &keys,
})),
Some("/v2/version_files"),
fetch_semaphore,
pool,
)
@@ -1393,6 +1414,7 @@ impl CachedEntry {
url,
None,
None,
None,
fetch_semaphore,
pool,
)
@@ -1421,6 +1443,7 @@ impl CachedEntry {
"minecraft/v{}/manifest.json",
daedalus::minecraft::CURRENT_FORMAT_VERSION
),
None,
CacheValue::MinecraftManifest
)
}
@@ -1429,6 +1452,7 @@ impl CachedEntry {
Categories,
env!("MODRINTH_API_URL"),
"tag/category",
Some("/v2/tag/category"),
CacheValue::Categories
)
}
@@ -1437,6 +1461,7 @@ impl CachedEntry {
ReportTypes,
env!("MODRINTH_API_URL"),
"tag/report_type",
Some("/v2/tag/report_type"),
CacheValue::ReportTypes
)
}
@@ -1445,6 +1470,7 @@ impl CachedEntry {
Loaders,
env!("MODRINTH_API_URL"),
"tag/loader",
Some("/v2/tag/loader"),
CacheValue::Loaders
)
}
@@ -1453,6 +1479,7 @@ impl CachedEntry {
GameVersions,
env!("MODRINTH_API_URL"),
"tag/game_version",
Some("/v2/tag/game_version"),
CacheValue::GameVersions
)
}
@@ -1461,6 +1488,7 @@ impl CachedEntry {
DonationPlatforms,
env!("MODRINTH_API_URL"),
"tag/donation_platform",
Some("/v2/tag/donation_platform"),
CacheValue::DonationPlatforms
)
}
@@ -1503,6 +1531,8 @@ impl CachedEntry {
project_type: ProjectType::get_from_parent_folder(
&full_path,
),
project_id: None,
version_id: None,
})
.get_entry(),
true,
@@ -1617,6 +1647,7 @@ impl CachedEntry {
"game_versions": [game_version],
"version_types": version_types
})),
Some("/v2/version_files/update_many"),
fetch_semaphore,
pool,
)
@@ -1653,13 +1684,32 @@ impl CachedEntry {
let versions = variation.remove(hash);
if let Some(versions) = versions {
let mut emitted_update = false;
for version in versions {
let version_id = version.id.clone();
let target_hash = version
.files
.iter()
.find(|file| file.primary)
.or_else(|| version.files.first())
.and_then(|file| file.hashes.get("sha1"))
.map(String::as_str);
// Some update responses point at a different version ID for the exact installed file.
let same_file =
target_hash == Some(hash.as_str());
vals.push((
CacheValue::Version(version).get_entry(),
false,
));
if same_file {
continue;
}
emitted_update = true;
vals.push((
CacheValue::FileUpdate(CachedFileUpdate {
hash: hash.clone(),
@@ -1676,6 +1726,16 @@ impl CachedEntry {
true,
));
}
if !emitted_update {
vals.push((
CacheValueType::FileUpdate
.get_empty_entry(format!(
"{hash}-{loaders_key}-{channel_policy_key}-{game_version}"
)),
true,
));
}
} else {
vals.push((
CacheValueType::FileUpdate.get_empty_entry(
@@ -1713,6 +1773,7 @@ impl CachedEntry {
url,
None,
None,
Some("/v2/search"),
fetch_semaphore,
pool,
)
@@ -1754,6 +1815,7 @@ impl CachedEntry {
&url,
None,
None,
Some("/v2/project/:id/version"),
fetch_semaphore,
pool,
)
@@ -1805,6 +1867,7 @@ impl CachedEntry {
url,
None,
None,
Some("/v3/search"),
fetch_semaphore,
pool,
)
@@ -2061,6 +2124,7 @@ pub async fn cache_file_hash(
path: &str,
known_hash: Option<&str>,
project_type: Option<ProjectType>,
known_modrinth_file: Option<KnownModrinthFile<'_>>,
exec: impl sqlx::Executor<'_, Database = sqlx::Sqlite>,
) -> crate::Result<()> {
let size = bytes.len();
@@ -2077,6 +2141,7 @@ pub async fn cache_file_hash(
size as u64,
hash,
project_type,
known_modrinth_file,
exec,
)
.await
@@ -2088,8 +2153,17 @@ pub async fn cache_file_hash_metadata(
size: u64,
hash: String,
project_type: Option<ProjectType>,
known_modrinth_file: Option<KnownModrinthFile<'_>>,
exec: impl sqlx::Executor<'_, Database = sqlx::Sqlite>,
) -> crate::Result<()> {
let (project_id, version_id) =
known_modrinth_file.map_or((None, None), |metadata| {
(
Some(metadata.project_id.to_string()),
Some(metadata.version_id.to_string()),
)
});
// Streamed extraction already computed these values, so avoid buffering the file just to cache them.
CachedEntry::upsert_many(
&[CacheValue::FileHash(CachedFileHash {
@@ -2097,6 +2171,8 @@ pub async fn cache_file_hash_metadata(
size,
hash,
project_type,
project_id,
version_id,
})
.get_entry()],
exec,
packages/app-lib/src/state/friends.rs
+3
View File
@@ -331,6 +331,7 @@ impl FriendsSocket {
concat!(env!("MODRINTH_API_URL_V3"), "friends"),
None,
None,
Some("/v3/friends"),
semaphore,
exec,
)
@@ -359,6 +360,7 @@ impl FriendsSocket {
None,
None,
None,
Some("/v3/friend/:user_id"),
semaphore,
exec,
)
@@ -392,6 +394,7 @@ impl FriendsSocket {
None,
None,
None,
Some("/v3/friend/:user_id"),
semaphore,
exec,
)
packages/app-lib/src/state/instances/content.rs
+1
View File
@@ -895,6 +895,7 @@ async fn get_modpack_identifiers(
&[&primary_file.url],
primary_file.hashes.get("sha1").map(|s| s.as_str()),
Some(&download_meta),
None,
fetch_semaphore,
pool,
)
packages/app-lib/src/state/legacy_converter.rs
+4
View File
@@ -226,6 +226,10 @@ where
ProjectType::get_from_parent_folder(
&full_path,
),
project_id: Some(
version.project_id.clone(),
),
version_id: Some(version.id.clone()),
},
));
}
packages/app-lib/src/state/mr_auth.rs
+2
View File
@@ -36,6 +36,7 @@ impl ModrinthCredentials {
Some(("Authorization", &*creds.session)),
None,
None,
Some("/v2/session/refresh"),
semaphore,
exec,
)
@@ -228,6 +229,7 @@ async fn fetch_info(
Some(("Authorization", token)),
None,
None,
Some("/v2/user"),
semaphore,
exec,
)
packages/app-lib/src/state/profiles.rs
+215 -2
View File
@@ -2,8 +2,8 @@ use super::settings::{Hooks, MemorySettings, WindowSize};
use crate::profile::get_full_path;
use crate::state::server_join_log::JoinLogEntry;
use crate::state::{
CacheBehaviour, CachedEntry, CachedFile, CachedFileHash, ReleaseChannel,
cache_file_hash,
CacheBehaviour, CachedEntry, CachedFile, CachedFileHash, KnownModrinthFile,
ReleaseChannel, Version, cache_file_hash,
};
use crate::util;
use crate::util::fetch::{FetchSemaphore, IoSemaphore, write_cached_icon};
@@ -1022,6 +1022,16 @@ impl Profile {
.into_iter()
.map(|f| (f.hash.clone(), f))
.collect();
let file_info_by_hash = Self::resolve_installed_file_metadata(
&file_hashes,
&keys,
file_info_by_hash,
self,
cache_behaviour,
pool,
fetch_semaphore,
)
.await?;
let file_hashes = file_hashes
.into_iter()
@@ -1080,6 +1090,16 @@ impl Profile {
.into_iter()
.map(|f| (f.hash.clone(), f))
.collect();
let file_info_by_hash = Self::resolve_installed_file_metadata(
&file_hashes,
&keys,
file_info_by_hash,
self,
cache_behaviour,
pool,
fetch_semaphore,
)
.await?;
let installed_channels = Self::get_installed_update_channels(
&file_info_by_hash,
@@ -1331,6 +1351,192 @@ impl Profile {
)
}
async fn resolve_installed_file_metadata(
file_hashes: &[CachedFileHash],
scan_files: &[InitialScanFile],
mut file_info_by_hash: HashMap<String, CachedFile>,
profile: &Profile,
cache_behaviour: Option<CacheBehaviour>,
pool: &SqlitePool,
fetch_semaphore: &FetchSemaphore,
) -> crate::Result<HashMap<String, CachedFile>> {
let scan_files_by_path: HashMap<&str, &InitialScanFile> = scan_files
.iter()
.map(|file| (file.path.as_str(), file))
.collect();
struct MetadataCandidate {
hash: String,
project_id: String,
version_id: String,
file_name: String,
project_type: ProjectType,
}
let mut candidates = Vec::new();
let mut version_ids = HashSet::new();
for file_hash in file_hashes {
if let (Some(project_id), Some(version_id)) =
(&file_hash.project_id, &file_hash.version_id)
{
file_info_by_hash.insert(
file_hash.hash.clone(),
CachedFile {
hash: file_hash.hash.clone(),
project_id: project_id.clone(),
version_id: version_id.clone(),
},
);
continue;
}
let Some(file_info) = file_info_by_hash.get(&file_hash.hash) else {
continue;
};
let Some(scan_file) = scan_files_by_path
.get(file_hash.path.trim_end_matches(".disabled"))
else {
continue;
};
version_ids.insert(file_info.version_id.clone());
candidates.push(MetadataCandidate {
hash: file_hash.hash.clone(),
project_id: file_info.project_id.clone(),
version_id: file_info.version_id.clone(),
file_name: scan_file.file_name.clone(),
project_type: scan_file.project_type,
});
}
let version_ids_ref =
version_ids.iter().map(|id| id.as_str()).collect::<Vec<_>>();
let versions_by_id: HashMap<String, Version> =
CachedEntry::get_version_many(
&version_ids_ref,
None,
pool,
fetch_semaphore,
)
.await?
.into_iter()
.map(|version| (version.id.clone(), version))
.collect();
let mut project_versions_by_id: HashMap<String, Option<Vec<Version>>> =
HashMap::new();
for candidate in candidates {
if versions_by_id.get(&candidate.version_id).is_some_and(
|version| {
Self::version_matches_file(
version,
&candidate.hash,
&candidate.file_name,
candidate.project_type,
profile,
)
},
) {
continue;
}
if !project_versions_by_id.contains_key(&candidate.project_id) {
let versions = CachedEntry::get_project_versions(
&candidate.project_id,
cache_behaviour,
pool,
fetch_semaphore,
)
.await?;
project_versions_by_id
.insert(candidate.project_id.clone(), versions);
}
let Some(Some(versions)) =
project_versions_by_id.get(&candidate.project_id)
else {
continue;
};
if let Some(version) = Self::find_matching_file_version(
versions,
&candidate.hash,
&candidate.file_name,
candidate.project_type,
profile,
) {
file_info_by_hash.insert(
candidate.hash.clone(),
CachedFile {
hash: candidate.hash,
project_id: version.project_id.clone(),
version_id: version.id.clone(),
},
);
}
}
Ok(file_info_by_hash)
}
fn find_matching_file_version<'a>(
versions: &'a [Version],
hash: &str,
file_name: &str,
project_type: ProjectType,
profile: &Profile,
) -> Option<&'a Version> {
versions.iter().find(|version| {
Self::version_matches_file(
version,
hash,
file_name,
project_type,
profile,
)
})
}
fn version_matches_file(
version: &Version,
hash: &str,
file_name: &str,
project_type: ProjectType,
profile: &Profile,
) -> bool {
version.game_versions.contains(&profile.game_version)
&& Self::version_loaders_match_profile(
version,
project_type,
profile,
)
&& version.files.iter().any(|file| {
file.hashes.get("sha1").is_some_and(|sha1| sha1 == hash)
&& (file.primary
|| file.filename
== file_name.trim_end_matches(".disabled"))
})
}
fn version_loaders_match_profile(
version: &Version,
project_type: ProjectType,
profile: &Profile,
) -> bool {
if project_type == ProjectType::Mod {
version
.loaders
.iter()
.any(|loader| loader == profile.loader.as_str())
} else {
version.loaders.iter().any(|loader| {
project_type.get_loaders().contains(&loader.as_str())
})
}
}
#[tracing::instrument(skip(pool))]
pub async fn add_project_version(
profile_path: &str,
@@ -1382,6 +1588,7 @@ impl Profile {
&file.url,
file.hashes.get("sha1").map(|x| &**x),
Some(&download_meta),
None,
fetch_semaphore,
pool,
)
@@ -1393,6 +1600,10 @@ impl Profile {
bytes,
file.hashes.get("sha1").map(|x| &**x),
ProjectType::get_from_loaders(version.loaders.clone()),
Some(KnownModrinthFile {
project_id: &version.project_id,
version_id: &version.id,
}),
io_semaphore,
pool,
)
@@ -1408,6 +1619,7 @@ impl Profile {
bytes: bytes::Bytes,
hash: Option<&str>,
project_type: Option<ProjectType>,
known_modrinth_file: Option<KnownModrinthFile<'_>>,
io_semaphore: &IoSemaphore,
exec: impl sqlx::Executor<'_, Database = sqlx::Sqlite>,
) -> crate::Result<String> {
@@ -1455,6 +1667,7 @@ impl Profile {
&project_path,
hash,
Some(project_type),
known_modrinth_file,
exec,
)
.await?;
packages/app-lib/src/util/fetch.rs
+97 -16
View File
@@ -10,7 +10,7 @@ use rand::Rng;
use reqwest::Method;
use serde::de::DeserializeOwned;
use serde::{Deserialize, Serialize};
use std::collections::VecDeque;
use std::collections::{HashMap, VecDeque};
use std::ffi::OsStr;
use std::path::{Path, PathBuf};
use std::sync::LazyLock;
@@ -50,20 +50,48 @@ pub struct IoSemaphore(pub Semaphore);
pub struct FetchSemaphore(pub Semaphore);
struct FetchFence {
inner: Mutex<FenceInner>,
inner: Mutex<HashMap<&'static str, FenceInner>>,
}
impl FetchFence {
pub fn is_blocked(&self) -> bool {
self.inner.lock().is_blocked()
pub fn is_blocked(&self, key: &'static str) -> bool {
self.inner
.lock()
.entry(key)
.or_insert_with(FenceInner::new)
.is_blocked()
}
pub fn record_ok(&self) {
self.inner.lock().record_ok()
pub fn record_ok(&self, key: &'static str) {
self.inner
.lock()
.entry(key)
.or_insert_with(FenceInner::new)
.record_ok()
}
pub fn record_fail(&self) {
self.inner.lock().record_fail()
pub fn record_fail(&self, key: &'static str) {
self.inner
.lock()
.entry(key)
.or_insert_with(FenceInner::new)
.record_fail()
}
pub fn latest_block_minutes(&self) -> u32 {
let now = Utc::now();
self.inner
.lock()
.values()
.filter_map(|fence| fence.block_until)
.filter(|until| *until > now)
.max()
.map(|until| {
let seconds = until.signed_duration_since(now).num_seconds();
(seconds.max(0) as u32).div_ceil(60).max(1)
})
.unwrap_or(1)
}
}
@@ -154,7 +182,7 @@ impl FenceInner {
static GLOBAL_FETCH_FENCE: LazyLock<FetchFence> =
LazyLock::new(|| FetchFence {
inner: Mutex::new(FenceInner::new()),
inner: Mutex::new(HashMap::new()),
});
fn reqwest_client_builder() -> reqwest::ClientBuilder {
@@ -184,6 +212,7 @@ pub async fn fetch(
url: &str,
sha1: Option<&str>,
download_meta: Option<&DownloadMeta>,
uri_path: Option<&'static str>,
semaphore: &FetchSemaphore,
exec: impl sqlx::Executor<'_, Database = sqlx::Sqlite>,
) -> crate::Result<Bytes> {
@@ -195,6 +224,7 @@ pub async fn fetch(
None,
download_meta,
None,
uri_path,
semaphore,
exec,
)
@@ -206,6 +236,7 @@ pub async fn fetch_with_client(
url: &str,
sha1: Option<&str>,
download_meta: Option<&DownloadMeta>,
uri_path: Option<&'static str>,
semaphore: &FetchSemaphore,
exec: impl sqlx::Executor<'_, Database = sqlx::Sqlite>,
client: &reqwest::Client,
@@ -218,6 +249,7 @@ pub async fn fetch_with_client(
None,
download_meta,
None,
uri_path,
semaphore,
exec,
client,
@@ -231,6 +263,7 @@ pub async fn fetch_json<T>(
url: &str,
sha1: Option<&str>,
json_body: Option<serde_json::Value>,
uri_path: Option<&'static str>,
semaphore: &FetchSemaphore,
exec: impl sqlx::Executor<'_, Database = sqlx::Sqlite>,
) -> crate::Result<T>
@@ -238,7 +271,8 @@ where
T: DeserializeOwned,
{
let result = fetch_advanced(
method, url, sha1, json_body, None, None, None, semaphore, exec,
method, url, sha1, json_body, None, None, None, uri_path, semaphore,
exec,
)
.await?;
let value = serde_json::from_slice(&result)?;
@@ -257,6 +291,7 @@ pub async fn fetch_advanced(
header: Option<(&str, &str)>,
download_meta: Option<&DownloadMeta>,
loading_bar: Option<(&LoadingBarId, f64)>,
uri_path: Option<&'static str>,
semaphore: &FetchSemaphore,
exec: impl sqlx::Executor<'_, Database = sqlx::Sqlite>,
) -> crate::Result<Bytes> {
@@ -268,6 +303,7 @@ pub async fn fetch_advanced(
header,
download_meta,
loading_bar,
uri_path,
semaphore,
exec,
&INSECURE_REQWEST_CLIENT,
@@ -286,6 +322,7 @@ pub async fn fetch_advanced_with_client(
header: Option<(&str, &str)>,
download_meta: Option<&DownloadMeta>,
loading_bar: Option<(&LoadingBarId, f64)>,
uri_path: Option<&'static str>,
semaphore: &FetchSemaphore,
exec: impl sqlx::Executor<'_, Database = sqlx::Sqlite>,
client: &reqwest::Client,
@@ -294,6 +331,7 @@ pub async fn fetch_advanced_with_client(
let is_api_url = url.starts_with(env!("MODRINTH_API_URL"))
|| url.starts_with(env!("MODRINTH_API_URL_V3"));
let fence_key = if is_api_url { uri_path } else { None };
let creds = if header
.as_ref()
@@ -309,8 +347,13 @@ pub async fn fetch_advanced_with_client(
.map(|m| (DOWNLOAD_META_HEADER.to_string(), m.to_header_value()));
for attempt in 1..=(FETCH_ATTEMPTS + 1) {
if is_api_url && GLOBAL_FETCH_FENCE.is_blocked() {
return Err(ErrorKind::ApiIsDownError.into());
if let Some(fence_key) = fence_key
&& GLOBAL_FETCH_FENCE.is_blocked(fence_key)
{
return Err(ErrorKind::ApiIsDownError(
GLOBAL_FETCH_FENCE.latest_block_minutes(),
)
.into());
}
let mut req = client.request(method.clone(), url);
@@ -336,8 +379,8 @@ pub async fn fetch_advanced_with_client(
match result {
Ok(resp) => {
if resp.status().is_server_error() {
if is_api_url {
GLOBAL_FETCH_FENCE.record_fail();
if let Some(fence_key) = fence_key {
GLOBAL_FETCH_FENCE.record_fail(fence_key);
}
if attempt <= FETCH_ATTEMPTS {
@@ -400,8 +443,8 @@ pub async fn fetch_advanced_with_client(
tracing::trace!("Done downloading URL {url}");
if is_api_url {
GLOBAL_FETCH_FENCE.record_ok();
if let Some(fence_key) = fence_key {
GLOBAL_FETCH_FENCE.record_ok(fence_key);
}
return Ok(bytes);
@@ -427,6 +470,7 @@ pub async fn fetch_mirrors(
mirrors: &[&str],
sha1: Option<&str>,
download_meta: Option<&DownloadMeta>,
uri_path: Option<&'static str>,
semaphore: &FetchSemaphore,
exec: impl sqlx::Executor<'_, Database = sqlx::Sqlite> + Copy,
) -> crate::Result<Bytes> {
@@ -441,6 +485,7 @@ pub async fn fetch_mirrors(
mirror,
sha1,
download_meta,
uri_path,
semaphore,
exec,
&REQWEST_CLIENT,
@@ -620,6 +665,42 @@ mod tests {
assert!(fence.is_blocked());
}
#[test]
fn test_fetch_fence_keys_are_independent() {
let fence = FetchFence {
inner: Mutex::new(HashMap::new()),
};
for _ in 0..FenceInner::FAILURE_THRESHOLD {
fence.record_fail("/v3/version_file/:sha1/update");
}
assert!(fence.is_blocked("/v3/version_file/:sha1/update"));
assert!(!fence.is_blocked("/v3/project/:id"));
}
#[test]
fn test_fetch_fence_latest_block_minutes() {
let fence = FetchFence {
inner: Mutex::new(HashMap::new()),
};
{
let mut inner = fence.inner.lock();
inner.insert("/expired", FenceInner::new());
inner.get_mut("/expired").unwrap().block_until =
Some(Utc::now() - TimeDelta::minutes(1));
inner.insert("/short", FenceInner::new());
inner.get_mut("/short").unwrap().block_until =
Some(Utc::now() + TimeDelta::seconds(61));
inner.insert("/long", FenceInner::new());
inner.get_mut("/long").unwrap().block_until =
Some(Utc::now() + TimeDelta::seconds(140));
}
assert_eq!(fence.latest_block_minutes(), 3);
}
#[test]
fn test_fence_block_after_4_fails_with_oks() {
// Update tests if the FenceInner constants change