tomasalias/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Wire Profile Backend to Frontend (#71)

Browse Source 
* Search updates

* fixes2

* Some more work

* start instance page wiring

* Pack installation + Profile viewing

* Remove print statement

* Fix disappearing profiles

* fix compile err

* Finish Instance Running

* remove print statement

* fix prettier

* Fix clippy + early return
This commit is contained in:
Geometrically
2023-04-08 18:54:38 -07:00
committed by GitHub
parent 764d75181f
commit a62d931fe2
27 changed files with 502 additions and 739 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
theseus/src/api/pack.rs
View File

@@ -1,10 +1,13 @@
use crate::config::{MODRINTH_API_URL, REQWEST_CLIENT};
use crate::config::MODRINTH_API_URL;
use crate::data::ModLoader;
use crate::state::{ModrinthProject, ModrinthVersion, SideType};
use crate::util::fetch::{fetch, fetch_mirrors, write, write_cached_icon};
use crate::util::fetch::{
fetch, fetch_json, fetch_mirrors, write, write_cached_icon,
};
use crate::State;
use async_zip::tokio::read::seek::ZipFileReader;
use futures::TryStreamExt;
use reqwest::Method;
use serde::{Deserialize, Serialize};
use std::collections::HashMap;
use std::io::Cursor;
@@ -70,12 +73,15 @@ enum PackDependency {
pub async fn install_pack_from_version_id(
version_id: String,
) -> crate::Result<PathBuf> {
let version: ModrinthVersion = REQWEST_CLIENT
.get(format!("{}version/{}", MODRINTH_API_URL, version_id))
.send()
.await?
.json()
.await?;
let state = State::get().await?;
let version: ModrinthVersion = fetch_json(
Method::GET,
&format!("{}version/{}", MODRINTH_API_URL, version_id),
None,
&state.io_semaphore,
)
.await?;
let (url, hash) =
if let Some(file) = version.files.iter().find(|x| x.primary) {
@@ -92,28 +98,19 @@ pub async fn install_pack_from_version_id(
)
})?;
let file = async {
let state = &State::get().await?;
let semaphore = state.io_semaphore.acquire().await?;
fetch(&url, hash.map(|x| &**x), &semaphore).await
}
.await?;
let file = fetch(&url, hash.map(|x| &**x), &state.io_semaphore).await?;
let project: ModrinthProject = REQWEST_CLIENT
.get(format!(
"{}project/{}",
MODRINTH_API_URL, version.project_id
))
.send()
.await?
.json()
.await?;
let project: ModrinthProject = fetch_json(
Method::GET,
&format!("{}project/{}", MODRINTH_API_URL, version.project_id),
None,
&state.io_semaphore,
)
.await?;
let icon = if let Some(icon_url) = project.icon_url {
let state = State::get().await?;
let semaphore = state.io_semaphore.acquire().await?;
let icon_bytes = fetch(&icon_url, None, &semaphore).await?;
let icon_bytes = fetch(&icon_url, None, &state.io_semaphore).await?;
let filename = icon_url.rsplit('/').next();
@@ -123,7 +120,7 @@ pub async fn install_pack_from_version_id(
filename,
&state.directories.caches_dir(),
icon_bytes,
&semaphore,
&state.io_semaphore,
)
.await?,
)
@@ -244,8 +241,6 @@ async fn install_pack(
}
}
let permit = state.io_semaphore.acquire().await?;
let file = fetch_mirrors(
&project
.downloads
@@ -253,7 +248,7 @@ async fn install_pack(
.map(|x| &**x)
.collect::<Vec<&str>>(),
project.hashes.get(&PackFileHash::Sha1).map(|x| &**x),
&permit,
&state.io_semaphore,
)
.await?;
@@ -263,7 +258,8 @@ async fn install_pack(
match path {
Component::CurDir | Component::Normal(_) => {
let path = profile.join(project.path);
write(&path, &file, &permit).await?;
write(&path, &file, &state.io_semaphore)
.await?;
}
_ => {}
};
@@ -312,9 +308,12 @@ async fn install_pack(
}
if new_path.file_name().is_some() {
let permit = state.io_semaphore.acquire().await?;
write(&profile.join(new_path), &content, &permit)
.await?;
write(
&profile.join(new_path),
&content,
&state.io_semaphore,
)
.await?;
}
}
}

10
theseus/src/api/profile_create.rs
View File

@@ -137,7 +137,15 @@ pub async fn profile_create(
let path = canonicalize(&path)?;
let mut profile = Profile::new(name, game_version, path.clone()).await?;
if let Some(ref icon) = icon {
profile.set_icon(icon).await?;
let bytes = tokio::fs::read(icon).await?;
profile
.set_icon(
&state.directories.caches_dir(),
&state.io_semaphore,
bytes::Bytes::from(bytes),
&icon.to_string_lossy(),
)
.await?;
}
if let Some((loader_version, loader)) = loader {
profile.metadata.loader = loader;

41
theseus/src/launcher/download.rs
View File

@@ -62,8 +62,7 @@ pub async fn download_version_info(
}
info.id = version_id.clone();
let permit = st.io_semaphore.acquire().await?;
write(&path, &serde_json::to_vec(&info)?, &permit).await?;
write(&path, &serde_json::to_vec(&info)?, &st.io_semaphore).await?;
Ok(info)
}?;
@@ -93,11 +92,13 @@ pub async fn download_client(
.join(format!("{version}.jar"));
if !path.exists() {
let permit = st.io_semaphore.acquire().await?;
let bytes =
fetch(&client_download.url, Some(&client_download.sha1), &permit)
.await?;
write(&path, &bytes, &permit).await?;
let bytes = fetch(
&client_download.url,
Some(&client_download.sha1),
&st.io_semaphore,
)
.await?;
write(&path, &bytes, &st.io_semaphore).await?;
log::info!("Fetched client version {version}");
}
@@ -123,8 +124,7 @@ pub async fn download_assets_index(
.and_then(|ref it| Ok(serde_json::from_slice(it)?))
} else {
let index = d::minecraft::fetch_assets_index(version).await?;
let permit = st.io_semaphore.acquire().await?;
write(&path, &serde_json::to_vec(&index)?, &permit).await?;
write(&path, &serde_json::to_vec(&index)?, &st.io_semaphore).await?;
log::info!("Fetched assets index");
Ok(index)
}?;
@@ -154,25 +154,23 @@ pub async fn download_assets(
tokio::try_join! {
async {
if !resource_path.exists() {
let permit = st.io_semaphore.acquire().await?;
let resource = fetch_cell
.get_or_try_init(|| fetch(&url, Some(hash), &permit))
.get_or_try_init(|| fetch(&url, Some(hash), &st.io_semaphore))
.await?;
write(&resource_path, resource, &permit).await?;
write(&resource_path, resource, &st.io_semaphore).await?;
log::info!("Fetched asset with hash {hash}");
}
Ok::<_, crate::Error>(())
},
async {
if with_legacy {
let permit = st.io_semaphore.acquire().await?;
let resource = fetch_cell
.get_or_try_init(|| fetch(&url, Some(hash), &permit))
.get_or_try_init(|| fetch(&url, Some(hash), &st.io_semaphore))
.await?;
let resource_path = st.directories.legacy_assets_dir().join(
name.replace('/', &String::from(std::path::MAIN_SEPARATOR))
);
write(&resource_path, resource, &permit).await?;
write(&resource_path, resource, &st.io_semaphore).await?;
log::info!("Fetched legacy asset with hash {hash}");
}
Ok::<_, crate::Error>(())
@@ -219,10 +217,9 @@ pub async fn download_libraries(
artifact: Some(ref artifact),
..
}) => {
let permit = st.io_semaphore.acquire().await?;
let bytes = fetch(&artifact.url, Some(&artifact.sha1), &permit)
let bytes = fetch(&artifact.url, Some(&artifact.sha1), &st.io_semaphore)
.await?;
write(&path, &bytes, &permit).await?;
write(&path, &bytes, &st.io_semaphore).await?;
log::info!("Fetched library {}", &library.name);
Ok::<_, crate::Error>(())
}
@@ -235,9 +232,8 @@ pub async fn download_libraries(
&artifact_path
].concat();
let permit = st.io_semaphore.acquire().await?;
let bytes = fetch(&url, None, &permit).await?;
write(&path, &bytes, &permit).await?;
let bytes = fetch(&url, None, &st.io_semaphore).await?;
write(&path, &bytes, &st.io_semaphore).await?;
log::info!("Fetched library {}", &library.name);
Ok::<_, crate::Error>(())
}
@@ -263,8 +259,7 @@ pub async fn download_libraries(
);
if let Some(native) = classifiers.get(&parsed_key) {
let permit = st.io_semaphore.acquire().await?;
let data = fetch(&native.url, Some(&native.sha1), &permit).await?;
let data = fetch(&native.url, Some(&native.sha1), &st.io_semaphore).await?;
let reader = std::io::Cursor::new(&data);
if let Ok(mut archive) = zip::ZipArchive::new(reader) {
match archive.extract(&st.directories.version_natives_dir(version)) {

2
theseus/src/state/dirs.rs
View File

@@ -21,7 +21,7 @@ impl DirectoryInfo {
// Config directory
let config_dir = Self::env_path("THESEUS_CONFIG_DIR")
.or_else(|| Some(dirs::config_dir()?.join("theseus")))
.or_else(|| Some(dirs::config_dir()?.join("com.modrinth.theseus")))
.ok_or(crate::ErrorKind::FSError(
"Could not find valid config dir".to_string(),
))?;

19
theseus/src/state/mod.rs
View File

@@ -2,7 +2,7 @@
use crate::config::sled_config;
use crate::jre;
use std::sync::Arc;
use tokio::sync::{Mutex, OnceCell, RwLock, Semaphore};
use tokio::sync::{OnceCell, RwLock, Semaphore};
// Submodules
mod dirs;
@@ -38,8 +38,6 @@ pub use self::java_globals::*;
// Global state
static LAUNCHER_STATE: OnceCell<Arc<State>> = OnceCell::const_new();
pub struct State {
/// Database, used to store some information
pub(self) database: sled::Db,
/// Information on the location of files used in the launcher
pub directories: DirectoryInfo,
/// Semaphore used to limit concurrent I/O and avoid errors
@@ -88,7 +86,7 @@ impl State {
// Launcher data
let (metadata, profiles) = tokio::try_join! {
Metadata::init(&database),
Profiles::init(&database, &directories, &io_semaphore),
Profiles::init(&directories, &io_semaphore),
}?;
let users = Users::init(&database)?;
@@ -112,7 +110,6 @@ impl State {
}
Ok(Arc::new(Self {
database,
directories,
io_semaphore,
metadata,
@@ -133,7 +130,6 @@ impl State {
/// Synchronize in-memory state with persistent state
pub async fn sync() -> crate::Result<()> {
let state = Self::get().await?;
let batch = Arc::new(Mutex::new(sled::Batch::default()));
let sync_settings = async {
let state = Arc::clone(&state);
@@ -148,13 +144,11 @@ impl State {
let sync_profiles = async {
let state = Arc::clone(&state);
let batch = Arc::clone(&batch);
tokio::spawn(async move {
let profiles = state.profiles.read().await;
let mut batch = batch.lock().await;
profiles.sync(&mut batch).await?;
profiles.sync().await?;
Ok::<_, crate::Error>(())
})
.await?
@@ -162,13 +156,6 @@ impl State {
tokio::try_join!(sync_settings, sync_profiles)?;
state.database.apply_batch(
Arc::try_unwrap(batch)
.expect("Error saving state by acquiring Arc")
.into_inner(),
)?;
state.database.flush_async().await?;
Ok(())
}
}

86
theseus/src/state/profiles.rs
View File

@@ -1,7 +1,7 @@
use super::settings::{Hooks, MemorySettings, WindowSize};
use crate::config::BINCODE_CONFIG;
use crate::data::DirectoryInfo;
use crate::state::projects::Project;
use crate::util::fetch::write_cached_icon;
use daedalus::modded::LoaderVersion;
use dunce::canonicalize;
use futures::prelude::*;
@@ -14,21 +14,15 @@ use tokio::fs;
use tokio::sync::Semaphore;
const PROFILE_JSON_PATH: &str = "profile.json";
const PROFILE_SUBTREE: &[u8] = b"profiles";
pub(crate) struct Profiles(pub HashMap<PathBuf, Profile>);
// TODO: possibly add defaults to some of these values
pub const CURRENT_FORMAT_VERSION: u32 = 1;
pub const SUPPORTED_ICON_FORMATS: &[&str] = &[
"bmp", "gif", "jpeg", "jpg", "jpe", "png", "svg", "svgz", "webp", "rgb",
"mp4",
];
// Represent a Minecraft instance.
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Profile {
#[serde(skip)]
pub path: PathBuf,
pub metadata: ProfileMetadata,
#[serde(skip_serializing_if = "Option::is_none")]
@@ -124,26 +118,15 @@ impl Profile {
#[tracing::instrument]
pub async fn set_icon<'a>(
&'a mut self,
icon: &'a Path,
cache_dir: &Path,
semaphore: &Semaphore,
icon: bytes::Bytes,
file_name: &str,
) -> crate::Result<&'a mut Self> {
let ext = icon
.extension()
.and_then(std::ffi::OsStr::to_str)
.unwrap_or("");
if SUPPORTED_ICON_FORMATS.contains(&ext) {
let file_name = format!("icon.{ext}");
fs::copy(icon, &self.path.join(&file_name)).await?;
self.metadata.icon =
Some(Path::new(&format!("./{file_name}")).to_owned());
Ok(self)
} else {
Err(crate::ErrorKind::InputError(format!(
"Unsupported image type: {ext}"
))
.into())
}
let file =
write_cached_icon(file_name, cache_dir, icon, semaphore).await?;
self.metadata.icon = Some(file);
Ok(self)
}
#[tracing::instrument]
@@ -161,7 +144,10 @@ impl Profile {
let new_path = self.path.join(path);
if new_path.exists() {
for path in std::fs::read_dir(self.path.join(path))? {
files.push(path?.path());
let path = path?.path();
if path.is_file() {
files.push(path);
}
}
}
Ok::<(), crate::Error>(())
@@ -177,26 +163,17 @@ impl Profile {
}
impl Profiles {
#[tracing::instrument(skip(db))]
#[tracing::instrument]
pub async fn init(
db: &sled::Db,
dirs: &DirectoryInfo,
io_sempahore: &Semaphore,
) -> crate::Result<Self> {
let profile_db = db.get(PROFILE_SUBTREE)?.map_or(
Ok(Default::default()),
|bytes| {
bincode::decode_from_slice::<Box<[PathBuf]>, _>(
&bytes,
*BINCODE_CONFIG,
)
.map(|it| it.0)
},
)?;
let mut profiles = HashMap::new();
let mut entries = fs::read_dir(dirs.profiles_dir()).await?;
let mut profiles = stream::iter(profile_db.iter())
.then(|it| async move {
let path = PathBuf::from(it);
while let Some(entry) = entries.next_entry().await? {
let path = entry.path();
if path.is_dir() {
let prof = match Self::read_profile_from_dir(&path).await {
Ok(prof) => Some(prof),
Err(err) => {
@@ -204,13 +181,12 @@ impl Profiles {
None
}
};
(path, prof)
})
.filter_map(|(key, opt_value)| async move {
opt_value.map(|value| (key, value))
})
.collect::<HashMap<PathBuf, Profile>>()
.await;
if let Some(profile) = prof {
let path = canonicalize(path)?;
profiles.insert(path, profile);
}
}
}
// project path, parent profile path
let mut files: HashMap<PathBuf, PathBuf> = HashMap::new();
@@ -278,10 +254,7 @@ impl Profiles {
}
#[tracing::instrument(skip_all)]
pub async fn sync<'a>(
&'a self,
batch: &'a mut sled::Batch,
) -> crate::Result<&Self> {
pub async fn sync(&self) -> crate::Result<&Self> {
stream::iter(self.0.iter())
.map(Ok::<_, crate::Error>)
.try_for_each_concurrent(None, |(path, profile)| async move {
@@ -295,13 +268,6 @@ impl Profiles {
})
.await?;
batch.insert(
PROFILE_SUBTREE,
bincode::encode_to_vec(
self.0.keys().collect::<Box<[_]>>(),
*BINCODE_CONFIG,
)?,
);
Ok(self)
}

113
theseus/src/state/projects.rs
View File

@@ -2,6 +2,7 @@
use crate::config::{MODRINTH_API_URL, REQWEST_CLIENT};
use crate::util::fetch::write_cached_icon;
use async_zip::tokio::read::fs::ZipFileReader;
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use serde_json::json;
@@ -10,14 +11,14 @@ use std::collections::HashMap;
use std::path::{Path, PathBuf};
use tokio::io::AsyncReadExt;
use tokio::sync::Semaphore;
// use zip::ZipArchive;
use async_zip::tokio::read::fs::ZipFileReader;
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Project {
pub sha512: String,
pub disabled: bool,
pub metadata: ProjectMetadata,
pub file_name: String,
pub update_available: bool,
}
#[derive(Serialize, Deserialize, Clone, Debug)]
@@ -50,7 +51,7 @@ pub struct ModrinthProject {
}
/// A specific version of a project
#[derive(Serialize, Deserialize)]
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct ModrinthVersion {
pub id: String,
pub project_id: String,
@@ -73,7 +74,7 @@ pub struct ModrinthVersion {
pub loaders: Vec<String>,
}
#[derive(Serialize, Deserialize)]
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct ModrinthVersionFile {
pub hashes: HashMap<String, String>,
pub url: String,
@@ -83,7 +84,7 @@ pub struct ModrinthVersionFile {
pub file_type: Option<FileType>,
}
#[derive(Serialize, Deserialize, Clone)]
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct Dependency {
pub version_id: Option<String>,
pub project_id: Option<String>,
@@ -91,7 +92,27 @@ pub struct Dependency {
pub dependency_type: DependencyType,
}
#[derive(Serialize, Deserialize, Copy, Clone)]
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct ModrinthTeamMember {
pub team_id: String,
pub user: ModrinthUser,
pub role: String,
pub ordering: i64,
}
#[derive(Serialize, Deserialize, Clone, Debug)]
pub struct ModrinthUser {
pub id: String,
pub github_id: Option<u64>,
pub username: String,
pub name: Option<String>,
pub avatar_url: Option<String>,
pub bio: Option<String>,
pub created: DateTime<Utc>,
pub role: String,
}
#[derive(Serialize, Deserialize, Copy, Clone, Debug)]
#[serde(rename_all = "lowercase")]
pub enum DependencyType {
Required,
@@ -120,7 +141,11 @@ pub enum FileType {
#[derive(Serialize, Deserialize, Clone, Debug)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum ProjectMetadata {
Modrinth(Box<ModrinthProject>),
Modrinth {
project: Box<ModrinthProject>,
version: Box<ModrinthVersion>,
members: Vec<ModrinthTeamMember>,
},
Inferred {
title: Option<String>,
description: Option<String>,
@@ -163,9 +188,11 @@ async fn read_icon_from_file(
.is_ok()
{
let bytes = bytes::Bytes::from(bytes);
let permit = io_semaphore.acquire().await?;
let path = write_cached_icon(
&icon_path, cache_dir, bytes, &permit,
&icon_path,
cache_dir,
bytes,
io_semaphore,
)
.await?;
@@ -196,12 +223,6 @@ pub async fn infer_data_from_files(
file_path_hashes.insert(hash, path.clone());
}
// TODO: add disabled mods
// TODO: add retrying
#[derive(Deserialize)]
pub struct ModrinthVersion {
pub project_id: String,
}
let files: HashMap<String, ModrinthVersion> = REQWEST_CLIENT
.post(format!("{}version_files", MODRINTH_API_URL))
.json(&json!({
@@ -229,22 +250,54 @@ pub async fn infer_data_from_files(
.json()
.await?;
let teams: Vec<ModrinthTeamMember> = REQWEST_CLIENT
.get(format!(
"{}teams?ids={}",
MODRINTH_API_URL,
serde_json::to_string(
&projects.iter().map(|x| x.team.clone()).collect::<Vec<_>>()
)?
))
.send()
.await?
.json::<Vec<Vec<ModrinthTeamMember>>>()
.await?
.into_iter()
.flatten()
.collect();
let mut return_projects = HashMap::new();
let mut further_analyze_projects: Vec<(String, PathBuf)> = Vec::new();
for (hash, path) in file_path_hashes {
if let Some(file) = files.get(&hash) {
if let Some(version) = files.get(&hash) {
if let Some(project) =
projects.iter().find(|x| file.project_id == x.id)
projects.iter().find(|x| version.project_id == x.id)
{
let file_name = path
.file_name()
.unwrap_or_default()
.to_string_lossy()
.to_string();
let team_members = teams
.iter()
.filter(|x| x.team_id == project.team)
.cloned()
.collect::<Vec<_>>();
return_projects.insert(
path,
Project {
sha512: hash,
disabled: false,
metadata: ProjectMetadata::Modrinth(Box::new(
project.clone(),
)),
metadata: ProjectMetadata::Modrinth {
project: Box::new(project.clone()),
version: Box::new(version.clone()),
members: team_members,
},
file_name,
update_available: false,
},
);
continue;
@@ -255,6 +308,12 @@ pub async fn infer_data_from_files(
}
for (hash, path) in further_analyze_projects {
let file_name = path
.file_name()
.unwrap_or_default()
.to_string_lossy()
.to_string();
let zip_file_reader = if let Ok(zip_file_reader) =
ZipFileReader::new(path.clone()).await
{
@@ -266,6 +325,8 @@ pub async fn infer_data_from_files(
sha512: hash,
disabled: path.ends_with(".disabled"),
metadata: ProjectMetadata::Unknown,
file_name,
update_available: false,
},
);
continue;
@@ -318,6 +379,8 @@ pub async fn infer_data_from_files(
Project {
sha512: hash,
disabled: path.ends_with(".disabled"),
file_name,
update_available: false,
metadata: ProjectMetadata::Inferred {
title: Some(
pack.display_name
@@ -383,6 +446,8 @@ pub async fn infer_data_from_files(
Project {
sha512: hash,
disabled: path.ends_with(".disabled"),
file_name,
update_available: false,
metadata: ProjectMetadata::Inferred {
title: Some(if pack.name.is_empty() {
pack.modid
@@ -447,6 +512,8 @@ pub async fn infer_data_from_files(
Project {
sha512: hash,
disabled: path.ends_with(".disabled"),
file_name,
update_available: false,
metadata: ProjectMetadata::Inferred {
title: Some(pack.name.unwrap_or(pack.id)),
description: pack.description,
@@ -511,6 +578,8 @@ pub async fn infer_data_from_files(
Project {
sha512: hash,
disabled: path.ends_with(".disabled"),
file_name,
update_available: false,
metadata: ProjectMetadata::Inferred {
title: Some(
pack.metadata
@@ -575,6 +644,8 @@ pub async fn infer_data_from_files(
Project {
sha512: hash,
disabled: path.ends_with(".disabled"),
file_name,
update_available: false,
metadata: ProjectMetadata::Inferred {
title: None,
description: pack.description,
@@ -594,6 +665,8 @@ pub async fn infer_data_from_files(
Project {
sha512: hash,
disabled: path.ends_with(".disabled"),
file_name,
update_available: false,
metadata: ProjectMetadata::Unknown,
},
);

70
theseus/src/util/fetch.rs
View File

@@ -1,25 +1,51 @@
//! Functions for fetching infromation from the Internet
use crate::config::REQWEST_CLIENT;
use bytes::Bytes;
use reqwest::Method;
use serde::de::DeserializeOwned;
use std::ffi::OsStr;
use std::path::{Path, PathBuf};
use tokio::sync::Semaphore;
use tokio::{
fs::{self, File},
io::AsyncWriteExt,
sync::SemaphorePermit,
};
const FETCH_ATTEMPTS: usize = 3;
/// Downloads a file with retry and checksum functionality
#[tracing::instrument(skip(_permit))]
pub async fn fetch<'a>(
pub async fn fetch(
url: &str,
sha1: Option<&str>,
_permit: &SemaphorePermit<'a>,
) -> crate::Result<bytes::Bytes> {
semaphore: &Semaphore,
) -> crate::Result<Bytes> {
fetch_advanced(Method::GET, url, sha1, semaphore).await
}
pub async fn fetch_json<T>(
method: Method,
url: &str,
sha1: Option<&str>,
semaphore: &Semaphore,
) -> crate::Result<T>
where
T: DeserializeOwned,
{
let result = fetch_advanced(method, url, sha1, semaphore).await?;
let value = serde_json::from_slice(&result)?;
Ok(value)
}
/// Downloads a file with retry and checksum functionality
#[tracing::instrument(skip(semaphore))]
pub async fn fetch_advanced(
method: Method,
url: &str,
sha1: Option<&str>,
semaphore: &Semaphore,
) -> crate::Result<Bytes> {
let _permit = semaphore.acquire().await?;
for attempt in 1..=(FETCH_ATTEMPTS + 1) {
let result = REQWEST_CLIENT.get(url).send().await;
let result = REQWEST_CLIENT.request(method.clone(), url).send().await;
match result {
Ok(x) => {
@@ -50,7 +76,9 @@ pub async fn fetch<'a>(
}
}
Err(_) if attempt <= 3 => continue,
Err(err) => return Err(err.into()),
Err(err) => {
return Err(err.into());
}
}
}
@@ -58,12 +86,12 @@ pub async fn fetch<'a>(
}
/// Downloads a file from specified mirrors
#[tracing::instrument(skip(permit))]
pub async fn fetch_mirrors<'a>(
#[tracing::instrument(skip(semaphore))]
pub async fn fetch_mirrors(
mirrors: &[&str],
sha1: Option<&str>,
permit: &SemaphorePermit<'a>,
) -> crate::Result<bytes::Bytes> {
semaphore: &Semaphore,
) -> crate::Result<Bytes> {
if mirrors.is_empty() {
return Err(crate::ErrorKind::InputError(
"No mirrors provided!".to_string(),
@@ -72,7 +100,7 @@ pub async fn fetch_mirrors<'a>(
}
for (index, mirror) in mirrors.iter().enumerate() {
let result = fetch(mirror, sha1, permit).await;
let result = fetch(mirror, sha1, semaphore).await;
if result.is_ok() || (result.is_err() && index == (mirrors.len() - 1)) {
return result;
@@ -82,28 +110,29 @@ pub async fn fetch_mirrors<'a>(
unreachable!()
}
#[tracing::instrument(skip(bytes, _permit))]
#[tracing::instrument(skip(bytes, semaphore))]
pub async fn write<'a>(
path: &Path,
bytes: &[u8],
_permit: &SemaphorePermit<'a>,
semaphore: &Semaphore,
) -> crate::Result<()> {
let _permit = semaphore.acquire().await?;
if let Some(parent) = path.parent() {
fs::create_dir_all(parent).await?;
}
let mut file = File::create(path).await?;
log::debug!("Done writing file {}", path.display());
file.write_all(bytes).await?;
log::debug!("Done writing file {}", path.display());
Ok(())
}
#[tracing::instrument(skip(bytes, permit))]
pub async fn write_cached_icon<'a>(
#[tracing::instrument(skip(bytes, semaphore))]
pub async fn write_cached_icon(
icon_path: &str,
cache_dir: &Path,
bytes: Bytes,
permit: &SemaphorePermit<'a>,
semaphore: &Semaphore,
) -> crate::Result<PathBuf> {
let extension = Path::new(&icon_path).extension().and_then(OsStr::to_str);
let hash = sha1_async(bytes.clone()).await?;
@@ -113,8 +142,9 @@ pub async fn write_cached_icon<'a>(
hash
});
write(&path, &bytes, permit).await?;
write(&path, &bytes, semaphore).await?;
let path = dunce::canonicalize(path)?;
Ok(path)
}