Misc improvements and fixes (#109)

* now utilizing tracing better * better tracing * fix mac vs pc oppositional env var issue * modified loading package * added droppable loadingbarid that sends completion message * loading bar * regressed bug on mac * fixed non-updated loading bar on playground * Loading bar improvements --------- Co-authored-by: Jai A <jaiagr+gpg@pm.me>
2023-05-08 12:14:08 -07:00
parent c79d5c32a6
commit 65c1942037
33 changed files with 726 additions and 294 deletions
--- a/theseus/src/state/children.rs
+++ b/theseus/src/state/children.rs
@@ -8,6 +8,7 @@ use tokio::process::Child;
 use tokio::process::Command;
 use tokio::process::{ChildStderr, ChildStdout};
 use tokio::sync::RwLock;
+use tracing::error;

 use crate::event::emit::emit_process;
 use crate::event::ProcessPayloadType;
@@ -55,7 +56,7 @@ impl Children {
            let stdout_clone = stdout.clone();
            tokio::spawn(async move {
                if let Err(e) = stdout_clone.read_stdout(child_stdout).await {
-                    eprintln!("Stdout process died with error: {}", e);
+                    error!("Stdout process died with error: {}", e);
                }
            });
        }
@@ -64,7 +65,7 @@ impl Children {
            let stderr_clone = stderr.clone();
            tokio::spawn(async move {
                if let Err(e) = stderr_clone.read_stderr(child_stderr).await {
-                    eprintln!("Stderr process died with error: {}", e);
+                    error!("Stderr process died with error: {}", e);
                }
            });
        }
--- a/theseus/src/state/metadata.rs
+++ b/theseus/src/state/metadata.rs
@@ -1,6 +1,6 @@
 //! Theseus metadata
 use crate::data::DirectoryInfo;
-use crate::util::fetch::{read_json, write};
+use crate::util::fetch::{read_json, write, IoSemaphore};
 use crate::State;
 use daedalus::{
    minecraft::{fetch_version_manifest, VersionManifest as MinecraftManifest},
@@ -9,7 +9,6 @@ use daedalus::{
    },
 };
 use serde::{Deserialize, Serialize};
-use tokio::sync::{RwLock, Semaphore};

 const METADATA_URL: &str = "https://meta.modrinth.com";

@@ -51,7 +50,7 @@ impl Metadata {
    // Attempt to fetch metadata and store in sled DB
    pub async fn init(
        dirs: &DirectoryInfo,
-        io_semaphore: &RwLock<Semaphore>,
+        io_semaphore: &IoSemaphore,
    ) -> crate::Result<Self> {
        let mut metadata = None;
        let metadata_path = dirs.caches_meta_dir().join("metadata.json");
@@ -79,7 +78,7 @@ impl Metadata {
            match res {
                Ok(()) => {}
                Err(err) => {
-                    log::warn!("Unable to fetch launcher metadata: {err}")
+                    tracing::warn!("Unable to fetch launcher metadata: {err}")
                }
            }
        }
@@ -120,7 +119,7 @@ impl Metadata {
        match res {
            Ok(()) => {}
            Err(err) => {
-                log::warn!("Unable to update launcher metadata: {err}")
+                tracing::warn!("Unable to update launcher metadata: {err}")
            }
        };
    }
--- a/theseus/src/state/mod.rs
+++ b/theseus/src/state/mod.rs
@@ -6,6 +6,7 @@ use crate::event::LoadingBarType;
 use crate::loading_join;

 use crate::state::users::Users;
+use crate::util::fetch::{FetchSemaphore, IoSemaphore};
 use std::sync::Arc;
 use tokio::sync::{OnceCell, RwLock, Semaphore};

@@ -44,10 +45,16 @@ static LAUNCHER_STATE: OnceCell<Arc<State>> = OnceCell::const_new();
 pub struct State {
    /// Information on the location of files used in the launcher
    pub directories: DirectoryInfo,
+
+    /// Semaphore used to limit concurrent network requests and avoid errors
+    pub fetch_semaphore: FetchSemaphore,
+    /// Stored maximum number of sempahores of current fetch_semaphore
+    pub fetch_semaphore_max: RwLock<u32>,
    /// Semaphore used to limit concurrent I/O and avoid errors
-    pub io_semaphore: RwLock<Semaphore>,
+    pub io_semaphore: IoSemaphore,
    /// Stored maximum number of sempahores of current io_semaphore
    pub io_semaphore_max: RwLock<u32>,
+
    /// Launcher metadata
    pub metadata: RwLock<Metadata>,
    /// Launcher configuration
@@ -73,7 +80,7 @@ impl State {
                    let loading_bar = init_loading(
                        LoadingBarType::StateInit,
                        100.0,
-                        "Initializing launcher...",
+                        "Initializing launcher",
                    )
                    .await?;

@@ -83,20 +90,26 @@ impl State {
                    // Settings
                    let settings =
                        Settings::init(&directories.settings_file()).await?;
-                    let io_semaphore = RwLock::new(Semaphore::new(
-                        settings.max_concurrent_downloads,
+                    let fetch_semaphore = FetchSemaphore(RwLock::new(
+                        Semaphore::new(settings.max_concurrent_downloads),
+                    ));
+                    let io_semaphore = IoSemaphore(RwLock::new(
+                        Semaphore::new(settings.max_concurrent_writes),
                    ));
                    emit_loading(&loading_bar, 10.0, None).await?;

                    let metadata_fut =
                        Metadata::init(&directories, &io_semaphore);
-                    let profiles_fut =
-                        Profiles::init(&directories, &io_semaphore);
-                    let tags_fut = Tags::init(&directories, &io_semaphore);
+                    let profiles_fut = Profiles::init(&directories);
+                    let tags_fut = Tags::init(
+                        &directories,
+                        &io_semaphore,
+                        &fetch_semaphore,
+                    );
                    let users_fut = Users::init(&directories, &io_semaphore);
                    // Launcher data
                    let (metadata, profiles, tags, users) = loading_join! {
-                        Some(&loading_bar), 70.0, Some("Initializing...");
+                        Some(&loading_bar), 70.0, Some("Loading metadata");
                        metadata_fut,
                        profiles_fut,
                        tags_fut,
@@ -109,9 +122,13 @@ impl State {

                    Ok(Arc::new(Self {
                        directories,
+                        fetch_semaphore,
+                        fetch_semaphore_max: RwLock::new(
+                            settings.max_concurrent_downloads as u32,
+                        ),
                        io_semaphore,
                        io_semaphore_max: RwLock::new(
-                            settings.max_concurrent_downloads as u32,
+                            settings.max_concurrent_writes as u32,
                        ),
                        metadata: RwLock::new(metadata),
                        settings: RwLock::new(settings),
@@ -169,17 +186,34 @@ impl State {
        .await
    }

-    /// Reset semaphores to default values
+    /// Reset IO semaphore to default values
    /// This will block until all uses of the semaphore are complete, so it should only be called
    /// when we are not in the middle of downloading something (ie: changing the settings!)
-    pub async fn reset_semaphore(&self) {
+    pub async fn reset_io_semaphore(&self) {
        let settings = self.settings.read().await;
-        let mut io_semaphore = self.io_semaphore.write().await;
+        let mut io_semaphore = self.io_semaphore.0.write().await;
        let mut total_permits = self.io_semaphore_max.write().await;

        // Wait to get all permits back
        let _ = io_semaphore.acquire_many(*total_permits).await;

+        // Reset the semaphore
+        io_semaphore.close();
+        *total_permits = settings.max_concurrent_writes as u32;
+        *io_semaphore = Semaphore::new(settings.max_concurrent_writes);
+    }
+
+    /// Reset IO semaphore to default values
+    /// This will block until all uses of the semaphore are complete, so it should only be called
+    /// when we are not in the middle of downloading something (ie: changing the settings!)
+    pub async fn reset_fetch_semaphore(&self) {
+        let settings = self.settings.read().await;
+        let mut io_semaphore = self.fetch_semaphore.0.write().await;
+        let mut total_permits = self.fetch_semaphore_max.write().await;
+
+        // Wait to get all permits back
+        let _ = io_semaphore.acquire_many(*total_permits).await;
+
        // Reset the semaphore
        io_semaphore.close();
        *total_permits = settings.max_concurrent_downloads as u32;
--- a/theseus/src/state/profiles.rs
+++ b/theseus/src/state/profiles.rs
@@ -5,7 +5,9 @@ use crate::event::emit::emit_profile;
 use crate::event::ProfilePayloadType;
 use crate::state::projects::Project;
 use crate::state::{ModrinthVersion, ProjectType};
-use crate::util::fetch::{fetch, fetch_json, write, write_cached_icon};
+use crate::util::fetch::{
+    fetch, fetch_json, write, write_cached_icon, IoSemaphore,
+};
 use crate::State;
 use daedalus::modded::LoaderVersion;
 use dunce::canonicalize;
@@ -17,8 +19,7 @@ use std::{
    collections::HashMap,
    path::{Path, PathBuf},
 };
-use tokio::sync::Semaphore;
-use tokio::{fs, sync::RwLock};
+use tokio::fs;
 use uuid::Uuid;

 const PROFILE_JSON_PATH: &str = "profile.json";
@@ -149,7 +150,7 @@ impl Profile {
    pub async fn set_icon<'a>(
        &'a mut self,
        cache_dir: &Path,
-        semaphore: &RwLock<Semaphore>,
+        semaphore: &IoSemaphore,
        icon: bytes::Bytes,
        file_name: &str,
    ) -> crate::Result<&'a mut Self> {
@@ -168,6 +169,7 @@ impl Profile {
            paths,
            state.directories.caches_dir(),
            &state.io_semaphore,
+            &state.fetch_semaphore,
        )
        .await?;

@@ -218,7 +220,7 @@ impl Profile {
            &format!("{MODRINTH_API_URL}version/{version_id}"),
            None,
            None,
-            &state.io_semaphore,
+            &state.fetch_semaphore,
        )
        .await?;

@@ -237,7 +239,7 @@ impl Profile {
        let bytes = fetch(
            &file.url,
            file.hashes.get("sha1").map(|x| &**x),
-            &state.io_semaphore,
+            &state.fetch_semaphore,
        )
        .await?;

@@ -345,10 +347,7 @@ impl Profile {

 impl Profiles {
    #[tracing::instrument]
-    pub async fn init(
-        dirs: &DirectoryInfo,
-        io_sempahore: &RwLock<Semaphore>,
-    ) -> crate::Result<Self> {
+    pub async fn init(dirs: &DirectoryInfo) -> crate::Result<Self> {
        let mut profiles = HashMap::new();
        fs::create_dir_all(dirs.profiles_dir()).await?;
        let mut entries = fs::read_dir(dirs.profiles_dir()).await?;
@@ -358,7 +357,9 @@ impl Profiles {
                let prof = match Self::read_profile_from_dir(&path).await {
                    Ok(prof) => Some(prof),
                    Err(err) => {
-                        log::warn!("Error loading profile: {err}. Skipping...");
+                        tracing::warn!(
+                            "Error loading profile: {err}. Skipping..."
+                        );
                        None
                    }
                };
@@ -395,6 +396,7 @@ impl Profiles {
                        files,
                        state.directories.caches_dir(),
                        &state.io_semaphore,
+                        &state.fetch_semaphore,
                    )
                    .await?;

@@ -417,7 +419,7 @@ impl Profiles {
        match res {
            Ok(()) => {}
            Err(err) => {
-                log::warn!("Unable to fetch profile projects: {err}")
+                tracing::warn!("Unable to fetch profile projects: {err}")
            }
        };
    }
--- a/theseus/src/state/projects.rs
+++ b/theseus/src/state/projects.rs
@@ -2,7 +2,9 @@

 use crate::config::MODRINTH_API_URL;
 use crate::state::Profile;
-use crate::util::fetch::{fetch_json, write_cached_icon};
+use crate::util::fetch::{
+    fetch_json, write_cached_icon, FetchSemaphore, IoSemaphore,
+};
 use async_zip::tokio::read::fs::ZipFileReader;
 use chrono::{DateTime, Utc};
 use reqwest::Method;
@@ -12,7 +14,6 @@ use sha2::Digest;
 use std::collections::HashMap;
 use std::path::{Path, PathBuf};
 use tokio::io::AsyncReadExt;
-use tokio::sync::{RwLock, Semaphore};

 #[derive(Serialize, Deserialize, Clone, Debug)]
 #[serde(rename_all = "lowercase")]
@@ -203,7 +204,7 @@ async fn read_icon_from_file(
    icon_path: Option<String>,
    cache_dir: &Path,
    path: &PathBuf,
-    io_semaphore: &RwLock<Semaphore>,
+    io_semaphore: &IoSemaphore,
 ) -> crate::Result<Option<PathBuf>> {
    if let Some(icon_path) = icon_path {
        // we have to repoen the zip twice here :(
@@ -252,7 +253,8 @@ pub async fn infer_data_from_files(
    profile: Profile,
    paths: Vec<PathBuf>,
    cache_dir: PathBuf,
-    io_semaphore: &RwLock<Semaphore>,
+    io_semaphore: &IoSemaphore,
+    fetch_semaphore: &FetchSemaphore,
 ) -> crate::Result<HashMap<PathBuf, Project>> {
    let mut file_path_hashes = HashMap::new();

@@ -278,7 +280,7 @@ pub async fn infer_data_from_files(
                "hashes": file_path_hashes.keys().collect::<Vec<_>>(),
                "algorithm": "sha512",
            })),
-            io_semaphore,
+            fetch_semaphore,
        ),
        fetch_json::<HashMap<String, ModrinthVersion>>(
            Method::POST,
@@ -290,7 +292,7 @@ pub async fn infer_data_from_files(
                "loaders": [profile.metadata.loader],
                "game_versions": [profile.metadata.game_version]
            })),
-            io_semaphore,
+            fetch_semaphore,
        )
    )?;

@@ -308,7 +310,7 @@ pub async fn infer_data_from_files(
        ),
        None,
        None,
-        io_semaphore,
+        fetch_semaphore,
    )
    .await?;

@@ -325,7 +327,7 @@ pub async fn infer_data_from_files(
        ),
        None,
        None,
-        io_semaphore,
+        fetch_semaphore,
    )
    .await?
    .into_iter()
--- a/theseus/src/state/settings.rs
+++ b/theseus/src/state/settings.rs
@@ -23,6 +23,7 @@ pub struct Settings {
    pub default_user: Option<uuid::Uuid>,
    pub hooks: Hooks,
    pub max_concurrent_downloads: usize,
+    pub max_concurrent_writes: usize,
    pub version: u32,
    pub collapsed_navigation: bool,
 }
@@ -39,6 +40,7 @@ impl Default for Settings {
            default_user: None,
            hooks: Hooks::default(),
            max_concurrent_downloads: 64,
+            max_concurrent_writes: 100,
            version: CURRENT_FORMAT_VERSION,
            collapsed_navigation: false,
        }
@@ -84,7 +86,7 @@ impl Settings {
        match res {
            Ok(()) => {}
            Err(err) => {
-                log::warn!("Unable to update launcher java: {err}")
+                tracing::warn!("Unable to update launcher java: {err}")
            }
        };
    }
--- a/theseus/src/state/tags.rs
+++ b/theseus/src/state/tags.rs
@@ -2,11 +2,12 @@ use std::path::PathBuf;

 use reqwest::Method;
 use serde::{Deserialize, Serialize};
-use tokio::sync::{RwLock, Semaphore};

 use crate::config::MODRINTH_API_URL;
 use crate::data::DirectoryInfo;
-use crate::util::fetch::{fetch_json, read_json, write};
+use crate::util::fetch::{
+    fetch_json, read_json, write, FetchSemaphore, IoSemaphore,
+};

 // Serializeable struct for all tags to be fetched together by the frontend
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -21,7 +22,8 @@ pub struct Tags {
 impl Tags {
    pub async fn init(
        dirs: &DirectoryInfo,
-        io_semaphore: &RwLock<Semaphore>,
+        io_semaphore: &IoSemaphore,
+        fetch_sempahore: &FetchSemaphore,
    ) -> crate::Result<Self> {
        let mut tags = None;
        let tags_path = dirs.caches_meta_dir().join("tags.json");
@@ -30,10 +32,10 @@ impl Tags {
        {
            tags = Some(tags_json);
        } else {
-            match Self::fetch(io_semaphore).await {
+            match Self::fetch(fetch_sempahore).await {
                Ok(tags_fetch) => tags = Some(tags_fetch),
                Err(err) => {
-                    log::warn!("Unable to fetch launcher tags: {err}")
+                    tracing::warn!("Unable to fetch launcher tags: {err}")
                }
            }
        }
@@ -51,7 +53,7 @@ impl Tags {
    pub async fn update() {
        let res = async {
            let state = crate::State::get().await?;
-            let tags_fetch = Tags::fetch(&state.io_semaphore).await?;
+            let tags_fetch = Tags::fetch(&state.fetch_semaphore).await?;

            let tags_path =
                state.directories.caches_meta_dir().join("tags.json");
@@ -74,7 +76,7 @@ impl Tags {
        match res {
            Ok(()) => {}
            Err(err) => {
-                log::warn!("Unable to update launcher tags: {err}")
+                tracing::warn!("Unable to update launcher tags: {err}")
            }
        };
    }
@@ -116,7 +118,7 @@ impl Tags {
    }

    // Fetches the tags from the Modrinth API and stores them in the database
-    pub async fn fetch(semaphore: &RwLock<Semaphore>) -> crate::Result<Self> {
+    pub async fn fetch(semaphore: &FetchSemaphore) -> crate::Result<Self> {
        let categories = format!("{MODRINTH_API_URL}tag/category");
        let loaders = format!("{MODRINTH_API_URL}tag/loader");
        let game_versions = format!("{MODRINTH_API_URL}tag/game_version");
--- a/theseus/src/state/users.rs
+++ b/theseus/src/state/users.rs
@@ -1,10 +1,9 @@
 //! User login info
 use crate::auth::Credentials;
 use crate::data::DirectoryInfo;
-use crate::util::fetch::{read_json, write};
+use crate::util::fetch::{read_json, write, IoSemaphore};
 use crate::State;
 use std::collections::HashMap;
-use tokio::sync::{RwLock, Semaphore};
 use uuid::Uuid;

 const USERS_JSON: &str = "users.json";
@@ -16,7 +15,7 @@ pub(crate) struct Users(pub(crate) HashMap<Uuid, Credentials>);
 impl Users {
    pub async fn init(
        dirs: &DirectoryInfo,
-        io_semaphore: &RwLock<Semaphore>,
+        io_semaphore: &IoSemaphore,
    ) -> crate::Result<Self> {
        let users_path = dirs.caches_meta_dir().join(USERS_JSON);
        let users = read_json(&users_path, io_semaphore).await.ok();