fix(app): make instances with non-UTF8 text file encodings launcheable and importable (#3721)

Previous to these changes, the app always assumed that Minecraft and
other launchers always use UTF-8, which is not necessarily always true.

Cargo.lock

@@ -1379,6 +1379,17 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
 
+[[package]]
+name = "chardetng"
+version = "0.1.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "14b8f0b65b7b08ae3c8187e8d77174de20cb6777864c6b832d8ad365999cf1ea"
+dependencies = [
+ "cfg-if",
+ "encoding_rs",
+ "memchr",
+]
+
 [[package]]
 name = "chrono"
 version = "0.4.41"
@@ -8865,6 +8876,7 @@ dependencies = [
  "async_zip",
  "base64 0.22.1",
  "bytes",
+ "chardetng",
  "chrono",
  "daedalus",
  "dashmap",
@@ -8872,6 +8884,7 @@ dependencies = [
  "discord-rich-presence",
  "dunce",
  "either",
+ "encoding_rs",
  "enumset",
  "flate2",
  "fs4",

Cargo.toml

@@ -36,6 +36,7 @@ base64 = "0.22.1"
 bitflags = "2.9.1"
 bytes = "1.10.1"
 censor = "0.3.0"
+chardetng = "0.1.17"
 chrono = "0.4.41"
 clap = "4.5.40"
 clickhouse = "0.13.3"
@@ -50,6 +51,7 @@ dotenv-build = "0.1.1"
 dotenvy = "0.15.7"
 dunce = "1.0.5"
 either = "1.15.0"
+encoding_rs = "0.8.35"
 enumset = "1.1.6"
 flate2 = "1.1.2"
 fs4 = { version = "0.13.1", default-features = false }

packages/app-lib/Cargo.toml

@@ -20,6 +20,8 @@ tempfile.workspace = true
 dashmap = { workspace = true, features = ["serde"] }
 quick-xml = { workspace = true, features = ["async-tokio"] }
 enumset.workspace = true
+chardetng.workspace = true
+encoding_rs.workspace = true
 
 chrono = { workspace = true, features = ["serde"] }
 daedalus.workspace = true

packages/app-lib/src/api/pack/import/atlauncher.rs

@@ -97,12 +97,15 @@ pub struct ATLauncherMod {
 
 // Check if folder has a instance.json that parses
 pub async fn is_valid_atlauncher(instance_folder: PathBuf) -> bool {
-    let instance: String =
+    let instance = serde_json::from_str::<ATInstance>(
-        io::read_to_string(&instance_folder.join("instance.json"))
+        &io::read_any_encoding_to_string(
-            .await
+            &instance_folder.join("instance.json"),
-            .unwrap_or("".to_string());
+        )
-    let instance: Result<ATInstance, serde_json::Error> =
+        .await
-        serde_json::from_str::<ATInstance>(&instance);
+        .unwrap_or(("".into(), encoding_rs::UTF_8))
+        .0,
+    );
+
     if let Err(e) = instance {
         tracing::warn!(
             "Could not parse instance.json at {}: {}",
@@ -124,14 +127,17 @@ pub async fn import_atlauncher(
 ) -> crate::Result<()> {
     let atlauncher_instance_path = atlauncher_base_path
         .join("instances")
-        .join(instance_folder.clone());
+        .join(&instance_folder);
 
     // Load instance.json
-    let atinstance: String =
+    let atinstance = serde_json::from_str::<ATInstance>(
-        io::read_to_string(&atlauncher_instance_path.join("instance.json"))
+        &io::read_any_encoding_to_string(
-            .await?;
+            &atlauncher_instance_path.join("instance.json"),
-    let atinstance: ATInstance =
+        )
-        serde_json::from_str::<ATInstance>(&atinstance)?;
+        .await
+        .unwrap_or(("".into(), encoding_rs::UTF_8))
+        .0,
+    )?;
 
     // Icon path should be {instance_folder}/instance.png if it exists,
     // Second possibility is ATLauncher/configs/images/{safe_pack_name}.png (safe pack name is alphanumeric lowercase)

packages/app-lib/src/api/pack/import/curseforge.rs

@@ -36,13 +36,15 @@ pub struct InstalledModpack {
 
 // Check if folder has a minecraftinstance.json that parses
 pub async fn is_valid_curseforge(instance_folder: PathBuf) -> bool {
-    let minecraftinstance: String =
+    let minecraft_instance = serde_json::from_str::<MinecraftInstance>(
-        io::read_to_string(&instance_folder.join("minecraftinstance.json"))
+        &io::read_any_encoding_to_string(
-            .await
+            &instance_folder.join("minecraftinstance.json"),
-            .unwrap_or("".to_string());
+        )
-    let minecraftinstance: Result<MinecraftInstance, serde_json::Error> =
+        .await
-        serde_json::from_str::<MinecraftInstance>(&minecraftinstance);
+        .unwrap_or(("".into(), encoding_rs::UTF_8))
-    minecraftinstance.is_ok()
+        .0,
+    );
+    minecraft_instance.is_ok()
 }
 
 pub async fn import_curseforge(
@@ -50,13 +52,15 @@ pub async fn import_curseforge(
     profile_path: &str,                  // path to profile
 ) -> crate::Result<()> {
     // Load minecraftinstance.json
-    let minecraft_instance: String = io::read_to_string(
+    let minecraft_instance = serde_json::from_str::<MinecraftInstance>(
-        &curseforge_instance_folder.join("minecraftinstance.json"),
+        &io::read_any_encoding_to_string(
-    )
+            &curseforge_instance_folder.join("minecraftinstance.json"),
-    .await?;
+        )
-    let minecraft_instance: MinecraftInstance =
+        .await
-        serde_json::from_str::<MinecraftInstance>(&minecraft_instance)?;
+        .unwrap_or(("".into(), encoding_rs::UTF_8))
-    let override_title: Option<String> = minecraft_instance.name.clone();
+        .0,
+    )?;
+    let override_title = minecraft_instance.name;
     let backup_name = format!(
         "Curseforge-{}",
         curseforge_instance_folder

packages/app-lib/src/api/pack/import/gdlauncher.rs

@@ -25,12 +25,12 @@ pub struct GDLauncherLoader {
 
 // Check if folder has a config.json that parses
 pub async fn is_valid_gdlauncher(instance_folder: PathBuf) -> bool {
-    let config: String =
+    let config = serde_json::from_str::<GDLauncherConfig>(
-        io::read_to_string(&instance_folder.join("config.json"))
+        &io::read_any_encoding_to_string(&instance_folder.join("config.json"))
             .await
-            .unwrap_or("".to_string());
+            .unwrap_or(("".into(), encoding_rs::UTF_8))
-    let config: Result<GDLauncherConfig, serde_json::Error> =
+            .0,
-        serde_json::from_str::<GDLauncherConfig>(&config);
+    );
     config.is_ok()
 }
 
@@ -39,12 +39,15 @@ pub async fn import_gdlauncher(
     profile_path: &str,                  // path to profile
 ) -> crate::Result<()> {
     // Load config.json
-    let config: String =
+    let config = serde_json::from_str::<GDLauncherConfig>(
-        io::read_to_string(&gdlauncher_instance_folder.join("config.json"))
+        &io::read_any_encoding_to_string(
-            .await?;
+            &gdlauncher_instance_folder.join("config.json"),
-    let config: GDLauncherConfig =
+        )
-        serde_json::from_str::<GDLauncherConfig>(&config)?;
+        .await
-    let override_title: Option<String> = config.loader.source_name.clone();
+        .unwrap_or(("".into(), encoding_rs::UTF_8))
+        .0,
+    )?;
+    let override_title = config.loader.source_name;
     let backup_name = format!(
         "GDLauncher-{}",
         gdlauncher_instance_folder

packages/app-lib/src/api/pack/import/mmc.rs

@@ -144,8 +144,8 @@ pub async fn is_valid_mmc(instance_folder: PathBuf) -> bool {
     let instance_cfg = instance_folder.join("instance.cfg");
     let mmc_pack = instance_folder.join("mmc-pack.json");
 
-    let mmc_pack = match io::read_to_string(&mmc_pack).await {
+    let mmc_pack = match io::read_any_encoding_to_string(&mmc_pack).await {
-        Ok(mmc_pack) => mmc_pack,
+        Ok((mmc_pack, _)) => mmc_pack,
         Err(_) => return false,
     };
 
@@ -155,7 +155,7 @@ pub async fn is_valid_mmc(instance_folder: PathBuf) -> bool {
 
 #[tracing::instrument]
 pub async fn get_instances_subpath(config: PathBuf) -> Option<String> {
-    let launcher = io::read_to_string(&config).await.ok()?;
+    let launcher = io::read_any_encoding_to_string(&config).await.ok()?.0;
     let launcher: MMCLauncherEnum = serde_ini::from_str(&launcher).ok()?;
     match launcher {
         MMCLauncherEnum::General(p) => Some(p.general.instance_dir),
@@ -165,10 +165,9 @@ pub async fn get_instances_subpath(config: PathBuf) -> Option<String> {
 
 // Loading the INI (instance.cfg) file
 async fn load_instance_cfg(file_path: &Path) -> crate::Result<MMCInstance> {
-    let instance_cfg: String = io::read_to_string(file_path).await?;
+    match serde_ini::from_str::<MMCInstanceEnum>(
-    let instance_cfg_enum: MMCInstanceEnum =
+        &io::read_any_encoding_to_string(file_path).await?.0,
-        serde_ini::from_str::<MMCInstanceEnum>(&instance_cfg)?;
+    )? {
-    match instance_cfg_enum {
         MMCInstanceEnum::General(instance_cfg) => Ok(instance_cfg.general),
         MMCInstanceEnum::Instance(instance_cfg) => Ok(instance_cfg),
     }
@@ -183,9 +182,13 @@ pub async fn import_mmc(
     let mmc_instance_path =
         mmc_base_path.join("instances").join(instance_folder);
 
-    let mmc_pack =
+    let mmc_pack = serde_json::from_str::<MMCPack>(
-        io::read_to_string(&mmc_instance_path.join("mmc-pack.json")).await?;
+        &io::read_any_encoding_to_string(
-    let mmc_pack: MMCPack = serde_json::from_str::<MMCPack>(&mmc_pack)?;
+            &mmc_instance_path.join("mmc-pack.json"),
+        )
+        .await?
+        .0,
+    )?;
 
     let instance_cfg =
         load_instance_cfg(&mmc_instance_path.join("instance.cfg")).await?;
@@ -243,7 +246,7 @@ pub async fn import_mmc(
             _ => return Err(crate::ErrorKind::InputError("Instance is managed, but managed pack type not specified in instance.cfg".to_string()).into())
         }
     } else {
-        // Direclty import unmanaged pack
+        // Directly import unmanaged pack
         import_mmc_unmanaged(
             profile_path,
             minecraft_folder,

packages/app-lib/src/launcher/mod.rs

@@ -14,6 +14,7 @@ use chrono::Utc;
 use daedalus as d;
 use daedalus::minecraft::{LoggingSide, RuleAction, VersionInfo};
 use daedalus::modded::LoaderVersion;
+use regex::Regex;
 use serde::Deserialize;
 use st::Profile;
 use std::collections::HashMap;
@@ -662,14 +663,29 @@ pub async fn launch_minecraft(
 
     // Overwrites the minecraft options.txt file with the settings from the profile
     // Uses 'a:b' syntax which is not quite yaml
-    use regex::Regex;
-
     if !mc_set_options.is_empty() {
         let options_path = instance_path.join("options.txt");
-        let mut options_string = String::new();
+
-        if options_path.exists() {
+        let (mut options_string, input_encoding) = if options_path.exists() {
-            options_string = io::read_to_string(&options_path).await?;
+            io::read_any_encoding_to_string(&options_path).await?
+        } else {
+            (String::new(), encoding_rs::UTF_8)
+        };
+
+        // UTF-16 encodings may be successfully detected and read, but we cannot encode
+        // them back, and it's technically possible that the game client strongly expects
+        // such encoding
+        if input_encoding != input_encoding.output_encoding() {
+            return Err(crate::ErrorKind::LauncherError(format!(
+                "The instance options.txt file uses an unsupported encoding: {}. \
+                Please either turn off instance options that need to modify this file, \
+                or convert the file to an encoding that both the game and this app support, \
+                such as UTF-8.",
+                input_encoding.name()
+            ))
+            .into());
         }
+
         for (key, value) in mc_set_options {
             let re = Regex::new(&format!(r"(?m)^{}:.*$", regex::escape(key)))?;
             // check if the regex exists in the file
@@ -684,7 +700,8 @@ pub async fn launch_minecraft(
             }
         }
 
-        io::write(&options_path, options_string).await?;
+        io::write(&options_path, input_encoding.encode(&options_string).0)
+            .await?;
     }
 
     crate::api::profile::edit(&profile.path, |prof| {

packages/app-lib/src/util/io.rs

@@ -35,7 +35,6 @@ impl IOError {
     }
 }
 
-// dunce canonicalize
 pub fn canonicalize(
     path: impl AsRef<std::path::Path>,
 ) -> Result<std::path::PathBuf, IOError> {
@@ -46,7 +45,6 @@ pub fn canonicalize(
     })
 }
 
-// read_dir
 pub async fn read_dir(
     path: impl AsRef<std::path::Path>,
 ) -> Result<tokio::fs::ReadDir, IOError> {
@@ -59,7 +57,6 @@ pub async fn read_dir(
         })
 }
 
-// create_dir
 pub async fn create_dir(
     path: impl AsRef<std::path::Path>,
 ) -> Result<(), IOError> {
@@ -72,7 +69,6 @@ pub async fn create_dir(
         })
 }
 
-// create_dir_all
 pub async fn create_dir_all(
     path: impl AsRef<std::path::Path>,
 ) -> Result<(), IOError> {
@@ -85,7 +81,6 @@ pub async fn create_dir_all(
         })
 }
 
-// remove_dir_all
 pub async fn remove_dir_all(
     path: impl AsRef<std::path::Path>,
 ) -> Result<(), IOError> {
@@ -98,20 +93,37 @@ pub async fn remove_dir_all(
         })
 }
 
-// read_to_string
+/// Reads a text file to a string, automatically detecting its encoding and
-pub async fn read_to_string(
+/// substituting any invalid characters with the Unicode replacement character.
+///
+/// This function is best suited for reading Minecraft instance files, whose
+/// encoding may vary depending on the platform, launchers, client versions
+/// (older Minecraft versions tended to rely on the system's default codepage
+/// more on Windows platforms), and mods used, while not being highly sensitive
+/// to occasional occurrences of mojibake or character replacements.
+pub async fn read_any_encoding_to_string(
     path: impl AsRef<std::path::Path>,
-) -> Result<String, IOError> {
+) -> Result<(String, &'static encoding_rs::Encoding), IOError> {
     let path = path.as_ref();
-    tokio::fs::read_to_string(path)
+    let file_bytes =
-        .await
+        tokio::fs::read(path)
-        .map_err(|e| IOError::IOPathError {
+            .await
-            source: e,
+            .map_err(|e| IOError::IOPathError {
-            path: path.to_string_lossy().to_string(),
+                source: e,
-        })
+                path: path.to_string_lossy().to_string(),
+            })?;
+
+    let file_encoding = {
+        let mut encoding_detector = chardetng::EncodingDetector::new();
+        encoding_detector.feed(&file_bytes, true);
+        encoding_detector.guess(None, true)
+    };
+
+    let (file_string, actual_file_encoding, _) =
+        file_encoding.decode(&file_bytes);
+    Ok((file_string.to_string(), actual_file_encoding))
 }
 
-// read
 pub async fn read(
     path: impl AsRef<std::path::Path>,
 ) -> Result<Vec<u8>, IOError> {
@@ -124,7 +136,6 @@ pub async fn read(
         })
 }
 
-// write
 pub async fn write(
     path: impl AsRef<std::path::Path>,
     data: impl AsRef<[u8]>,
@@ -186,7 +197,6 @@ pub fn is_same_disk(old_dir: &Path, new_dir: &Path) -> Result<bool, IOError> {
     }
 }
 
-// rename
 pub async fn rename_or_move(
     from: impl AsRef<std::path::Path>,
     to: impl AsRef<std::path::Path>,
@@ -228,7 +238,6 @@ async fn move_recursive(from: &Path, to: &Path) -> Result<(), IOError> {
     Ok(())
 }
 
-// copy
 pub async fn copy(
     from: impl AsRef<std::path::Path>,
     to: impl AsRef<std::path::Path>,
@@ -243,7 +252,6 @@ pub async fn copy(
         })
 }
 
-// remove file
 pub async fn remove_file(
     path: impl AsRef<std::path::Path>,
 ) -> Result<(), IOError> {
@@ -256,7 +264,6 @@ pub async fn remove_file(
         })
 }
 
-// open file
 pub async fn open_file(
     path: impl AsRef<std::path::Path>,
 ) -> Result<tokio::fs::File, IOError> {
@@ -269,7 +276,6 @@ pub async fn open_file(
         })
 }
 
-// remove dir
 pub async fn remove_dir(
     path: impl AsRef<std::path::Path>,
 ) -> Result<(), IOError> {
@@ -282,7 +288,6 @@ pub async fn remove_dir(
         })
 }
 
-// metadata
 pub async fn metadata(
     path: impl AsRef<std::path::Path>,
 ) -> Result<std::fs::Metadata, IOError> {