Optimize and fix some bugs in indexing (#98)

* Improve curseforge and local indexing

This should make curseforge indexing more efficient, and reuses
some of the normal local indexing for the queued indexing of
recently created mods.

* Unify impls for single and multiple routes for mods and versions

This uses the same backend for the single and multiple query
routes so that they no longer return inconsistent information.

* Cache valid curseforge mod ids to reduce request load

This caches the ids of minecraft mods and reuses them on indexing
to reduce the amount of unused addons that are returned.

src/search/indexing/curseforge_import.rs

@@ -6,217 +6,318 @@ use std::borrow::Cow;
 
 #[derive(Serialize, Deserialize, Debug)]
 #[serde(rename_all = "camelCase")]
-pub struct Attachment {
-    pub url: String,
-    pub thumbnail_url: String,
+pub struct Attachment<'a> {
+    pub url: Cow<'a, str>,
+    pub thumbnail_url: Cow<'a, str>,
     pub is_default: bool,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
-pub struct Category {
-    pub name: String,
+pub struct Category<'a> {
+    pub name: Cow<'a, str>,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
-pub struct Author {
-    pub name: String,
-    pub url: String,
+pub struct Author<'a> {
+    pub name: Cow<'a, str>,
+    pub url: Cow<'a, str>,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
 #[serde(rename_all = "camelCase")]
-pub struct CurseVersion {
-    pub game_version: String,
+pub struct CurseVersion<'a> {
+    pub game_version: Cow<'a, str>,
 }
 
 #[derive(Serialize, Deserialize, Debug)]
 #[serde(rename_all = "camelCase")]
-pub struct CurseForgeMod {
-    pub id: i32,
-    pub name: String,
-    pub authors: Vec<Author>,
-    pub attachments: Vec<Attachment>,
-    pub website_url: String,
-    pub summary: String,
+pub struct LatestFile<'a> {
+    pub game_version: Vec<Cow<'a, str>>,
+    pub modules: Vec<VersionModule<'a>>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct VersionModule<'a> {
+    pub foldername: Cow<'a, str>,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct CurseForgeMod<'a> {
+    pub id: u32,
+    pub name: Cow<'a, str>,
+    pub authors: Vec<Option<Author<'a>>>,
+    pub attachments: Vec<Attachment<'a>>,
+    pub website_url: Cow<'a, str>,
+    pub summary: Cow<'a, str>,
     pub download_count: f32,
-    pub categories: Vec<Category>,
-    pub game_version_latest_files: Vec<CurseVersion>,
-    pub date_created: String,
-    pub date_modified: String,
-    pub game_slug: String,
+    pub categories: Vec<Category<'a>>,
+    pub latest_files: Vec<LatestFile<'a>>,
+    pub game_version_latest_files: Vec<CurseVersion<'a>>,
+    pub date_created: chrono::DateTime<chrono::Utc>,
+    pub date_modified: chrono::DateTime<chrono::Utc>,
+    pub category_section: CategorySection,
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+#[serde(rename_all = "camelCase")]
+pub struct CategorySection {
+    pub id: u32,
+}
+
+#[derive(Default)]
+struct Loaders {
+    forge: bool,
+    fabric: bool,
+    liteloader: bool,
+    rift: bool,
+}
+
+lazy_static::lazy_static! {
+    static ref CURSEFORGE_CATEGORIES: std::collections::HashMap<&'static str, &'static str> = {
+        let mut map = std::collections::HashMap::new();
+        map.insert("World Gen", "worldgen");
+        map.insert("Biomes", "worldgen");
+        map.insert("Ores and Resources", "worldgen");
+        map.insert("Structures", "worldgen");
+        map.insert("Dimensions", "worldgen");
+        map.insert("Mobs", "worldgen");
+        map.insert("Technology", "technology");
+        map.insert("Processing", "technology");
+        map.insert("Player Transport", "technology");
+        map.insert("Energy, Fluid, and Item Transport", "technology");
+        map.insert("Food", "food");
+        map.insert("Farming", "food");
+        map.insert("Energy", "technology");
+        map.insert("Redstone", "technology");
+        map.insert("Genetics", "technology");
+        map.insert("Magic", "magic");
+        map.insert("Storage", "storage");
+        map.insert("API and Library", "library");
+        map.insert("Adventure and RPG", "adventure");
+        map.insert("Map and Information", "utility");
+        map.insert("Cosmetic", "decoration");
+        map.insert("Addons", "misc");
+        map.insert("Thermal Expansion", "misc");
+        map.insert("Tinker's Construct", "misc");
+        map.insert("Industrial Craft", "misc");
+        map.insert("Thaumcraft", "misc");
+        map.insert("Buildcraft", "misc");
+        map.insert("Forestry", "misc");
+        map.insert("Blood Magic", "misc");
+        map.insert("Lucky Blocks", "misc");
+        map.insert("Applied Energistics 2", "misc");
+        map.insert("CraftTweaker", "misc");
+        map.insert("Miscellaneous", "misc");
+        map.insert("Armor, Tools, and Weapons", "equipment");
+        map.insert("Server Utility", "utility");
+        map
+    };
 }
 
 pub async fn index_curseforge(
-    start_index: i32,
-    end_index: i32,
+    start_index: u32,
+    end_index: u32,
+    cache_path: &std::path::Path,
 ) -> Result<Vec<UploadSearchMod>, IndexingError> {
     info!("Indexing curseforge mods!");
+    let start = std::time::Instant::now();
 
     let mut docs_to_add: Vec<UploadSearchMod> = vec![];
 
+    let cache = std::fs::File::open(cache_path)
+        .map(std::io::BufReader::new)
+        .map(serde_json::from_reader::<_, Vec<u32>>);
+
+    let requested_ids;
+
+    // This caching system can't handle segmented indexing
+    if let Ok(Ok(mut cache)) = cache {
+        let end = cache.last().copied().unwrap_or(start_index);
+        cache.extend(end..end_index);
+        requested_ids = serde_json::to_string(&cache)?;
+    } else {
+        // This ends up being around 3 MiB
+        // Serde json is better than using debug formatting since it doesn't
+        // include spaces after commas, removing a lot of the extra size
+        requested_ids = serde_json::to_string(&(start_index..end_index).collect::<Vec<_>>())?;
+    }
+
     let res = reqwest::Client::new()
         .post("https://addons-ecs.forgesvc.net/api/v2/addon")
         .header(reqwest::header::CONTENT_TYPE, "application/json")
-        .body(format!(
-            "{:?}",
-            (start_index..end_index).collect::<Vec<_>>()
-        ))
+        .body(requested_ids)
         .send()
-        .await
-        .map_err(IndexingError::CurseforgeImportError)?;
+        .await?;
 
-    let curseforge_mods: Vec<CurseForgeMod> = res
-        .json()
-        .await
-        .map_err(IndexingError::CurseforgeImportError)?;
+    // The response ends up being about 300MiB, so we have to deal with
+    // it efficiently.  Reading it as bytes and then deserializing with
+    // borrowed data should avoid copying it, but it may take a bit more
+    // memory.  To do this efficiently, we would have to get serde_json
+    // to skip deserializing mods with category_section.id != 8
+    // It's only 100MiB when using the cached ids, since that eliminates
+    // all "addons" that aren't minecraft mods
+    let buffer = res.bytes().await?;
 
-    for curseforge_mod in curseforge_mods {
-        if curseforge_mod.game_slug != "minecraft"
-            || !curseforge_mod.website_url.contains("/mc-mods/")
-        {
-            continue;
+    let mut curseforge_mods: Vec<CurseForgeMod> = serde_json::from_slice(&buffer)?;
+    // This should remove many of the mods from the list before processing
+    curseforge_mods.retain(|m| m.category_section.id == 8);
+
+    // Only write to the cache if this doesn't skip mods at the start
+    // The caching system iterates through all ids normally past the last
+    // id in the cache, so the end_index shouldn't matter.
+    if start_index <= 1 {
+        let mut ids = curseforge_mods.iter().map(|m| m.id).collect::<Vec<_>>();
+        ids.sort_unstable();
+        if let Err(e) = std::fs::write(cache_path, serde_json::to_string(&ids)?) {
+            log::warn!("Error writing to index id cache: {}", e);
         }
+    }
+
+    for mut curseforge_mod in curseforge_mods {
+        // The gameId of minecraft is 432
+        // The categorySection.id for mods is always 8
+        // The categorySection.id 8 appears to be unique to minecraft mods
+        // if curseforge_mod.game_slug != "minecraft"
+        //     || !curseforge_mod.website_url.contains("/mc-mods/")
+        // if curseforge_mod.category_section.id != 8 {
+        //     continue;
+        // }
 
         let mut mod_game_versions = vec![];
 
-        let mut using_forge = false;
-        let mut using_fabric = false;
+        let mut loaders = Loaders::default();
 
-        for version in curseforge_mod.game_version_latest_files {
-            if let Some(parsed) = version
-                .game_version
-                .get(2..)
-                .and_then(|f| f.parse::<f32>().ok())
-            {
-                if parsed < 14.0 {
-                    using_forge = true;
+        for file in curseforge_mod.latest_files {
+            for version in file.game_version {
+                match &*version {
+                    "Fabric" => loaders.forge = true,
+                    "Forge" => loaders.fabric = true,
+                    "Rift" => loaders.rift = true,
+                    _ => (),
                 }
             }
+            for module in file.modules {
+                match &*module.foldername {
+                    "fabric.mod.json" => loaders.fabric = true,
+                    "mcmod.info" => loaders.forge = true, // 1.13+ forge uses META-INF/mods.toml
+                    "riftmod.json" => loaders.rift = true,
+                    "litemod.json" => loaders.liteloader = true,
+                    _ => (),
+                }
+            }
+            // TODO: files ending with .litemod should also enable liteloader
+            // if we decide to add true support for it;  That requires extra
+            // deserializing work, so I'm not adding it for now
+        }
 
+        let mut latest = None;
+
+        for version in curseforge_mod.game_version_latest_files {
+            let mut split = version.game_version.split('.');
+            let version_numbers = (
+                split.next().and_then(|s| s.parse::<u8>().ok()).unwrap_or(0),
+                split.next().and_then(|s| s.parse::<u8>().ok()).unwrap_or(0),
+                split.next().and_then(|s| s.parse::<u8>().ok()).unwrap_or(0),
+            );
+
+            if let Some((number, _)) = latest {
+                if version_numbers > number {
+                    latest = Some((version_numbers, version.game_version.clone()));
+                }
+            } else {
+                latest = Some((version_numbers, version.game_version.clone()))
+            }
+
+            if ((1, 0, 0)..(1, 14, 0)).contains(&version_numbers) {
+                // Is this a reasonable assumption to make?
+                loaders.forge = true;
+            }
             mod_game_versions.push(version.game_version);
         }
 
-        let mut mod_categories = vec![];
+        let mut mod_categories = std::collections::HashSet::new();
 
         for category in curseforge_mod.categories {
-            match &category.name[..] {
-                "World Gen" => mod_categories.push(String::from("worldgen")),
-                "Biomes" => mod_categories.push(String::from("worldgen")),
-                "Ores and Resources" => mod_categories.push(String::from("worldgen")),
-                "Structures" => mod_categories.push(String::from("worldgen")),
-                "Dimensions" => mod_categories.push(String::from("worldgen")),
-                "Mobs" => mod_categories.push(String::from("worldgen")),
-                "Technology" => mod_categories.push(String::from("technology")),
-                "Processing" => mod_categories.push(String::from("technology")),
-                "Player Transport" => mod_categories.push(String::from("technology")),
-                "Energy, Fluid, and Item Transport" => {
-                    mod_categories.push(String::from("technology"))
-                }
-                "Food" => mod_categories.push(String::from("food")),
-                "Farming" => mod_categories.push(String::from("food")),
-                "Energy" => mod_categories.push(String::from("technology")),
-                "Redstone" => mod_categories.push(String::from("technology")),
-                "Genetics" => mod_categories.push(String::from("technology")),
-                "Magic" => mod_categories.push(String::from("magic")),
-                "Storage" => mod_categories.push(String::from("storage")),
-                "API and Library" => mod_categories.push(String::from("library")),
-                "Adventure and RPG" => mod_categories.push(String::from("adventure")),
-                "Map and Information" => mod_categories.push(String::from("utility")),
-                "Cosmetic" => mod_categories.push(String::from("decoration")),
-                "Addons" => mod_categories.push(String::from("misc")),
-                "Thermal Expansion" => mod_categories.push(String::from("misc")),
-                "Tinker's Construct" => mod_categories.push(String::from("misc")),
-                "Industrial Craft" => mod_categories.push(String::from("misc")),
-                "Thaumcraft" => mod_categories.push(String::from("misc")),
-                "Buildcraft" => mod_categories.push(String::from("misc")),
-                "Forestry" => mod_categories.push(String::from("misc")),
-                "Blood Magic" => mod_categories.push(String::from("misc")),
-                "Lucky Blocks" => mod_categories.push(String::from("misc")),
-                "Applied Energistics 2" => mod_categories.push(String::from("misc")),
-                "CraftTweaker" => mod_categories.push(String::from("misc")),
-                "Miscellaneous" => mod_categories.push(String::from("misc")),
-                "Armor, Tools, and Weapons" => mod_categories.push(String::from("equipment")),
-                "Server Utility" => mod_categories.push(String::from("utility")),
-                "Fabric" => mod_categories.push(String::from("fabric")),
-                _ => {}
+            if category.name == "Fabric" {
+                loaders.fabric = true;
+            } else if let Some(category) = CURSEFORGE_CATEGORIES.get(&*category.name) {
+                mod_categories.insert(*category);
             }
         }
 
-        if mod_categories.iter().any(|e| e == "fabric") {
-            using_fabric = true;
+        if !(loaders.fabric || loaders.rift || loaders.liteloader || loaders.forge) {
+            // Assume that mods without loaders will be
+            loaders.forge = true;
         }
 
-        mod_categories.sort_unstable();
-        mod_categories.dedup();
-        mod_categories.truncate(3);
+        let mut mod_categories = mod_categories
+            .into_iter()
+            .take(3)
+            .map(Cow::Borrowed)
+            .collect::<Vec<_>>();
 
-        if using_forge {
-            mod_categories.push(String::from("forge"));
+        if loaders.forge {
+            mod_categories.push(Cow::Borrowed("forge"));
         }
-        if using_fabric {
-            // The only way this could happen is if "fabric" is already a category
-            // mod_categories.push(String::from("fabric"));
+        if loaders.fabric {
+            mod_categories.push(Cow::Borrowed("fabric"));
         }
 
-        let mut mod_attachments = curseforge_mod.attachments;
-        mod_attachments.retain(|x| x.is_default);
+        let latest_version = latest
+            .map(|(_, name)| name)
+            .unwrap_or_else(|| Cow::Borrowed("None"));
 
-        if mod_attachments.is_empty() {
-            mod_attachments.push(Attachment {
-                url: String::new(),
-                thumbnail_url: String::new(),
-                is_default: true,
-            })
-        }
-
-        let latest_version = if !mod_game_versions.is_empty() {
-            mod_game_versions[0].to_string()
-        } else {
-            "None".to_string()
-        };
-
-        let icon_url = mod_attachments[0]
-            .thumbnail_url
-            .replace("/256/256/", "/64/64/");
-
-        let created = curseforge_mod
-            .date_created
-            .parse::<chrono::DateTime<chrono::Utc>>()?;
-        let modified = curseforge_mod
-            .date_modified
-            .parse::<chrono::DateTime<chrono::Utc>>()?;
+        let icon_url = curseforge_mod
+            .attachments
+            .iter()
+            .find(|a| a.is_default)
+            .map(|a| a.thumbnail_url.replace("/256/256/", "/64/64/"))
+            .unwrap_or_default();
 
         let author;
         let author_url;
 
-        if let Some(user) = curseforge_mod.authors.get(0) {
-            author = user.name.clone();
-            author_url = user.url.clone();
+        if let Some(user) = curseforge_mod
+            .authors
+            .get_mut(0)
+            .map(Option::take)
+            .flatten()
+        {
+            author = user.name.into_owned();
+            author_url = user.url.into_owned();
         } else {
-            author = String::from("unknown");
-            author_url = curseforge_mod.website_url.clone();
+            author = "unknown".to_owned();
+            author_url = String::from(&*curseforge_mod.website_url);
         }
 
         docs_to_add.push(UploadSearchMod {
             mod_id: format!("curse-{}", curseforge_mod.id),
             author,
-            title: curseforge_mod.name,
+            title: curseforge_mod.name.into_owned(),
             description: curseforge_mod.summary.chars().take(150).collect(),
             categories: mod_categories,
-            versions: mod_game_versions.clone(),
+            versions: mod_game_versions.into_iter().map(String::from).collect(),
             downloads: curseforge_mod.download_count as i32,
-            page_url: curseforge_mod.website_url,
+            page_url: curseforge_mod.website_url.into_owned(),
             icon_url,
             author_url,
-            date_created: created,
-            created_timestamp: created.timestamp(),
-            date_modified: modified,
-            modified_timestamp: modified.timestamp(),
+            date_created: curseforge_mod.date_created,
+            created_timestamp: curseforge_mod.date_created.timestamp(),
+            date_modified: curseforge_mod.date_modified,
+            modified_timestamp: curseforge_mod.date_modified.timestamp(),
             latest_version,
             host: Cow::Borrowed("curseforge"),
             empty: Cow::Borrowed("{}{}{}"),
         })
     }
 
+    let duration = start.elapsed();
+    info!(
+        "Finished indexing curseforge; Took {:5.2}s",
+        duration.as_secs_f32()
+    );
+
     Ok(docs_to_add)
 }

src/search/indexing/local_import.rs

@@ -20,12 +20,13 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchMod>, IndexingE
 
     while let Some(result) = mods.next().await {
         if let Ok(mod_data) = result {
-            let versions: Vec<String> = sqlx::query!(
+            let versions = sqlx::query!(
                 "
                 SELECT gv.version FROM versions
                     INNER JOIN game_versions_versions gvv ON gvv.joining_version_id=versions.id
                     INNER JOIN game_versions gv ON gvv.game_version_id=gv.id
                 WHERE versions.mod_id = $1
+                ORDER BY gv.created ASC
                 ",
                 mod_data.id
             )
@@ -34,7 +35,7 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchMod>, IndexingE
             .try_collect::<Vec<String>>()
             .await?;
 
-            let loaders: Vec<String> = sqlx::query!(
+            let loaders = sqlx::query!(
                 "
                 SELECT loaders.loader FROM versions
                 INNER JOIN loaders_versions lv ON lv.version_id = versions.id
@@ -44,8 +45,8 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchMod>, IndexingE
                 mod_data.id
             )
             .fetch_many(&pool)
-            .try_filter_map(|e| async { Ok(e.right().map(|c| c.loader)) })
-            .try_collect::<Vec<String>>()
+            .try_filter_map(|e| async { Ok(e.right().map(|c| Cow::Owned(c.loader))) })
+            .try_collect::<Vec<Cow<str>>>()
             .await?;
 
             let mut categories = sqlx::query!(
@@ -58,8 +59,8 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchMod>, IndexingE
                 mod_data.id
             )
             .fetch_many(&pool)
-            .try_filter_map(|e| async { Ok(e.right().map(|c| c.category)) })
-            .try_collect::<Vec<String>>()
+            .try_filter_map(|e| async { Ok(e.right().map(|c| Cow::Owned(c.category))) })
+            .try_collect::<Vec<Cow<str>>>()
             .await?;
 
             categories.extend(loaders);
@@ -85,6 +86,15 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchMod>, IndexingE
             let mod_id = crate::models::ids::ModId(mod_data.id as u64);
             let author_id = crate::models::ids::UserId(user.id as u64);
 
+            // TODO: is this correct? This just gets the latest version of
+            // minecraft that this mod has a version that supports; it doesn't
+            // take betas or other info into account.
+            let latest_version = versions
+                .get(0)
+                .cloned()
+                .map(Cow::Owned)
+                .unwrap_or_else(|| Cow::Borrowed(""));
+
             docs_to_add.push(UploadSearchMod {
                 mod_id: format!("local-{}", mod_id),
                 title: mod_data.title,
@@ -100,7 +110,7 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchMod>, IndexingE
                 created_timestamp: mod_data.published.timestamp(),
                 date_modified: mod_data.updated,
                 modified_timestamp: mod_data.updated.timestamp(),
-                latest_version: "".to_string(), // TODO: Info about latest version
+                latest_version,
                 host: Cow::Borrowed("modrinth"),
                 empty: Cow::Borrowed("{}{}{}"),
             });
@@ -109,3 +119,112 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchMod>, IndexingE
 
     Ok(docs_to_add)
 }
+
+pub async fn query_one(
+    id: crate::database::models::ModId,
+    exec: &mut sqlx::PgConnection,
+) -> Result<UploadSearchMod, IndexingError> {
+    let mod_data = sqlx::query!(
+        "
+        SELECT m.id, m.title, m.description, m.downloads, m.icon_url, m.body_url, m.published, m.updated, m.team_id
+        FROM mods m
+        WHERE id = $1
+        ",
+        id.0,
+    ).fetch_one(&mut *exec).await?;
+
+    let versions = sqlx::query!(
+        "
+        SELECT gv.version FROM versions
+            INNER JOIN game_versions_versions gvv ON gvv.joining_version_id=versions.id
+            INNER JOIN game_versions gv ON gvv.game_version_id=gv.id
+        WHERE versions.mod_id = $1
+        ORDER BY gv.created ASC
+        ",
+        mod_data.id
+    )
+    .fetch_many(&mut *exec)
+    .try_filter_map(|e| async { Ok(e.right().map(|c| c.version)) })
+    .try_collect::<Vec<String>>()
+    .await?;
+
+    let loaders = sqlx::query!(
+        "
+        SELECT loaders.loader FROM versions
+        INNER JOIN loaders_versions lv ON lv.version_id = versions.id
+        INNER JOIN loaders ON loaders.id = lv.loader_id
+        WHERE versions.mod_id = $1
+        ",
+        mod_data.id
+    )
+    .fetch_many(&mut *exec)
+    .try_filter_map(|e| async { Ok(e.right().map(|c| Cow::Owned(c.loader))) })
+    .try_collect::<Vec<Cow<str>>>()
+    .await?;
+
+    let mut categories = sqlx::query!(
+        "
+        SELECT c.category
+        FROM mods_categories mc
+            INNER JOIN categories c ON mc.joining_category_id=c.id
+        WHERE mc.joining_mod_id = $1
+        ",
+        mod_data.id
+    )
+    .fetch_many(&mut *exec)
+    .try_filter_map(|e| async { Ok(e.right().map(|c| Cow::Owned(c.category))) })
+    .try_collect::<Vec<Cow<str>>>()
+    .await?;
+
+    categories.extend(loaders);
+
+    let user = sqlx::query!(
+        "
+        SELECT u.id, u.username FROM users u
+        INNER JOIN team_members tm ON tm.user_id = u.id
+        WHERE tm.team_id = $2 AND tm.role = $1
+        ",
+        crate::models::teams::OWNER_ROLE,
+        mod_data.team_id,
+    )
+    .fetch_one(&mut *exec)
+    .await?;
+
+    let mut icon_url = "".to_string();
+
+    if let Some(url) = mod_data.icon_url {
+        icon_url = url;
+    }
+
+    let mod_id = crate::models::ids::ModId(mod_data.id as u64);
+    let author_id = crate::models::ids::UserId(user.id as u64);
+
+    // TODO: is this correct? This just gets the latest version of
+    // minecraft that this mod has a version that supports; it doesn't
+    // take betas or other info into account.
+    let latest_version = versions
+        .get(0)
+        .cloned()
+        .map(Cow::Owned)
+        .unwrap_or_else(|| Cow::Borrowed(""));
+
+    Ok(UploadSearchMod {
+        mod_id: format!("local-{}", mod_id),
+        title: mod_data.title,
+        description: mod_data.description,
+        categories,
+        versions,
+        downloads: mod_data.downloads,
+        page_url: format!("https://modrinth.com/mod/{}", mod_id),
+        icon_url,
+        author: user.username,
+        author_url: format!("https://modrinth.com/user/{}", author_id),
+        date_created: mod_data.published,
+        created_timestamp: mod_data.published.timestamp(),
+        date_modified: mod_data.updated,
+        modified_timestamp: mod_data.updated.timestamp(),
+        latest_version,
+        host: Cow::Borrowed("modrinth"),
+        empty: Cow::Borrowed("{}{}{}"),
+    })
+}

src/search/indexing/mod.rs

@@ -18,7 +18,7 @@ pub enum IndexingError {
     #[error("Error while connecting to the MeiliSearch database")]
     IndexDBError(#[from] meilisearch_sdk::errors::Error),
     #[error("Error while importing mods from CurseForge")]
-    CurseforgeImportError(reqwest::Error),
+    CurseforgeImportError(#[from] reqwest::Error),
     #[error("Error while serializing or deserializing JSON: {0}")]
     SerDeError(#[from] serde_json::Error),
     #[error("Error while parsing a timestamp: {0}")]
@@ -63,6 +63,8 @@ pub async fn index_mods(
 ) -> Result<(), IndexingError> {
     let mut docs_to_add: Vec<UploadSearchMod> = vec![];
 
+    let cache_path = std::path::PathBuf::from(std::env::var_os("INDEX_CACHE_PATH").unwrap());
+
     if settings.index_local {
         docs_to_add.append(&mut index_local(pool.clone()).await?);
     }
@@ -72,7 +74,7 @@ pub async fn index_mods(
             .map(|i| i.parse().unwrap())
             .unwrap_or(450_000);
 
-        docs_to_add.append(&mut index_curseforge(1, end_index).await?);
+        docs_to_add.append(&mut index_curseforge(1, end_index, &cache_path).await?);
     }
 
     // Write Indices
@@ -284,3 +286,60 @@ fn default_settings() -> Settings {
 }
 
 //endregion
+
+// This shouldn't be relied on for proper sorting, but it makes an
+// attempt at getting proper sorting for mojang's versions.
+// This isn't currenly used, but I wrote it and it works, so I'm
+// keeping this mess in case someone needs it in the future.
+#[allow(dead_code)]
+pub fn sort_mods(a: &str, b: &str) -> std::cmp::Ordering {
+    use std::cmp::Ordering;
+
+    let cmp = a.contains('.').cmp(&b.contains('.'));
+    if cmp != Ordering::Equal {
+        return cmp;
+    }
+    let mut a = a.split(&['.', '-'] as &[char]);
+    let mut b = b.split(&['.', '-'] as &[char]);
+    let a = (a.next(), a.next(), a.next(), a.next());
+    let b = (b.next(), b.next(), b.next(), b.next());
+    if a.0 == b.0 {
+        let cmp =
+            a.1.map(|s| s.chars().all(|c| c.is_ascii_digit()))
+                .cmp(&b.1.map(|s| s.chars().all(|c| c.is_ascii_digit())));
+        if cmp != Ordering::Equal {
+            return cmp;
+        }
+        if a.1 == b.1 {
+            let cmp =
+                a.2.map(|s| s.chars().all(|c| c.is_ascii_digit()))
+                    .unwrap_or(true)
+                    .cmp(
+                        &b.2.map(|s| s.chars().all(|c| c.is_ascii_digit()))
+                            .unwrap_or(true),
+                    );
+            if cmp != Ordering::Equal {
+                return cmp;
+            }
+            if a.2 == b.2 {
+                match (a.3.is_some(), b.3.is_some()) {
+                    (false, false) => Ordering::Equal,
+                    (false, true) => Ordering::Greater,
+                    (true, false) => Ordering::Less,
+                    (true, true) => a.3.cmp(&b.3),
+                }
+            } else {
+                a.2.cmp(&b.2)
+            }
+        } else {
+            a.1.cmp(&b.1)
+        }
+    } else {
+        match (a.0 == Some("1"), b.0 == Some("1")) {
+            (false, false) => a.0.cmp(&b.0),
+            (true, false) => Ordering::Greater,
+            (false, true) => Ordering::Less,
+            (true, true) => Ordering::Equal, // unreachable
+        }
+    }
+}

src/search/mod.rs

@@ -66,13 +66,13 @@ pub struct UploadSearchMod {
     pub author: String,
     pub title: String,
     pub description: String,
-    pub categories: Vec<String>,
+    pub categories: Vec<Cow<'static, str>>,
     pub versions: Vec<String>,
     pub downloads: i32,
     pub page_url: String,
     pub icon_url: String,
     pub author_url: String,
-    pub latest_version: String,
+    pub latest_version: Cow<'static, str>,
 
     /// RFC 3339 formatted creation date of the mod
     pub date_created: DateTime<Utc>,