Optimize and fix some bugs in indexing (#98)

* Improve curseforge and local indexing This should make curseforge indexing more efficient, and reuses some of the normal local indexing for the queued indexing of recently created mods. * Unify impls for single and multiple routes for mods and versions This uses the same backend for the single and multiple query routes so that they no longer return inconsistent information. * Cache valid curseforge mod ids to reduce request load This caches the ids of minecraft mods and reuses them on indexing to reduce the amount of unused addons that are returned.
2020-11-03 18:55:50 -06:00
parent da79386cc3
commit d477874535
15 changed files with 745 additions and 343 deletions
--- a/src/search/indexing/local_import.rs
+++ b/src/search/indexing/local_import.rs
@@ -20,12 +20,13 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchMod>, IndexingE

    while let Some(result) = mods.next().await {
        if let Ok(mod_data) = result {
-            let versions: Vec<String> = sqlx::query!(
+            let versions = sqlx::query!(
                "
                SELECT gv.version FROM versions
                    INNER JOIN game_versions_versions gvv ON gvv.joining_version_id=versions.id
                    INNER JOIN game_versions gv ON gvv.game_version_id=gv.id
                WHERE versions.mod_id = $1
+                ORDER BY gv.created ASC
                ",
                mod_data.id
            )
@@ -34,7 +35,7 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchMod>, IndexingE
            .try_collect::<Vec<String>>()
            .await?;

-            let loaders: Vec<String> = sqlx::query!(
+            let loaders = sqlx::query!(
                "
                SELECT loaders.loader FROM versions
                INNER JOIN loaders_versions lv ON lv.version_id = versions.id
@@ -44,8 +45,8 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchMod>, IndexingE
                mod_data.id
            )
            .fetch_many(&pool)
-            .try_filter_map(|e| async { Ok(e.right().map(|c| c.loader)) })
-            .try_collect::<Vec<String>>()
+            .try_filter_map(|e| async { Ok(e.right().map(|c| Cow::Owned(c.loader))) })
+            .try_collect::<Vec<Cow<str>>>()
            .await?;

            let mut categories = sqlx::query!(
@@ -58,8 +59,8 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchMod>, IndexingE
                mod_data.id
            )
            .fetch_many(&pool)
-            .try_filter_map(|e| async { Ok(e.right().map(|c| c.category)) })
-            .try_collect::<Vec<String>>()
+            .try_filter_map(|e| async { Ok(e.right().map(|c| Cow::Owned(c.category))) })
+            .try_collect::<Vec<Cow<str>>>()
            .await?;

            categories.extend(loaders);
@@ -85,6 +86,15 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchMod>, IndexingE
            let mod_id = crate::models::ids::ModId(mod_data.id as u64);
            let author_id = crate::models::ids::UserId(user.id as u64);

+            // TODO: is this correct? This just gets the latest version of
+            // minecraft that this mod has a version that supports; it doesn't
+            // take betas or other info into account.
+            let latest_version = versions
+                .get(0)
+                .cloned()
+                .map(Cow::Owned)
+                .unwrap_or_else(|| Cow::Borrowed(""));
+
            docs_to_add.push(UploadSearchMod {
                mod_id: format!("local-{}", mod_id),
                title: mod_data.title,
@@ -100,7 +110,7 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchMod>, IndexingE
                created_timestamp: mod_data.published.timestamp(),
                date_modified: mod_data.updated,
                modified_timestamp: mod_data.updated.timestamp(),
-                latest_version: "".to_string(), // TODO: Info about latest version
+                latest_version,
                host: Cow::Borrowed("modrinth"),
                empty: Cow::Borrowed("{}{}{}"),
            });
@@ -109,3 +119,112 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchMod>, IndexingE

    Ok(docs_to_add)
 }
+
+pub async fn query_one(
+    id: crate::database::models::ModId,
+    exec: &mut sqlx::PgConnection,
+) -> Result<UploadSearchMod, IndexingError> {
+    let mod_data = sqlx::query!(
+        "
+        SELECT m.id, m.title, m.description, m.downloads, m.icon_url, m.body_url, m.published, m.updated, m.team_id
+        FROM mods m
+        WHERE id = $1
+        ",
+        id.0,
+    ).fetch_one(&mut *exec).await?;
+
+    let versions = sqlx::query!(
+        "
+        SELECT gv.version FROM versions
+            INNER JOIN game_versions_versions gvv ON gvv.joining_version_id=versions.id
+            INNER JOIN game_versions gv ON gvv.game_version_id=gv.id
+        WHERE versions.mod_id = $1
+        ORDER BY gv.created ASC
+        ",
+        mod_data.id
+    )
+    .fetch_many(&mut *exec)
+    .try_filter_map(|e| async { Ok(e.right().map(|c| c.version)) })
+    .try_collect::<Vec<String>>()
+    .await?;
+
+    let loaders = sqlx::query!(
+        "
+        SELECT loaders.loader FROM versions
+        INNER JOIN loaders_versions lv ON lv.version_id = versions.id
+        INNER JOIN loaders ON loaders.id = lv.loader_id
+        WHERE versions.mod_id = $1
+        ",
+        mod_data.id
+    )
+    .fetch_many(&mut *exec)
+    .try_filter_map(|e| async { Ok(e.right().map(|c| Cow::Owned(c.loader))) })
+    .try_collect::<Vec<Cow<str>>>()
+    .await?;
+
+    let mut categories = sqlx::query!(
+        "
+        SELECT c.category
+        FROM mods_categories mc
+            INNER JOIN categories c ON mc.joining_category_id=c.id
+        WHERE mc.joining_mod_id = $1
+        ",
+        mod_data.id
+    )
+    .fetch_many(&mut *exec)
+    .try_filter_map(|e| async { Ok(e.right().map(|c| Cow::Owned(c.category))) })
+    .try_collect::<Vec<Cow<str>>>()
+    .await?;
+
+    categories.extend(loaders);
+
+    let user = sqlx::query!(
+        "
+        SELECT u.id, u.username FROM users u
+        INNER JOIN team_members tm ON tm.user_id = u.id
+        WHERE tm.team_id = $2 AND tm.role = $1
+        ",
+        crate::models::teams::OWNER_ROLE,
+        mod_data.team_id,
+    )
+    .fetch_one(&mut *exec)
+    .await?;
+
+    let mut icon_url = "".to_string();
+
+    if let Some(url) = mod_data.icon_url {
+        icon_url = url;
+    }
+
+    let mod_id = crate::models::ids::ModId(mod_data.id as u64);
+    let author_id = crate::models::ids::UserId(user.id as u64);
+
+    // TODO: is this correct? This just gets the latest version of
+    // minecraft that this mod has a version that supports; it doesn't
+    // take betas or other info into account.
+    let latest_version = versions
+        .get(0)
+        .cloned()
+        .map(Cow::Owned)
+        .unwrap_or_else(|| Cow::Borrowed(""));
+
+    Ok(UploadSearchMod {
+        mod_id: format!("local-{}", mod_id),
+        title: mod_data.title,
+        description: mod_data.description,
+        categories,
+        versions,
+        downloads: mod_data.downloads,
+        page_url: format!("https://modrinth.com/mod/{}", mod_id),
+        icon_url,
+        author: user.username,
+        author_url: format!("https://modrinth.com/user/{}", author_id),
+        date_created: mod_data.published,
+        created_timestamp: mod_data.published.timestamp(),
+        date_modified: mod_data.updated,
+        modified_timestamp: mod_data.updated.timestamp(),
+        latest_version,
+        host: Cow::Borrowed("modrinth"),
+        empty: Cow::Borrowed("{}{}{}"),
+    })
+}