Game Version types, indexing, and bugfixes (#91)

* Add types to game_versions, allow filtering by version type

- Fixes an issue with version numbers in the initial mod indexing
  queue
- Modifies the /api/v1/categories/game_versions route to take an
  optional query parameter `type` to filter the listed game versions
- Creating tags is now idempotent
- Creating game_versions now requires a JSON body that specifies
  the version type

* Implement automatic indexing of new Minecraft versions

It's currently set to run every 6 hours and isn't configurable; we
could add config for it, but it doesn't seem likely to be rate
limited or have issues with frequency.

migrations/20201021214908_extend-game-version.sql

@@ -0,0 +1,3 @@
+
+ALTER TABLE game_versions
+ADD COLUMN type varchar(16) NOT NULL DEFAULT 'other';

sqlx-data.json

@@ -278,26 +278,6 @@
       ]
     }
   },
-  "1b74bdb59773ffd2a78a56e4d920bb83c322e180e6174c741d4bb722c353de43": {
-    "query": "\n            INSERT INTO loaders (loader)\n            VALUES ($1)\n            RETURNING id\n            ",
-    "describe": {
-      "columns": [
-        {
-          "ordinal": 0,
-          "name": "id",
-          "type_info": "Int4"
-        }
-      ],
-      "parameters": {
-        "Left": [
-          "Varchar"
-        ]
-      },
-      "nullable": [
-        false
-      ]
-    }
-  },
   "1c7b0eb4341af5a7942e52f632cf582561f10b4b6a41a082fb8a60f04ac17c6e": {
     "query": "SELECT EXISTS(SELECT 1 FROM states WHERE id=$1)",
     "describe": {
@@ -499,6 +479,27 @@
       "nullable": []
     }
   },
+  "3d18702f07161c0cdbc31d70b89ffeb3678617ccc44dfc6fb03dd63f47226c7b": {
+    "query": "\n            INSERT INTO game_versions (version, type)\n            VALUES ($1, $2)\n            ON CONFLICT (version) DO UPDATE\n                SET type = excluded.type\n            RETURNING id\n            ",
+    "describe": {
+      "columns": [
+        {
+          "ordinal": 0,
+          "name": "id",
+          "type_info": "Int4"
+        }
+      ],
+      "parameters": {
+        "Left": [
+          "Varchar",
+          "Varchar"
+        ]
+      },
+      "nullable": [
+        false
+      ]
+    }
+  },
   "42e072309779598d0c213280dd8052d1b4889cb24ef5204ca13b74f693b94328": {
     "query": "\n                SELECT user_id FROM team_members tm\n                INNER JOIN mods ON mods.team_id = tm.team_id\n                WHERE mods.id = $1\n                ",
     "describe": {
@@ -554,6 +555,26 @@
       "nullable": []
     }
   },
+  "49e36828e3a0214b48234435e34311735ae32e08d8be1270f8f0db4b27e708ba": {
+    "query": "\n            INSERT INTO loaders (loader)\n            VALUES ($1)\n            ON CONFLICT (loader) DO NOTHING\n            RETURNING id\n            ",
+    "describe": {
+      "columns": [
+        {
+          "ordinal": 0,
+          "name": "id",
+          "type_info": "Int4"
+        }
+      ],
+      "parameters": {
+        "Left": [
+          "Varchar"
+        ]
+      },
+      "nullable": [
+        false
+      ]
+    }
+  },
   "4c99c0840159d18e88cd6094a41117258f2337346c145d926b5b610c76b5125f": {
     "query": "\n                SELECT c.category\n                FROM mods_categories mc\n                    INNER JOIN categories c ON mc.joining_category_id=c.id\n                WHERE mc.joining_mod_id = $1\n                ",
     "describe": {
@@ -620,26 +641,6 @@
       "nullable": []
     }
   },
-  "56cb9274e92f185dee3accf69cca2e34c035efbef908baefeb60548fb14e02bd": {
-    "query": "\n            INSERT INTO categories (category)\n            VALUES ($1)\n            RETURNING id\n            ",
-    "describe": {
-      "columns": [
-        {
-          "ordinal": 0,
-          "name": "id",
-          "type_info": "Int4"
-        }
-      ],
-      "parameters": {
-        "Left": [
-          "Varchar"
-        ]
-      },
-      "nullable": [
-        false
-      ]
-    }
-  },
   "59cf9d085593887595ea45246291f2cd64fc6677d551e96bdb60c09ff1eebf99": {
     "query": "\n                SELECT files.id, files.url, files.filename FROM files\n                WHERE files.version_id = $1\n                ",
     "describe": {
@@ -1224,26 +1225,6 @@
       "nullable": []
     }
   },
-  "bee1e8b7f3588c6b0534443775f3d0d66d960e96a5ae8422aa96a69238f375a4": {
-    "query": "\n            INSERT INTO game_versions (version)\n            VALUES ($1)\n            RETURNING id\n            ",
-    "describe": {
-      "columns": [
-        {
-          "ordinal": 0,
-          "name": "id",
-          "type_info": "Int4"
-        }
-      ],
-      "parameters": {
-        "Left": [
-          "Varchar"
-        ]
-      },
-      "nullable": [
-        false
-      ]
-    }
-  },
   "bf7f721664f5e0ed41adc41b5483037256635f28ff6c4e5d3cbcec4387f9c8ef": {
     "query": "SELECT EXISTS(SELECT 1 FROM users WHERE id=$1)",
     "describe": {
@@ -1735,6 +1716,26 @@
       ]
     }
   },
+  "ec4a3ef12a35bb78002fdafccbdb198b15f9a0fdb2b3e4108f9081b7e56e8769": {
+    "query": "\n            SELECT version FROM game_versions\n            WHERE type = $1\n            ",
+    "describe": {
+      "columns": [
+        {
+          "ordinal": 0,
+          "name": "version",
+          "type_info": "Varchar"
+        }
+      ],
+      "parameters": {
+        "Left": [
+          "Text"
+        ]
+      },
+      "nullable": [
+        false
+      ]
+    }
+  },
   "f0db9d8606ccc2196a9cfafe0e7090dab42bf790f25e0469b8947fac1cf043d5": {
     "query": "\n            SELECT version FROM game_versions\n            WHERE id = $1\n            ",
     "describe": {
@@ -1775,6 +1776,26 @@
       ]
     }
   },
+  "f12ae54acf02e06e9b8774e8c2ea95058a78f6d724645adcd02f9dea6538024f": {
+    "query": "\n            INSERT INTO categories (category)\n            VALUES ($1)\n            ON CONFLICT (category) DO NOTHING\n            RETURNING id\n            ",
+    "describe": {
+      "columns": [
+        {
+          "ordinal": 0,
+          "name": "id",
+          "type_info": "Int4"
+        }
+      ],
+      "parameters": {
+        "Left": [
+          "Varchar"
+        ]
+      },
+      "nullable": [
+        false
+      ]
+    }
+  },
   "f78dac3d15be1ea0d0ed43a4beadc04ec00d8ba68be2bb68cbc3f2ebe5c93dbd": {
     "query": "\n            SELECT title, description, downloads,\n                   icon_url, body_url, published,\n                   updated, status,\n                   issues_url, source_url, wiki_url,\n                   team_id\n            FROM mods\n            WHERE id = $1\n            ",
     "describe": {

src/database/models/categories.rs

@@ -130,6 +130,7 @@ impl<'a> CategoryBuilder<'a> {
             "
             INSERT INTO categories (category)
             VALUES ($1)
+            ON CONFLICT (category) DO NOTHING
             RETURNING id
             ",
             self.name
@@ -255,6 +256,7 @@ impl<'a> LoaderBuilder<'a> {
             "
             INSERT INTO loaders (loader)
             VALUES ($1)
+            ON CONFLICT (loader) DO NOTHING
             RETURNING id
             ",
             self.name
@@ -266,13 +268,15 @@ impl<'a> LoaderBuilder<'a> {
     }
 }
 
+#[derive(Default)]
 pub struct GameVersionBuilder<'a> {
     pub version: Option<&'a str>,
+    pub version_type: Option<&'a str>,
 }
 
 impl GameVersion {
     pub fn builder() -> GameVersionBuilder<'static> {
-        GameVersionBuilder { version: None }
+        GameVersionBuilder::default()
     }
 
     pub async fn get_id<'a, E>(
@@ -302,7 +306,7 @@ impl GameVersion {
         Ok(result.map(|r| GameVersionId(r.id)))
     }
 
-    pub async fn get_name<'a, E>(id: VersionId, exec: E) -> Result<String, DatabaseError>
+    pub async fn get_name<'a, E>(id: GameVersionId, exec: E) -> Result<String, DatabaseError>
     where
         E: sqlx::Executor<'a, Database = sqlx::Postgres>,
     {
@@ -311,7 +315,7 @@ impl GameVersion {
             SELECT version FROM game_versions
             WHERE id = $1
             ",
-            id as VersionId
+            id as GameVersionId
         )
         .fetch_one(exec)
         .await?;
@@ -336,6 +340,25 @@ impl GameVersion {
         Ok(result)
     }
 
+    pub async fn list_type<'a, E>(version_type: &str, exec: E) -> Result<Vec<String>, DatabaseError>
+    where
+        E: sqlx::Executor<'a, Database = sqlx::Postgres>,
+    {
+        let result = sqlx::query!(
+            "
+            SELECT version FROM game_versions
+            WHERE type = $1
+            ",
+            version_type
+        )
+        .fetch_many(exec)
+        .try_filter_map(|e| async { Ok(e.right().map(|c| c.version)) })
+        .try_collect::<Vec<String>>()
+        .await?;
+
+        Ok(result)
+    }
+
     pub async fn remove<'a, E>(name: &str, exec: E) -> Result<Option<()>, DatabaseError>
     where
         E: sqlx::Executor<'a, Database = sqlx::Postgres>,
@@ -370,23 +393,44 @@ impl<'a> GameVersionBuilder<'a> {
         {
             Ok(Self {
                 version: Some(version),
+                ..self
             })
         } else {
             Err(DatabaseError::InvalidIdentifier(version.to_string()))
         }
     }
 
+    pub fn version_type(
+        self,
+        version_type: &'a str,
+    ) -> Result<GameVersionBuilder<'a>, DatabaseError> {
+        if version_type
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || "-_.".contains(c))
+        {
+            Ok(Self {
+                version_type: Some(version_type),
+                ..self
+            })
+        } else {
+            Err(DatabaseError::InvalidIdentifier(version_type.to_string()))
+        }
+    }
+
     pub async fn insert<'b, E>(self, exec: E) -> Result<GameVersionId, DatabaseError>
     where
         E: sqlx::Executor<'b, Database = sqlx::Postgres>,
     {
         let result = sqlx::query!(
             "
-            INSERT INTO game_versions (version)
+            INSERT INTO game_versions (version, type)
-            VALUES ($1)
+            VALUES ($1, $2)
+            ON CONFLICT (version) DO UPDATE
+                SET type = excluded.type
             RETURNING id
             ",
-            self.version
+            self.version,
+            self.version_type,
         )
         .fetch_one(exec)
         .await?;

src/main.rs

@@ -201,6 +201,8 @@ async fn main() -> std::io::Result<()> {
         });
     }
 
+    scheduler::schedule_versions(&mut scheduler, pool.clone(), skip_initial);
+
     let allowed_origins = dotenv::var("CORS_ORIGINS")
         .ok()
         .and_then(|s| serde_json::from_str::<Vec<String>>(&s).ok())

src/routes/mod_creation.rs

@@ -460,10 +460,10 @@ async fn mod_create_inner(
             status: status_id,
         };
 
-        let versions_list = mod_builder
+        let versions_list = mod_create_data
             .initial_versions
             .iter()
-            .flat_map(|v| v.game_versions.iter().map(|id| id.0.to_string()))
+            .flat_map(|v| v.game_versions.iter().map(|name| name.0.clone()))
             .collect::<std::collections::HashSet<String>>()
             .into_iter()
             .collect::<Vec<_>>();

src/routes/tags.rs

@@ -28,9 +28,6 @@ pub async fn category_list(pool: web::Data<PgPool>) -> Result<HttpResponse, ApiE
     Ok(HttpResponse::Ok().json(results))
 }
 
-// At some point this may take more info, but it should be able to
-// remain idempotent
-// TODO: don't fail if category already exists
 #[put("category/{name}")]
 pub async fn category_create(
     req: HttpRequest,
@@ -93,9 +90,6 @@ pub async fn loader_list(pool: web::Data<PgPool>) -> Result<HttpResponse, ApiErr
     Ok(HttpResponse::Ok().json(results))
 }
 
-// At some point this may take more info, but it should be able to
-// remain idempotent
-// TODO: don't fail if loader already exists
 #[put("loader/{name}")]
 pub async fn loader_create(
     req: HttpRequest,
@@ -152,19 +146,38 @@ pub async fn loader_delete(
     }
 }
 
-#[get("game_version")]
+#[derive(serde::Deserialize)]
-pub async fn game_version_list(pool: web::Data<PgPool>) -> Result<HttpResponse, ApiError> {
+pub struct GameVersionQueryData {
-    let results = GameVersion::list(&**pool).await?;
+    #[serde(rename = "type")]
-    Ok(HttpResponse::Ok().json(results))
+    type_: Option<String>,
+}
+
+#[get("game_version")]
+pub async fn game_version_list(
+    pool: web::Data<PgPool>,
+    query: web::Query<GameVersionQueryData>,
+) -> Result<HttpResponse, ApiError> {
+    if let Some(type_) = &query.type_ {
+        let results = GameVersion::list_type(type_, &**pool).await?;
+        Ok(HttpResponse::Ok().json(results))
+    } else {
+        let results = GameVersion::list(&**pool).await?;
+        Ok(HttpResponse::Ok().json(results))
+    }
+}
+
+#[derive(serde::Deserialize)]
+pub struct GameVersionData {
+    #[serde(rename = "type")]
+    type_: String,
 }
 
-// At some point this may take more info, but it should be able to
-// remain idempotent
 #[put("game_version/{name}")]
 pub async fn game_version_create(
     req: HttpRequest,
     pool: web::Data<PgPool>,
     game_version: web::Path<(String,)>,
+    version_data: web::Json<GameVersionData>,
 ) -> Result<HttpResponse, ApiError> {
     check_is_admin_from_headers(
         req.headers(),
@@ -178,8 +191,12 @@ pub async fn game_version_create(
 
     let name = game_version.into_inner().0;
 
+    // The version type currently isn't limited, but it should be one of:
+    // "release", "snapshot", "alpha", "beta", "other"
+
     let _id = GameVersion::builder()
         .version(&name)?
+        .version_type(&version_data.type_)?
         .insert(&**pool)
         .await?;
 

src/scheduler.rs

@@ -28,3 +28,106 @@ impl Drop for Scheduler {
         self.arbiter.stop();
     }
 }
+
+use log::{info, warn};
+
+pub fn schedule_versions(
+    scheduler: &mut Scheduler,
+    pool: sqlx::Pool<sqlx::Postgres>,
+    skip_initial: bool,
+) {
+    // Check mojang's versions every 6 hours
+    let version_index_interval = std::time::Duration::from_secs(60 * 60 * 6);
+
+    let mut skip = skip_initial;
+    scheduler.run(version_index_interval, move || {
+        let pool_ref = pool.clone();
+        let local_skip = skip;
+        if skip {
+            skip = false;
+        }
+        async move {
+            if local_skip {
+                return;
+            }
+            info!("Indexing game versions list from Mojang");
+            let result = update_versions(&pool_ref).await;
+            if let Err(e) = result {
+                warn!("Version update failed: {}", e);
+            }
+            info!("Done indexing game versions");
+        }
+    });
+}
+
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum VersionIndexingError {
+    #[error("Network error while updating game versions list: {0}")]
+    NetworkError(#[from] reqwest::Error),
+    #[error("Database error while updating game versions list: {0}")]
+    DatabaseError(#[from] crate::database::models::DatabaseError),
+}
+
+use serde::Deserialize;
+
+#[derive(Deserialize)]
+struct InputFormat<'a> {
+    // latest: LatestFormat,
+    versions: Vec<VersionFormat<'a>>,
+}
+#[derive(Deserialize)]
+struct VersionFormat<'a> {
+    id: String,
+    #[serde(rename = "type")]
+    type_: std::borrow::Cow<'a, str>,
+}
+
+async fn update_versions(pool: &sqlx::Pool<sqlx::Postgres>) -> Result<(), VersionIndexingError> {
+    let input = reqwest::get("https://launchermeta.mojang.com/mc/game/version_manifest.json")
+        .await?
+        .json::<InputFormat>()
+        .await?;
+
+    let mut skipped_versions_count = 0u32;
+
+    for version in input.versions.into_iter() {
+        let name = version.id;
+        if !name
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || "-_.".contains(c))
+        {
+            // We'll deal with these manually
+            skipped_versions_count += 1;
+            continue;
+        }
+
+        let type_ = match &*version.type_ {
+            "release" => "release",
+            "snapshot" => "snapshot",
+            "old_alpha" => "alpha",
+            "old_beta" => "beta",
+            _ => "other",
+        };
+
+        crate::database::models::categories::GameVersion::builder()
+            .version(&name)?
+            .version_type(type_)?
+            .insert(pool)
+            .await?;
+    }
+
+    if skipped_versions_count > 0 {
+        // This will currently always trigger due to 1.14 pre releases
+        // and the shareware april fools update. We could set a threshold
+        // that accounts for those versions and update it whenever we
+        // manually fix another version.
+        warn!(
+            "Skipped {} game versions; check for new versions and add them manually",
+            skipped_versions_count
+        );
+    }
+
+    Ok(())
+}