Game Version types, indexing, and bugfixes (#91)

* Add types to game_versions, allow filtering by version type

- Fixes an issue with version numbers in the initial mod indexing
  queue
- Modifies the /api/v1/categories/game_versions route to take an
  optional query parameter `type` to filter the listed game versions
- Creating tags is now idempotent
- Creating game_versions now requires a JSON body that specifies
  the version type

* Implement automatic indexing of new Minecraft versions

It's currently set to run every 6 hours and isn't configurable; we
could add config for it, but it doesn't seem likely to be rate
limited or have issues with frequency.

src/database/models/categories.rs

@@ -130,6 +130,7 @@ impl<'a> CategoryBuilder<'a> {
             "
             INSERT INTO categories (category)
             VALUES ($1)
+            ON CONFLICT (category) DO NOTHING
             RETURNING id
             ",
             self.name
@@ -255,6 +256,7 @@ impl<'a> LoaderBuilder<'a> {
             "
             INSERT INTO loaders (loader)
             VALUES ($1)
+            ON CONFLICT (loader) DO NOTHING
             RETURNING id
             ",
             self.name
@@ -266,13 +268,15 @@ impl<'a> LoaderBuilder<'a> {
     }
 }
 
+#[derive(Default)]
 pub struct GameVersionBuilder<'a> {
     pub version: Option<&'a str>,
+    pub version_type: Option<&'a str>,
 }
 
 impl GameVersion {
     pub fn builder() -> GameVersionBuilder<'static> {
-        GameVersionBuilder { version: None }
+        GameVersionBuilder::default()
     }
 
     pub async fn get_id<'a, E>(
@@ -302,7 +306,7 @@ impl GameVersion {
         Ok(result.map(|r| GameVersionId(r.id)))
     }
 
-    pub async fn get_name<'a, E>(id: VersionId, exec: E) -> Result<String, DatabaseError>
+    pub async fn get_name<'a, E>(id: GameVersionId, exec: E) -> Result<String, DatabaseError>
     where
         E: sqlx::Executor<'a, Database = sqlx::Postgres>,
     {
@@ -311,7 +315,7 @@ impl GameVersion {
             SELECT version FROM game_versions
             WHERE id = $1
             ",
-            id as VersionId
+            id as GameVersionId
         )
         .fetch_one(exec)
         .await?;
@@ -336,6 +340,25 @@ impl GameVersion {
         Ok(result)
     }
 
+    pub async fn list_type<'a, E>(version_type: &str, exec: E) -> Result<Vec<String>, DatabaseError>
+    where
+        E: sqlx::Executor<'a, Database = sqlx::Postgres>,
+    {
+        let result = sqlx::query!(
+            "
+            SELECT version FROM game_versions
+            WHERE type = $1
+            ",
+            version_type
+        )
+        .fetch_many(exec)
+        .try_filter_map(|e| async { Ok(e.right().map(|c| c.version)) })
+        .try_collect::<Vec<String>>()
+        .await?;
+
+        Ok(result)
+    }
+
     pub async fn remove<'a, E>(name: &str, exec: E) -> Result<Option<()>, DatabaseError>
     where
         E: sqlx::Executor<'a, Database = sqlx::Postgres>,
@@ -370,23 +393,44 @@ impl<'a> GameVersionBuilder<'a> {
         {
             Ok(Self {
                 version: Some(version),
+                ..self
             })
         } else {
             Err(DatabaseError::InvalidIdentifier(version.to_string()))
         }
     }
 
+    pub fn version_type(
+        self,
+        version_type: &'a str,
+    ) -> Result<GameVersionBuilder<'a>, DatabaseError> {
+        if version_type
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || "-_.".contains(c))
+        {
+            Ok(Self {
+                version_type: Some(version_type),
+                ..self
+            })
+        } else {
+            Err(DatabaseError::InvalidIdentifier(version_type.to_string()))
+        }
+    }
+
     pub async fn insert<'b, E>(self, exec: E) -> Result<GameVersionId, DatabaseError>
     where
         E: sqlx::Executor<'b, Database = sqlx::Postgres>,
     {
         let result = sqlx::query!(
             "
-            INSERT INTO game_versions (version)
-            VALUES ($1)
+            INSERT INTO game_versions (version, type)
+            VALUES ($1, $2)
+            ON CONFLICT (version) DO UPDATE
+                SET type = excluded.type
             RETURNING id
             ",
-            self.version
+            self.version,
+            self.version_type,
         )
         .fetch_one(exec)
         .await?;

src/main.rs

@@ -201,6 +201,8 @@ async fn main() -> std::io::Result<()> {
         });
     }
 
+    scheduler::schedule_versions(&mut scheduler, pool.clone(), skip_initial);
+
     let allowed_origins = dotenv::var("CORS_ORIGINS")
         .ok()
         .and_then(|s| serde_json::from_str::<Vec<String>>(&s).ok())

src/routes/mod_creation.rs

@@ -460,10 +460,10 @@ async fn mod_create_inner(
             status: status_id,
         };
 
-        let versions_list = mod_builder
+        let versions_list = mod_create_data
             .initial_versions
             .iter()
-            .flat_map(|v| v.game_versions.iter().map(|id| id.0.to_string()))
+            .flat_map(|v| v.game_versions.iter().map(|name| name.0.clone()))
             .collect::<std::collections::HashSet<String>>()
             .into_iter()
             .collect::<Vec<_>>();

src/routes/tags.rs

@@ -28,9 +28,6 @@ pub async fn category_list(pool: web::Data<PgPool>) -> Result<HttpResponse, ApiE
     Ok(HttpResponse::Ok().json(results))
 }
 
-// At some point this may take more info, but it should be able to
-// remain idempotent
-// TODO: don't fail if category already exists
 #[put("category/{name}")]
 pub async fn category_create(
     req: HttpRequest,
@@ -93,9 +90,6 @@ pub async fn loader_list(pool: web::Data<PgPool>) -> Result<HttpResponse, ApiErr
     Ok(HttpResponse::Ok().json(results))
 }
 
-// At some point this may take more info, but it should be able to
-// remain idempotent
-// TODO: don't fail if loader already exists
 #[put("loader/{name}")]
 pub async fn loader_create(
     req: HttpRequest,
@@ -152,19 +146,38 @@ pub async fn loader_delete(
     }
 }
 
-#[get("game_version")]
-pub async fn game_version_list(pool: web::Data<PgPool>) -> Result<HttpResponse, ApiError> {
-    let results = GameVersion::list(&**pool).await?;
-    Ok(HttpResponse::Ok().json(results))
+#[derive(serde::Deserialize)]
+pub struct GameVersionQueryData {
+    #[serde(rename = "type")]
+    type_: Option<String>,
+}
+
+#[get("game_version")]
+pub async fn game_version_list(
+    pool: web::Data<PgPool>,
+    query: web::Query<GameVersionQueryData>,
+) -> Result<HttpResponse, ApiError> {
+    if let Some(type_) = &query.type_ {
+        let results = GameVersion::list_type(type_, &**pool).await?;
+        Ok(HttpResponse::Ok().json(results))
+    } else {
+        let results = GameVersion::list(&**pool).await?;
+        Ok(HttpResponse::Ok().json(results))
+    }
+}
+
+#[derive(serde::Deserialize)]
+pub struct GameVersionData {
+    #[serde(rename = "type")]
+    type_: String,
 }
 
-// At some point this may take more info, but it should be able to
-// remain idempotent
 #[put("game_version/{name}")]
 pub async fn game_version_create(
     req: HttpRequest,
     pool: web::Data<PgPool>,
     game_version: web::Path<(String,)>,
+    version_data: web::Json<GameVersionData>,
 ) -> Result<HttpResponse, ApiError> {
     check_is_admin_from_headers(
         req.headers(),
@@ -178,8 +191,12 @@ pub async fn game_version_create(
 
     let name = game_version.into_inner().0;
 
+    // The version type currently isn't limited, but it should be one of:
+    // "release", "snapshot", "alpha", "beta", "other"
+
     let _id = GameVersion::builder()
         .version(&name)?
+        .version_type(&version_data.type_)?
         .insert(&**pool)
         .await?;
 

src/scheduler.rs

@@ -28,3 +28,106 @@ impl Drop for Scheduler {
         self.arbiter.stop();
     }
 }
+
+use log::{info, warn};
+
+pub fn schedule_versions(
+    scheduler: &mut Scheduler,
+    pool: sqlx::Pool<sqlx::Postgres>,
+    skip_initial: bool,
+) {
+    // Check mojang's versions every 6 hours
+    let version_index_interval = std::time::Duration::from_secs(60 * 60 * 6);
+
+    let mut skip = skip_initial;
+    scheduler.run(version_index_interval, move || {
+        let pool_ref = pool.clone();
+        let local_skip = skip;
+        if skip {
+            skip = false;
+        }
+        async move {
+            if local_skip {
+                return;
+            }
+            info!("Indexing game versions list from Mojang");
+            let result = update_versions(&pool_ref).await;
+            if let Err(e) = result {
+                warn!("Version update failed: {}", e);
+            }
+            info!("Done indexing game versions");
+        }
+    });
+}
+
+use thiserror::Error;
+
+#[derive(Error, Debug)]
+pub enum VersionIndexingError {
+    #[error("Network error while updating game versions list: {0}")]
+    NetworkError(#[from] reqwest::Error),
+    #[error("Database error while updating game versions list: {0}")]
+    DatabaseError(#[from] crate::database::models::DatabaseError),
+}
+
+use serde::Deserialize;
+
+#[derive(Deserialize)]
+struct InputFormat<'a> {
+    // latest: LatestFormat,
+    versions: Vec<VersionFormat<'a>>,
+}
+#[derive(Deserialize)]
+struct VersionFormat<'a> {
+    id: String,
+    #[serde(rename = "type")]
+    type_: std::borrow::Cow<'a, str>,
+}
+
+async fn update_versions(pool: &sqlx::Pool<sqlx::Postgres>) -> Result<(), VersionIndexingError> {
+    let input = reqwest::get("https://launchermeta.mojang.com/mc/game/version_manifest.json")
+        .await?
+        .json::<InputFormat>()
+        .await?;
+
+    let mut skipped_versions_count = 0u32;
+
+    for version in input.versions.into_iter() {
+        let name = version.id;
+        if !name
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || "-_.".contains(c))
+        {
+            // We'll deal with these manually
+            skipped_versions_count += 1;
+            continue;
+        }
+
+        let type_ = match &*version.type_ {
+            "release" => "release",
+            "snapshot" => "snapshot",
+            "old_alpha" => "alpha",
+            "old_beta" => "beta",
+            _ => "other",
+        };
+
+        crate::database::models::categories::GameVersion::builder()
+            .version(&name)?
+            .version_type(type_)?
+            .insert(pool)
+            .await?;
+    }
+
+    if skipped_versions_count > 0 {
+        // This will currently always trigger due to 1.14 pre releases
+        // and the shareware april fools update. We could set a threshold
+        // that accounts for those versions and update it whenever we
+        // manually fix another version.
+        warn!(
+            "Skipped {} game versions; check for new versions and add them manually",
+            skipped_versions_count
+        );
+    }
+
+    Ok(())
+}