diff --git a/migrations/20201021214908_extend-game-version.sql b/migrations/20201021214908_extend-game-version.sql new file mode 100644 index 000000000..d40cb420b --- /dev/null +++ b/migrations/20201021214908_extend-game-version.sql @@ -0,0 +1,3 @@ + +ALTER TABLE game_versions +ADD COLUMN type varchar(16) NOT NULL DEFAULT 'other'; diff --git a/sqlx-data.json b/sqlx-data.json index 099a40018..00a6ee0c3 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -278,26 +278,6 @@ ] } }, - "1b74bdb59773ffd2a78a56e4d920bb83c322e180e6174c741d4bb722c353de43": { - "query": "\n INSERT INTO loaders (loader)\n VALUES ($1)\n RETURNING id\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Varchar" - ] - }, - "nullable": [ - false - ] - } - }, "1c7b0eb4341af5a7942e52f632cf582561f10b4b6a41a082fb8a60f04ac17c6e": { "query": "SELECT EXISTS(SELECT 1 FROM states WHERE id=$1)", "describe": { @@ -499,6 +479,27 @@ "nullable": [] } }, + "3d18702f07161c0cdbc31d70b89ffeb3678617ccc44dfc6fb03dd63f47226c7b": { + "query": "\n INSERT INTO game_versions (version, type)\n VALUES ($1, $2)\n ON CONFLICT (version) DO UPDATE\n SET type = excluded.type\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar", + "Varchar" + ] + }, + "nullable": [ + false + ] + } + }, "42e072309779598d0c213280dd8052d1b4889cb24ef5204ca13b74f693b94328": { "query": "\n SELECT user_id FROM team_members tm\n INNER JOIN mods ON mods.team_id = tm.team_id\n WHERE mods.id = $1\n ", "describe": { @@ -554,6 +555,26 @@ "nullable": [] } }, + "49e36828e3a0214b48234435e34311735ae32e08d8be1270f8f0db4b27e708ba": { + "query": "\n INSERT INTO loaders (loader)\n VALUES ($1)\n ON CONFLICT (loader) DO NOTHING\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [ + false + ] + } + }, "4c99c0840159d18e88cd6094a41117258f2337346c145d926b5b610c76b5125f": { "query": "\n SELECT c.category\n FROM mods_categories mc\n INNER JOIN categories c ON mc.joining_category_id=c.id\n WHERE mc.joining_mod_id = $1\n ", "describe": { @@ -620,26 +641,6 @@ "nullable": [] } }, - "56cb9274e92f185dee3accf69cca2e34c035efbef908baefeb60548fb14e02bd": { - "query": "\n INSERT INTO categories (category)\n VALUES ($1)\n RETURNING id\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Varchar" - ] - }, - "nullable": [ - false - ] - } - }, "59cf9d085593887595ea45246291f2cd64fc6677d551e96bdb60c09ff1eebf99": { "query": "\n SELECT files.id, files.url, files.filename FROM files\n WHERE files.version_id = $1\n ", "describe": { @@ -1224,26 +1225,6 @@ "nullable": [] } }, - "bee1e8b7f3588c6b0534443775f3d0d66d960e96a5ae8422aa96a69238f375a4": { - "query": "\n INSERT INTO game_versions (version)\n VALUES ($1)\n RETURNING id\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int4" - } - ], - "parameters": { - "Left": [ - "Varchar" - ] - }, - "nullable": [ - false - ] - } - }, "bf7f721664f5e0ed41adc41b5483037256635f28ff6c4e5d3cbcec4387f9c8ef": { "query": "SELECT EXISTS(SELECT 1 FROM users WHERE id=$1)", "describe": { @@ -1735,6 +1716,26 @@ ] } }, + "ec4a3ef12a35bb78002fdafccbdb198b15f9a0fdb2b3e4108f9081b7e56e8769": { + "query": "\n SELECT version FROM game_versions\n WHERE type = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "version", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false + ] + } + }, "f0db9d8606ccc2196a9cfafe0e7090dab42bf790f25e0469b8947fac1cf043d5": { "query": "\n SELECT version FROM game_versions\n WHERE id = $1\n ", "describe": { @@ -1775,6 +1776,26 @@ ] } }, + "f12ae54acf02e06e9b8774e8c2ea95058a78f6d724645adcd02f9dea6538024f": { + "query": "\n INSERT INTO categories (category)\n VALUES ($1)\n ON CONFLICT (category) DO NOTHING\n RETURNING id\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + } + ], + "parameters": { + "Left": [ + "Varchar" + ] + }, + "nullable": [ + false + ] + } + }, "f78dac3d15be1ea0d0ed43a4beadc04ec00d8ba68be2bb68cbc3f2ebe5c93dbd": { "query": "\n SELECT title, description, downloads,\n icon_url, body_url, published,\n updated, status,\n issues_url, source_url, wiki_url,\n team_id\n FROM mods\n WHERE id = $1\n ", "describe": { diff --git a/src/database/models/categories.rs b/src/database/models/categories.rs index 9eac9b61c..ea8f9776f 100644 --- a/src/database/models/categories.rs +++ b/src/database/models/categories.rs @@ -130,6 +130,7 @@ impl<'a> CategoryBuilder<'a> { " INSERT INTO categories (category) VALUES ($1) + ON CONFLICT (category) DO NOTHING RETURNING id ", self.name @@ -255,6 +256,7 @@ impl<'a> LoaderBuilder<'a> { " INSERT INTO loaders (loader) VALUES ($1) + ON CONFLICT (loader) DO NOTHING RETURNING id ", self.name @@ -266,13 +268,15 @@ impl<'a> LoaderBuilder<'a> { } } +#[derive(Default)] pub struct GameVersionBuilder<'a> { pub version: Option<&'a str>, + pub version_type: Option<&'a str>, } impl GameVersion { pub fn builder() -> GameVersionBuilder<'static> { - GameVersionBuilder { version: None } + GameVersionBuilder::default() } pub async fn get_id<'a, E>( @@ -302,7 +306,7 @@ impl GameVersion { Ok(result.map(|r| GameVersionId(r.id))) } - pub async fn get_name<'a, E>(id: VersionId, exec: E) -> Result + pub async fn get_name<'a, E>(id: GameVersionId, exec: E) -> Result where E: sqlx::Executor<'a, Database = sqlx::Postgres>, { @@ -311,7 +315,7 @@ impl GameVersion { SELECT version FROM game_versions WHERE id = $1 ", - id as VersionId + id as GameVersionId ) .fetch_one(exec) .await?; @@ -336,6 +340,25 @@ impl GameVersion { Ok(result) } + pub async fn list_type<'a, E>(version_type: &str, exec: E) -> Result, DatabaseError> + where + E: sqlx::Executor<'a, Database = sqlx::Postgres>, + { + let result = sqlx::query!( + " + SELECT version FROM game_versions + WHERE type = $1 + ", + version_type + ) + .fetch_many(exec) + .try_filter_map(|e| async { Ok(e.right().map(|c| c.version)) }) + .try_collect::>() + .await?; + + Ok(result) + } + pub async fn remove<'a, E>(name: &str, exec: E) -> Result, DatabaseError> where E: sqlx::Executor<'a, Database = sqlx::Postgres>, @@ -370,23 +393,44 @@ impl<'a> GameVersionBuilder<'a> { { Ok(Self { version: Some(version), + ..self }) } else { Err(DatabaseError::InvalidIdentifier(version.to_string())) } } + pub fn version_type( + self, + version_type: &'a str, + ) -> Result, DatabaseError> { + if version_type + .chars() + .all(|c| c.is_ascii_alphanumeric() || "-_.".contains(c)) + { + Ok(Self { + version_type: Some(version_type), + ..self + }) + } else { + Err(DatabaseError::InvalidIdentifier(version_type.to_string())) + } + } + pub async fn insert<'b, E>(self, exec: E) -> Result where E: sqlx::Executor<'b, Database = sqlx::Postgres>, { let result = sqlx::query!( " - INSERT INTO game_versions (version) - VALUES ($1) + INSERT INTO game_versions (version, type) + VALUES ($1, $2) + ON CONFLICT (version) DO UPDATE + SET type = excluded.type RETURNING id ", - self.version + self.version, + self.version_type, ) .fetch_one(exec) .await?; diff --git a/src/main.rs b/src/main.rs index 4c7881d1c..0c2d0cbcf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -201,6 +201,8 @@ async fn main() -> std::io::Result<()> { }); } + scheduler::schedule_versions(&mut scheduler, pool.clone(), skip_initial); + let allowed_origins = dotenv::var("CORS_ORIGINS") .ok() .and_then(|s| serde_json::from_str::>(&s).ok()) diff --git a/src/routes/mod_creation.rs b/src/routes/mod_creation.rs index b08b05d9c..e08169445 100644 --- a/src/routes/mod_creation.rs +++ b/src/routes/mod_creation.rs @@ -460,10 +460,10 @@ async fn mod_create_inner( status: status_id, }; - let versions_list = mod_builder + let versions_list = mod_create_data .initial_versions .iter() - .flat_map(|v| v.game_versions.iter().map(|id| id.0.to_string())) + .flat_map(|v| v.game_versions.iter().map(|name| name.0.clone())) .collect::>() .into_iter() .collect::>(); diff --git a/src/routes/tags.rs b/src/routes/tags.rs index 8a6e75bea..a79c16f93 100644 --- a/src/routes/tags.rs +++ b/src/routes/tags.rs @@ -28,9 +28,6 @@ pub async fn category_list(pool: web::Data) -> Result) -> Result) -> Result { - let results = GameVersion::list(&**pool).await?; - Ok(HttpResponse::Ok().json(results)) +#[derive(serde::Deserialize)] +pub struct GameVersionQueryData { + #[serde(rename = "type")] + type_: Option, +} + +#[get("game_version")] +pub async fn game_version_list( + pool: web::Data, + query: web::Query, +) -> Result { + if let Some(type_) = &query.type_ { + let results = GameVersion::list_type(type_, &**pool).await?; + Ok(HttpResponse::Ok().json(results)) + } else { + let results = GameVersion::list(&**pool).await?; + Ok(HttpResponse::Ok().json(results)) + } +} + +#[derive(serde::Deserialize)] +pub struct GameVersionData { + #[serde(rename = "type")] + type_: String, } -// At some point this may take more info, but it should be able to -// remain idempotent #[put("game_version/{name}")] pub async fn game_version_create( req: HttpRequest, pool: web::Data, game_version: web::Path<(String,)>, + version_data: web::Json, ) -> Result { check_is_admin_from_headers( req.headers(), @@ -178,8 +191,12 @@ pub async fn game_version_create( let name = game_version.into_inner().0; + // The version type currently isn't limited, but it should be one of: + // "release", "snapshot", "alpha", "beta", "other" + let _id = GameVersion::builder() .version(&name)? + .version_type(&version_data.type_)? .insert(&**pool) .await?; diff --git a/src/scheduler.rs b/src/scheduler.rs index 5714eadf9..01999b5b2 100644 --- a/src/scheduler.rs +++ b/src/scheduler.rs @@ -28,3 +28,106 @@ impl Drop for Scheduler { self.arbiter.stop(); } } + +use log::{info, warn}; + +pub fn schedule_versions( + scheduler: &mut Scheduler, + pool: sqlx::Pool, + skip_initial: bool, +) { + // Check mojang's versions every 6 hours + let version_index_interval = std::time::Duration::from_secs(60 * 60 * 6); + + let mut skip = skip_initial; + scheduler.run(version_index_interval, move || { + let pool_ref = pool.clone(); + let local_skip = skip; + if skip { + skip = false; + } + async move { + if local_skip { + return; + } + info!("Indexing game versions list from Mojang"); + let result = update_versions(&pool_ref).await; + if let Err(e) = result { + warn!("Version update failed: {}", e); + } + info!("Done indexing game versions"); + } + }); +} + +use thiserror::Error; + +#[derive(Error, Debug)] +pub enum VersionIndexingError { + #[error("Network error while updating game versions list: {0}")] + NetworkError(#[from] reqwest::Error), + #[error("Database error while updating game versions list: {0}")] + DatabaseError(#[from] crate::database::models::DatabaseError), +} + +use serde::Deserialize; + +#[derive(Deserialize)] +struct InputFormat<'a> { + // latest: LatestFormat, + versions: Vec>, +} +#[derive(Deserialize)] +struct VersionFormat<'a> { + id: String, + #[serde(rename = "type")] + type_: std::borrow::Cow<'a, str>, +} + +async fn update_versions(pool: &sqlx::Pool) -> Result<(), VersionIndexingError> { + let input = reqwest::get("https://launchermeta.mojang.com/mc/game/version_manifest.json") + .await? + .json::() + .await?; + + let mut skipped_versions_count = 0u32; + + for version in input.versions.into_iter() { + let name = version.id; + if !name + .chars() + .all(|c| c.is_ascii_alphanumeric() || "-_.".contains(c)) + { + // We'll deal with these manually + skipped_versions_count += 1; + continue; + } + + let type_ = match &*version.type_ { + "release" => "release", + "snapshot" => "snapshot", + "old_alpha" => "alpha", + "old_beta" => "beta", + _ => "other", + }; + + crate::database::models::categories::GameVersion::builder() + .version(&name)? + .version_type(type_)? + .insert(pool) + .await?; + } + + if skipped_versions_count > 0 { + // This will currently always trigger due to 1.14 pre releases + // and the shareware april fools update. We could set a threshold + // that accounts for those versions and update it whenever we + // manually fix another version. + warn!( + "Skipped {} game versions; check for new versions and add them manually", + skipped_versions_count + ); + } + + Ok(()) +}