tomasalias/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Game Version types, indexing, and bugfixes (#91)

Browse Source 
* Add types to game_versions, allow filtering by version type

- Fixes an issue with version numbers in the initial mod indexing
  queue
- Modifies the /api/v1/categories/game_versions route to take an
  optional query parameter `type` to filter the listed game versions
- Creating tags is now idempotent
- Creating game_versions now requires a JSON body that specifies
  the version type

* Implement automatic indexing of new Minecraft versions

It's currently set to run every 6 hours and isn't configurable; we
could add config for it, but it doesn't seem likely to be rate
limited or have issues with frequency.
This commit is contained in:
Aeledfyr
2020-10-28 11:11:49 -05:00
committed by GitHub
parent ef28459b61
commit a4ba6d1444
7 changed files with 270 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
migrations/20201021214908_extend-game-version.sql Normal file
View File

@@ -0,0 +1,3 @@
ALTER TABLE game_versions
ADD COLUMN type varchar(16) NOT NULL DEFAULT 'other';

141
sqlx-data.json
View File

@@ -278,26 +278,6 @@
]
}
},
"1b74bdb59773ffd2a78a56e4d920bb83c322e180e6174c741d4bb722c353de43": {
"query": "\n INSERT INTO loaders (loader)\n VALUES ($1)\n RETURNING id\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Varchar"
]
},
"nullable": [
false
]
}
},
"1c7b0eb4341af5a7942e52f632cf582561f10b4b6a41a082fb8a60f04ac17c6e": {
"query": "SELECT EXISTS(SELECT 1 FROM states WHERE id=$1)",
"describe": {
@@ -499,6 +479,27 @@
"nullable": []
}
},
"3d18702f07161c0cdbc31d70b89ffeb3678617ccc44dfc6fb03dd63f47226c7b": {
"query": "\n INSERT INTO game_versions (version, type)\n VALUES ($1, $2)\n ON CONFLICT (version) DO UPDATE\n SET type = excluded.type\n RETURNING id\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Varchar",
"Varchar"
]
},
"nullable": [
false
]
}
},
"42e072309779598d0c213280dd8052d1b4889cb24ef5204ca13b74f693b94328": {
"query": "\n SELECT user_id FROM team_members tm\n INNER JOIN mods ON mods.team_id = tm.team_id\n WHERE mods.id = $1\n ",
"describe": {
@@ -554,6 +555,26 @@
"nullable": []
}
},
"49e36828e3a0214b48234435e34311735ae32e08d8be1270f8f0db4b27e708ba": {
"query": "\n INSERT INTO loaders (loader)\n VALUES ($1)\n ON CONFLICT (loader) DO NOTHING\n RETURNING id\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Varchar"
]
},
"nullable": [
false
]
}
},
"4c99c0840159d18e88cd6094a41117258f2337346c145d926b5b610c76b5125f": {
"query": "\n SELECT c.category\n FROM mods_categories mc\n INNER JOIN categories c ON mc.joining_category_id=c.id\n WHERE mc.joining_mod_id = $1\n ",
"describe": {
@@ -620,26 +641,6 @@
"nullable": []
}
},
"56cb9274e92f185dee3accf69cca2e34c035efbef908baefeb60548fb14e02bd": {
"query": "\n INSERT INTO categories (category)\n VALUES ($1)\n RETURNING id\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Varchar"
]
},
"nullable": [
false
]
}
},
"59cf9d085593887595ea45246291f2cd64fc6677d551e96bdb60c09ff1eebf99": {
"query": "\n SELECT files.id, files.url, files.filename FROM files\n WHERE files.version_id = $1\n ",
"describe": {
@@ -1224,26 +1225,6 @@
"nullable": []
}
},
"bee1e8b7f3588c6b0534443775f3d0d66d960e96a5ae8422aa96a69238f375a4": {
"query": "\n INSERT INTO game_versions (version)\n VALUES ($1)\n RETURNING id\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Varchar"
]
},
"nullable": [
false
]
}
},
"bf7f721664f5e0ed41adc41b5483037256635f28ff6c4e5d3cbcec4387f9c8ef": {
"query": "SELECT EXISTS(SELECT 1 FROM users WHERE id=$1)",
"describe": {
@@ -1735,6 +1716,26 @@
]
}
},
"ec4a3ef12a35bb78002fdafccbdb198b15f9a0fdb2b3e4108f9081b7e56e8769": {
"query": "\n SELECT version FROM game_versions\n WHERE type = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "version",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text"
]
},
"nullable": [
false
]
}
},
"f0db9d8606ccc2196a9cfafe0e7090dab42bf790f25e0469b8947fac1cf043d5": {
"query": "\n SELECT version FROM game_versions\n WHERE id = $1\n ",
"describe": {
@@ -1775,6 +1776,26 @@
]
}
},
"f12ae54acf02e06e9b8774e8c2ea95058a78f6d724645adcd02f9dea6538024f": {
"query": "\n INSERT INTO categories (category)\n VALUES ($1)\n ON CONFLICT (category) DO NOTHING\n RETURNING id\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int4"
}
],
"parameters": {
"Left": [
"Varchar"
]
},
"nullable": [
false
]
}
},
"f78dac3d15be1ea0d0ed43a4beadc04ec00d8ba68be2bb68cbc3f2ebe5c93dbd": {
"query": "\n SELECT title, description, downloads,\n icon_url, body_url, published,\n updated, status,\n issues_url, source_url, wiki_url,\n team_id\n FROM mods\n WHERE id = $1\n ",
"describe": {

56
src/database/models/categories.rs
View File

@@ -130,6 +130,7 @@ impl<'a> CategoryBuilder<'a> {
"
INSERT INTO categories (category)
VALUES ($1)
ON CONFLICT (category) DO NOTHING
RETURNING id
",
self.name
@@ -255,6 +256,7 @@ impl<'a> LoaderBuilder<'a> {
"
INSERT INTO loaders (loader)
VALUES ($1)
ON CONFLICT (loader) DO NOTHING
RETURNING id
",
self.name
@@ -266,13 +268,15 @@ impl<'a> LoaderBuilder<'a> {
}
}
#[derive(Default)]
pub struct GameVersionBuilder<'a> {
pub version: Option<&'a str>,
pub version_type: Option<&'a str>,
}
impl GameVersion {
pub fn builder() -> GameVersionBuilder<'static> {
GameVersionBuilder { version: None }
GameVersionBuilder::default()
}
pub async fn get_id<'a, E>(
@@ -302,7 +306,7 @@ impl GameVersion {
Ok(result.map(|r| GameVersionId(r.id)))
}
pub async fn get_name<'a, E>(id: VersionId, exec: E) -> Result<String, DatabaseError>
pub async fn get_name<'a, E>(id: GameVersionId, exec: E) -> Result<String, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
@@ -311,7 +315,7 @@ impl GameVersion {
SELECT version FROM game_versions
WHERE id = $1
",
id as VersionId
id as GameVersionId
)
.fetch_one(exec)
.await?;
@@ -336,6 +340,25 @@ impl GameVersion {
Ok(result)
}
pub async fn list_type<'a, E>(version_type: &str, exec: E) -> Result<Vec<String>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let result = sqlx::query!(
"
SELECT version FROM game_versions
WHERE type = $1
",
version_type
)
.fetch_many(exec)
.try_filter_map(|e| async { Ok(e.right().map(|c| c.version)) })
.try_collect::<Vec<String>>()
.await?;
Ok(result)
}
pub async fn remove<'a, E>(name: &str, exec: E) -> Result<Option<()>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
@@ -370,23 +393,44 @@ impl<'a> GameVersionBuilder<'a> {
{
Ok(Self {
version: Some(version),
..self
})
} else {
Err(DatabaseError::InvalidIdentifier(version.to_string()))
}
}
pub fn version_type(
self,
version_type: &'a str,
) -> Result<GameVersionBuilder<'a>, DatabaseError> {
if version_type
.chars()
.all(|c| c.is_ascii_alphanumeric() || "-_.".contains(c))
{
Ok(Self {
version_type: Some(version_type),
..self
})
} else {
Err(DatabaseError::InvalidIdentifier(version_type.to_string()))
}
}
pub async fn insert<'b, E>(self, exec: E) -> Result<GameVersionId, DatabaseError>
where
E: sqlx::Executor<'b, Database = sqlx::Postgres>,
{
let result = sqlx::query!(
"
INSERT INTO game_versions (version)
VALUES ($1)
INSERT INTO game_versions (version, type)
VALUES ($1, $2)
ON CONFLICT (version) DO UPDATE
SET type = excluded.type
RETURNING id
",
self.version
self.version,
self.version_type,
)
.fetch_one(exec)
.await?;

2
src/main.rs
View File

@@ -201,6 +201,8 @@ async fn main() -> std::io::Result<()> {
});
}
scheduler::schedule_versions(&mut scheduler, pool.clone(), skip_initial);
let allowed_origins = dotenv::var("CORS_ORIGINS")
.ok()
.and_then(|s| serde_json::from_str::<Vec<String>>(&s).ok())

4
src/routes/mod_creation.rs
View File

@@ -460,10 +460,10 @@ async fn mod_create_inner(
status: status_id,
};
let versions_list = mod_builder
let versions_list = mod_create_data
.initial_versions
.iter()
.flat_map(|v| v.game_versions.iter().map(|id| id.0.to_string()))
.flat_map(|v| v.game_versions.iter().map(|name| name.0.clone()))
.collect::<std::collections::HashSet<String>>()
.into_iter()
.collect::<Vec<_>>();

41
src/routes/tags.rs
View File

@@ -28,9 +28,6 @@ pub async fn category_list(pool: web::Data<PgPool>) -> Result<HttpResponse, ApiE
Ok(HttpResponse::Ok().json(results))
}
// At some point this may take more info, but it should be able to
// remain idempotent
// TODO: don't fail if category already exists
#[put("category/{name}")]
pub async fn category_create(
req: HttpRequest,
@@ -93,9 +90,6 @@ pub async fn loader_list(pool: web::Data<PgPool>) -> Result<HttpResponse, ApiErr
Ok(HttpResponse::Ok().json(results))
}
// At some point this may take more info, but it should be able to
// remain idempotent
// TODO: don't fail if loader already exists
#[put("loader/{name}")]
pub async fn loader_create(
req: HttpRequest,
@@ -152,19 +146,38 @@ pub async fn loader_delete(
}
}
#[get("game_version")]
pub async fn game_version_list(pool: web::Data<PgPool>) -> Result<HttpResponse, ApiError> {
let results = GameVersion::list(&**pool).await?;
Ok(HttpResponse::Ok().json(results))
#[derive(serde::Deserialize)]
pub struct GameVersionQueryData {
#[serde(rename = "type")]
type_: Option<String>,
}
#[get("game_version")]
pub async fn game_version_list(
pool: web::Data<PgPool>,
query: web::Query<GameVersionQueryData>,
) -> Result<HttpResponse, ApiError> {
if let Some(type_) = &query.type_ {
let results = GameVersion::list_type(type_, &**pool).await?;
Ok(HttpResponse::Ok().json(results))
} else {
let results = GameVersion::list(&**pool).await?;
Ok(HttpResponse::Ok().json(results))
}
}
#[derive(serde::Deserialize)]
pub struct GameVersionData {
#[serde(rename = "type")]
type_: String,
}
// At some point this may take more info, but it should be able to
// remain idempotent
#[put("game_version/{name}")]
pub async fn game_version_create(
req: HttpRequest,
pool: web::Data<PgPool>,
game_version: web::Path<(String,)>,
version_data: web::Json<GameVersionData>,
) -> Result<HttpResponse, ApiError> {
check_is_admin_from_headers(
req.headers(),
@@ -178,8 +191,12 @@ pub async fn game_version_create(
let name = game_version.into_inner().0;
// The version type currently isn't limited, but it should be one of:
// "release", "snapshot", "alpha", "beta", "other"
let _id = GameVersion::builder()
.version(&name)?
.version_type(&version_data.type_)?
.insert(&**pool)
.await?;

103
src/scheduler.rs
View File

@@ -28,3 +28,106 @@ impl Drop for Scheduler {
self.arbiter.stop();
}
}
use log::{info, warn};
pub fn schedule_versions(
scheduler: &mut Scheduler,
pool: sqlx::Pool<sqlx::Postgres>,
skip_initial: bool,
) {
// Check mojang's versions every 6 hours
let version_index_interval = std::time::Duration::from_secs(60 * 60 * 6);
let mut skip = skip_initial;
scheduler.run(version_index_interval, move || {
let pool_ref = pool.clone();
let local_skip = skip;
if skip {
skip = false;
}
async move {
if local_skip {
return;
}
info!("Indexing game versions list from Mojang");
let result = update_versions(&pool_ref).await;
if let Err(e) = result {
warn!("Version update failed: {}", e);
}
info!("Done indexing game versions");
}
});
}
use thiserror::Error;
#[derive(Error, Debug)]
pub enum VersionIndexingError {
#[error("Network error while updating game versions list: {0}")]
NetworkError(#[from] reqwest::Error),
#[error("Database error while updating game versions list: {0}")]
DatabaseError(#[from] crate::database::models::DatabaseError),
}
use serde::Deserialize;
#[derive(Deserialize)]
struct InputFormat<'a> {
// latest: LatestFormat,
versions: Vec<VersionFormat<'a>>,
}
#[derive(Deserialize)]
struct VersionFormat<'a> {
id: String,
#[serde(rename = "type")]
type_: std::borrow::Cow<'a, str>,
}
async fn update_versions(pool: &sqlx::Pool<sqlx::Postgres>) -> Result<(), VersionIndexingError> {
let input = reqwest::get("https://launchermeta.mojang.com/mc/game/version_manifest.json")
.await?
.json::<InputFormat>()
.await?;
let mut skipped_versions_count = 0u32;
for version in input.versions.into_iter() {
let name = version.id;
if !name
.chars()
.all(|c| c.is_ascii_alphanumeric() || "-_.".contains(c))
{
// We'll deal with these manually
skipped_versions_count += 1;
continue;
}
let type_ = match &*version.type_ {
"release" => "release",
"snapshot" => "snapshot",
"old_alpha" => "alpha",
"old_beta" => "beta",
_ => "other",
};
crate::database::models::categories::GameVersion::builder()
.version(&name)?
.version_type(type_)?
.insert(pool)
.await?;
}
if skipped_versions_count > 0 {
// This will currently always trigger due to 1.14 pre releases
// and the shareware april fools update. We could set a threshold
// that accounts for those versions and update it whenever we
// manually fix another version.
warn!(
"Skipped {} game versions; check for new versions and add them manually",
skipped_versions_count
);
}
Ok(())
}