eightequal/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Search test + v3 (#731)

Browse Source 
* search patch for accurate loader/gv filtering

* backup

* basic search test

* finished test

* incomplete commit; backing up

* Working multipat reroute backup

* working rough draft v3

* most tests passing

* works

* search v2 conversion

* added some tags.rs v2 conversions

* Worked through warnings, unwraps, prints

* refactors

* new search test

* version files changes fixes

* redesign to revs

* removed old caches

* removed games

* fmt clippy

* merge conflicts

* fmt, prepare

* moved v2 routes over to v3

* fixes; tests passing

* project type changes

* moved files over

* fmt, clippy, prepare, etc

* loaders to loader_fields, added tests

* fmt, clippy, prepare

* fixed sorting bug

* reversed back- wrong order for consistency

* fmt; clippy; prepare

---------

Co-authored-by: Jai A <jaiagr+gpg@pm.me>
This commit is contained in:
Wyatt Verchere
2023-11-11 16:40:10 -08:00
committed by GitHub
parent 97ccb7df94
commit ae1c5342f2
133 changed files with 18153 additions and 11320 deletions
Download Patch File Download Diff File Expand all files Collapse all files

309
src/database/models/categories.rs
View File

@@ -1,9 +1,9 @@
use std::collections::HashMap;
use crate::database::redis::RedisPool;
use super::ids::*;
use super::DatabaseError;
use chrono::DateTime;
use chrono::Utc;
use futures::TryStreamExt;
use serde::{Deserialize, Serialize};
@@ -14,29 +14,6 @@ pub struct ProjectType {
pub name: String,
}
pub struct SideType {
pub id: SideTypeId,
pub name: String,
}
#[derive(Serialize, Deserialize)]
pub struct Loader {
pub id: LoaderId,
pub loader: String,
pub icon: String,
pub supported_project_types: Vec<String>,
}
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct GameVersion {
pub id: GameVersionId,
pub version: String,
#[serde(rename = "type")]
pub type_: String,
pub created: DateTime<Utc>,
pub major: bool,
}
#[derive(Serialize, Deserialize)]
pub struct Category {
pub id: CategoryId,
@@ -59,21 +36,32 @@ pub struct DonationPlatform {
}
impl Category {
pub async fn get_id<'a, E>(name: &str, exec: E) -> Result<Option<CategoryId>, DatabaseError>
// Gets hashmap of category ids matching a name
// Multiple categories can have the same name, but different project types, so we need to return a hashmap
// ProjectTypeId -> CategoryId
pub async fn get_ids<'a, E>(
name: &str,
exec: E,
) -> Result<HashMap<ProjectTypeId, CategoryId>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let result = sqlx::query!(
"
SELECT id FROM categories
SELECT id, project_type FROM categories
WHERE category = $1
",
name,
)
.fetch_optional(exec)
.fetch_all(exec)
.await?;
Ok(result.map(|r| CategoryId(r.id)))
let mut map = HashMap::new();
for r in result {
map.insert(ProjectTypeId(r.project_type), CategoryId(r.id));
}
Ok(map)
}
pub async fn get_id_project<'a, E>(
@@ -139,221 +127,6 @@ impl Category {
}
}
impl Loader {
pub async fn get_id<'a, E>(name: &str, exec: E) -> Result<Option<LoaderId>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let result = sqlx::query!(
"
SELECT id FROM loaders
WHERE loader = $1
",
name
)
.fetch_optional(exec)
.await?;
Ok(result.map(|r| LoaderId(r.id)))
}
pub async fn list<'a, E>(exec: E, redis: &RedisPool) -> Result<Vec<Loader>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let res: Option<Vec<Loader>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "loader")
.await?;
if let Some(res) = res {
return Ok(res);
}
let result = sqlx::query!(
"
SELECT l.id id, l.loader loader, l.icon icon,
ARRAY_AGG(DISTINCT pt.name) filter (where pt.name is not null) project_types
FROM loaders l
LEFT OUTER JOIN loaders_project_types lpt ON joining_loader_id = l.id
LEFT OUTER JOIN project_types pt ON lpt.joining_project_type_id = pt.id
GROUP BY l.id;
"
)
.fetch_many(exec)
.try_filter_map(|e| async {
Ok(e.right().map(|x| Loader {
id: LoaderId(x.id),
loader: x.loader,
icon: x.icon,
supported_project_types: x
.project_types
.unwrap_or_default()
.iter()
.map(|x| x.to_string())
.collect(),
}))
})
.try_collect::<Vec<_>>()
.await?;
redis
.set_serialized_to_json(TAGS_NAMESPACE, "loader", &result, None)
.await?;
Ok(result)
}
}
#[derive(Default)]
pub struct GameVersionBuilder<'a> {
pub version: Option<&'a str>,
pub version_type: Option<&'a str>,
pub date: Option<&'a DateTime<Utc>>,
}
impl GameVersion {
pub fn builder() -> GameVersionBuilder<'static> {
GameVersionBuilder::default()
}
pub async fn get_id<'a, E>(
version: &str,
exec: E,
) -> Result<Option<GameVersionId>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let result = sqlx::query!(
"
SELECT id FROM game_versions
WHERE version = $1
",
version
)
.fetch_optional(exec)
.await?;
Ok(result.map(|r| GameVersionId(r.id)))
}
pub async fn list<'a, E>(exec: E, redis: &RedisPool) -> Result<Vec<GameVersion>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let res: Option<Vec<GameVersion>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "game_version")
.await?;
if let Some(res) = res {
return Ok(res);
}
let result = sqlx::query!(
"
SELECT gv.id id, gv.version version_, gv.type type_, gv.created created, gv.major FROM game_versions gv
ORDER BY created DESC
"
)
.fetch_many(exec)
.try_filter_map(|e| async { Ok(e.right().map(|c| GameVersion {
id: GameVersionId(c.id),
version: c.version_,
type_: c.type_,
created: c.created,
major: c.major
})) })
.try_collect::<Vec<GameVersion>>()
.await?;
redis
.set_serialized_to_json(TAGS_NAMESPACE, "game_version", &result, None)
.await?;
Ok(result)
}
pub async fn list_filter<'a, E>(
version_type_option: Option<&str>,
major_option: Option<bool>,
exec: E,
redis: &RedisPool,
) -> Result<Vec<GameVersion>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let result = Self::list(exec, redis)
.await?
.into_iter()
.filter(|x| {
let mut bool = true;
if let Some(version_type) = version_type_option {
bool &= &*x.type_ == version_type;
}
if let Some(major) = major_option {
bool &= x.major == major;
}
bool
})
.collect();
Ok(result)
}
}
impl<'a> GameVersionBuilder<'a> {
/// The game version. Spaces must be replaced with '_' for it to be valid
pub fn version(self, version: &'a str) -> Result<GameVersionBuilder<'a>, DatabaseError> {
Ok(Self {
version: Some(version),
..self
})
}
pub fn version_type(
self,
version_type: &'a str,
) -> Result<GameVersionBuilder<'a>, DatabaseError> {
Ok(Self {
version_type: Some(version_type),
..self
})
}
pub fn created(self, created: &'a DateTime<Utc>) -> GameVersionBuilder<'a> {
Self {
date: Some(created),
..self
}
}
pub async fn insert<'b, E>(self, exec: E) -> Result<GameVersionId, DatabaseError>
where
E: sqlx::Executor<'b, Database = sqlx::Postgres>,
{
// This looks like a mess, but it *should* work
// This allows game versions to be partially updated without
// replacing the unspecified fields with defaults.
let result = sqlx::query!(
"
INSERT INTO game_versions (version, type, created)
VALUES ($1, COALESCE($2, 'other'), COALESCE($3, timezone('utc', now())))
ON CONFLICT (version) DO UPDATE
SET type = COALESCE($2, game_versions.type),
created = COALESCE($3, game_versions.created)
RETURNING id
",
self.version,
self.version_type,
self.date.map(chrono::DateTime::naive_utc),
)
.fetch_one(exec)
.await?;
Ok(GameVersionId(result.id))
}
}
impl DonationPlatform {
pub async fn get_id<'a, E>(
id: &str,
@@ -509,51 +282,3 @@ impl ProjectType {
Ok(result)
}
}
impl SideType {
pub async fn get_id<'a, E>(name: &str, exec: E) -> Result<Option<SideTypeId>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let result = sqlx::query!(
"
SELECT id FROM side_types
WHERE name = $1
",
name
)
.fetch_optional(exec)
.await?;
Ok(result.map(|r| SideTypeId(r.id)))
}
pub async fn list<'a, E>(exec: E, redis: &RedisPool) -> Result<Vec<String>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let res: Option<Vec<String>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "side_type")
.await?;
if let Some(res) = res {
return Ok(res);
}
let result = sqlx::query!(
"
SELECT name FROM side_types
"
)
.fetch_many(exec)
.try_filter_map(|e| async { Ok(e.right().map(|c| c.name)) })
.try_collect::<Vec<String>>()
.await?;
redis
.set_serialized_to_json(TAGS_NAMESPACE, "side_type", &result, None)
.await?;
Ok(result)
}
}

19
src/database/models/ids.rs
View File

@@ -202,7 +202,7 @@ pub struct OrganizationId(pub i64);
#[derive(Copy, Clone, Debug, Type, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[sqlx(transparent)]
pub struct ProjectId(pub i64);
#[derive(Copy, Clone, Debug, Type, Serialize, Deserialize)]
#[derive(Copy, Clone, Debug, Type, Serialize, Deserialize, PartialEq, Eq, Hash)]
#[sqlx(transparent)]
pub struct ProjectTypeId(pub i32);
@@ -219,10 +219,7 @@ pub struct DonationPlatformId(pub i32);
#[derive(Copy, Clone, Debug, Type, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[sqlx(transparent)]
pub struct VersionId(pub i64);
#[derive(Copy, Clone, Debug, Type, Deserialize, Serialize)]
#[sqlx(transparent)]
pub struct GameVersionId(pub i32);
#[derive(Copy, Clone, Debug, Type, Serialize, Deserialize)]
#[derive(Copy, Clone, Debug, Type, Serialize, Deserialize, PartialEq, Eq, Hash)]
#[sqlx(transparent)]
pub struct LoaderId(pub i32);
#[derive(Copy, Clone, Debug, Type, Serialize, Deserialize)]
@@ -270,6 +267,18 @@ pub struct SessionId(pub i64);
#[sqlx(transparent)]
pub struct ImageId(pub i64);
#[derive(Copy, Clone, Debug, Type, Serialize, Deserialize, Eq, PartialEq, Hash)]
#[sqlx(transparent)]
pub struct LoaderFieldId(pub i32);
#[derive(Copy, Clone, Debug, Type, Serialize, Deserialize, Eq, PartialEq, Hash)]
#[sqlx(transparent)]
pub struct LoaderFieldEnumId(pub i32);
#[derive(Copy, Clone, Debug, Type, Serialize, Deserialize, Eq, PartialEq, Hash)]
#[sqlx(transparent)]
pub struct LoaderFieldEnumValueId(pub i32);
#[derive(Copy, Clone, Debug, Type, Serialize, Deserialize, Eq, PartialEq, Hash)]
#[sqlx(transparent)]
pub struct OAuthClientId(pub i64);

208
src/database/models/legacy_loader_fields.rs Normal file
View File

@@ -0,0 +1,208 @@
// In V3, we switched to dynamic loader fields for a better support for more loaders, games, and potential metadata.
// This file contains the legacy loader fields, which are still used by V2 projects.
// They are still useful to have in several places where minecraft-java functionality is hardcoded- for example,
// for fetching data from forge, maven, etc.
// These fields only apply to minecraft-java, and are hardcoded to the minecraft-java game.
use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize};
use serde_json::json;
use crate::database::redis::RedisPool;
use super::{
loader_fields::{LoaderFieldEnum, LoaderFieldEnumValue, VersionField, VersionFieldValue},
DatabaseError, LoaderFieldEnumValueId,
};
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct MinecraftGameVersion {
pub id: LoaderFieldEnumValueId,
pub version: String,
#[serde(rename = "type")]
pub type_: String,
pub created: DateTime<Utc>,
pub major: bool,
}
impl MinecraftGameVersion {
// The name under which this legacy field is stored as a LoaderField
pub const FIELD_NAME: &'static str = "game_versions";
pub fn builder() -> MinecraftGameVersionBuilder<'static> {
MinecraftGameVersionBuilder::default()
}
pub async fn list<'a, E>(
exec: E,
redis: &RedisPool,
) -> Result<Vec<MinecraftGameVersion>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy,
{
let game_version_enum = LoaderFieldEnum::get(Self::FIELD_NAME, exec, redis)
.await?
.ok_or_else(|| {
DatabaseError::SchemaError("Could not find game version enum.".to_string())
})?;
let game_version_enum_values =
LoaderFieldEnumValue::list(game_version_enum.id, exec, redis).await?;
Ok(game_version_enum_values
.into_iter()
.map(MinecraftGameVersion::from_enum_value)
.collect())
}
// TODO: remove this
pub async fn list_transaction(
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
redis: &RedisPool,
) -> Result<Vec<MinecraftGameVersion>, DatabaseError> {
let game_version_enum = LoaderFieldEnum::get(Self::FIELD_NAME, &mut **transaction, redis)
.await?
.ok_or_else(|| {
DatabaseError::SchemaError("Could not find game version enum.".to_string())
})?;
let game_version_enum_values =
LoaderFieldEnumValue::list(game_version_enum.id, &mut **transaction, redis).await?;
Ok(game_version_enum_values
.into_iter()
.map(MinecraftGameVersion::from_enum_value)
.collect())
}
// Tries to create a MinecraftGameVersion from a VersionField
// Clones on success
pub fn try_from_version_field(
version_field: &VersionField,
) -> Result<Vec<Self>, DatabaseError> {
if version_field.field_name != Self::FIELD_NAME {
return Err(DatabaseError::SchemaError(format!(
"Field name {} is not {}",
version_field.field_name,
Self::FIELD_NAME
)));
}
let game_versions = match version_field.clone() {
VersionField {
value: VersionFieldValue::ArrayEnum(_, values),
..
} => values.into_iter().map(Self::from_enum_value).collect(),
VersionField {
value: VersionFieldValue::Enum(_, value),
..
} => {
vec![Self::from_enum_value(value)]
}
_ => {
return Err(DatabaseError::SchemaError(format!(
"Game version requires field value to be an enum: {:?}",
version_field
)));
}
};
Ok(game_versions)
}
pub fn from_enum_value(loader_field_enum_value: LoaderFieldEnumValue) -> MinecraftGameVersion {
MinecraftGameVersion {
id: loader_field_enum_value.id,
version: loader_field_enum_value.value,
created: loader_field_enum_value.created,
type_: loader_field_enum_value
.metadata
.get("type")
.and_then(|x| x.as_str())
.map(|x| x.to_string())
.unwrap_or_default(),
major: loader_field_enum_value
.metadata
.get("major")
.and_then(|x| x.as_bool())
.unwrap_or_default(),
}
}
}
#[derive(Default)]
pub struct MinecraftGameVersionBuilder<'a> {
pub version: Option<&'a str>,
pub version_type: Option<&'a str>,
pub date: Option<&'a DateTime<Utc>>,
}
impl<'a> MinecraftGameVersionBuilder<'a> {
pub fn new() -> Self {
Self::default()
}
/// The game version. Spaces must be replaced with '_' for it to be valid
pub fn version(
self,
version: &'a str,
) -> Result<MinecraftGameVersionBuilder<'a>, DatabaseError> {
Ok(Self {
version: Some(version),
..self
})
}
pub fn version_type(
self,
version_type: &'a str,
) -> Result<MinecraftGameVersionBuilder<'a>, DatabaseError> {
Ok(Self {
version_type: Some(version_type),
..self
})
}
pub fn created(self, created: &'a DateTime<Utc>) -> MinecraftGameVersionBuilder<'a> {
Self {
date: Some(created),
..self
}
}
pub async fn insert<'b, E>(
self,
exec: E,
redis: &RedisPool,
) -> Result<LoaderFieldEnumValueId, DatabaseError>
where
E: sqlx::Executor<'b, Database = sqlx::Postgres> + Copy,
{
let game_versions_enum = LoaderFieldEnum::get("game_versions", exec, redis)
.await?
.ok_or(DatabaseError::SchemaError(
"Missing loaders field: 'game_versions'".to_string(),
))?;
// Get enum id for game versions
let metadata = json!({
"type": self.version_type,
"major": false
});
// This looks like a mess, but it *should* work
// This allows game versions to be partially updated without
// replacing the unspecified fields with defaults.
let result = sqlx::query!(
"
INSERT INTO loader_field_enum_values (enum_id, value, created, metadata)
VALUES ($1, $2, COALESCE($3, timezone('utc', now())), $4)
ON CONFLICT (enum_id, value) DO UPDATE
SET metadata = COALESCE($4, loader_field_enum_values.metadata),
created = COALESCE($3, loader_field_enum_values.created)
RETURNING id
",
game_versions_enum.id.0,
self.version,
self.date.map(chrono::DateTime::naive_utc),
metadata
)
.fetch_one(exec)
.await?;
Ok(LoaderFieldEnumValueId(result.id))
}
}

959
src/database/models/loader_fields.rs Normal file
View File

@@ -0,0 +1,959 @@
use std::collections::HashMap;
use super::ids::*;
use super::DatabaseError;
use crate::database::redis::RedisPool;
use chrono::DateTime;
use chrono::Utc;
use futures::TryStreamExt;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
const LOADER_ID: &str = "loader_id";
const LOADERS_LIST_NAMESPACE: &str = "loaders";
const LOADER_FIELDS_NAMESPACE: &str = "loader_fields";
const LOADER_FIELD_ENUMS_ID_NAMESPACE: &str = "loader_field_enums";
const LOADER_FIELD_ENUM_VALUES_NAMESPACE: &str = "loader_field_enum_values";
#[derive(Clone, Serialize, Deserialize, Debug, Copy)]
pub enum Game {
MinecraftJava,
// MinecraftBedrock
// Future games
}
impl Game {
pub fn name(&self) -> &'static str {
match self {
Game::MinecraftJava => "minecraft-java",
// Game::MinecraftBedrock => "minecraft-bedrock"
// Future games
}
}
pub fn from_name(name: &str) -> Option<Game> {
match name {
"minecraft-java" => Some(Game::MinecraftJava),
// "minecraft-bedrock" => Some(Game::MinecraftBedrock)
// Future games
_ => None,
}
}
}
#[derive(Serialize, Deserialize, Clone)]
pub struct Loader {
pub id: LoaderId,
pub loader: String,
pub icon: String,
pub supported_project_types: Vec<String>,
pub supported_games: Vec<Game>,
}
impl Loader {
pub async fn get_id<'a, E>(
name: &str,
exec: E,
redis: &RedisPool,
) -> Result<Option<LoaderId>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let cached_id: Option<i32> = redis.get_deserialized_from_json(LOADER_ID, name).await?;
if let Some(cached_id) = cached_id {
return Ok(Some(LoaderId(cached_id)));
}
let result = sqlx::query!(
"
SELECT id FROM loaders
WHERE loader = $1
",
name
)
.fetch_optional(exec)
.await?
.map(|r| LoaderId(r.id));
if let Some(result) = result {
redis
.set_serialized_to_json(LOADER_ID, name, &result.0, None)
.await?;
}
Ok(result)
}
pub async fn list<'a, E>(exec: E, redis: &RedisPool) -> Result<Vec<Loader>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let cached_loaders: Option<Vec<Loader>> = redis
.get_deserialized_from_json(LOADERS_LIST_NAMESPACE, "all")
.await?;
if let Some(cached_loaders) = cached_loaders {
return Ok(cached_loaders);
}
let result = sqlx::query!(
"
SELECT l.id id, l.loader loader, l.icon icon,
ARRAY_AGG(DISTINCT pt.name) filter (where pt.name is not null) project_types,
ARRAY_AGG(DISTINCT g.name) filter (where g.name is not null) games
FROM loaders l
LEFT OUTER JOIN loaders_project_types lpt ON joining_loader_id = l.id
LEFT OUTER JOIN project_types pt ON lpt.joining_project_type_id = pt.id
LEFT OUTER JOIN loaders_project_types_games lptg ON lptg.loader_id = lpt.joining_loader_id AND lptg.project_type_id = lpt.joining_project_type_id
LEFT OUTER JOIN games g ON lptg.game_id = g.id
GROUP BY l.id;
",
)
.fetch_many(exec)
.try_filter_map(|e| async {
Ok(e.right().map(|x| Loader {
id: LoaderId(x.id),
loader: x.loader,
icon: x.icon,
supported_project_types: x
.project_types
.unwrap_or_default()
.iter()
.map(|x| x.to_string())
.collect(),
supported_games: x
.games
.unwrap_or_default()
.iter()
.filter_map(|x| Game::from_name(x))
.collect(),
}))
})
.try_collect::<Vec<_>>()
.await?;
redis
.set_serialized_to_json(LOADERS_LIST_NAMESPACE, "all", &result, None)
.await?;
Ok(result)
}
}
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct LoaderField {
pub id: LoaderFieldId,
pub field: String,
pub field_type: LoaderFieldType,
pub optional: bool,
pub min_val: Option<i32>,
pub max_val: Option<i32>,
}
#[derive(Clone, Serialize, Deserialize, Debug)]
pub enum LoaderFieldType {
Integer,
Text,
Enum(LoaderFieldEnumId),
Boolean,
ArrayInteger,
ArrayText,
ArrayEnum(LoaderFieldEnumId),
ArrayBoolean,
}
impl LoaderFieldType {
pub fn build(field_type_name: &str, loader_field_enum: Option<i32>) -> Option<LoaderFieldType> {
Some(match (field_type_name, loader_field_enum) {
("integer", _) => LoaderFieldType::Integer,
("text", _) => LoaderFieldType::Text,
("boolean", _) => LoaderFieldType::Boolean,
("array_integer", _) => LoaderFieldType::ArrayInteger,
("array_text", _) => LoaderFieldType::ArrayText,
("array_boolean", _) => LoaderFieldType::ArrayBoolean,
("enum", Some(id)) => LoaderFieldType::Enum(LoaderFieldEnumId(id)),
("array_enum", Some(id)) => LoaderFieldType::ArrayEnum(LoaderFieldEnumId(id)),
_ => return None,
})
}
pub fn to_str(&self) -> &'static str {
match self {
LoaderFieldType::Integer => "integer",
LoaderFieldType::Text => "text",
LoaderFieldType::Boolean => "boolean",
LoaderFieldType::ArrayInteger => "array_integer",
LoaderFieldType::ArrayText => "array_text",
LoaderFieldType::ArrayBoolean => "array_boolean",
LoaderFieldType::Enum(_) => "enum",
LoaderFieldType::ArrayEnum(_) => "array_enum",
}
}
}
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct LoaderFieldEnum {
pub id: LoaderFieldEnumId,
pub enum_name: String,
pub ordering: Option<i32>,
pub hidable: bool,
}
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)]
pub struct LoaderFieldEnumValue {
pub id: LoaderFieldEnumValueId,
pub enum_id: LoaderFieldEnumId,
pub value: String,
pub ordering: Option<i32>,
pub created: DateTime<Utc>,
#[serde(flatten)]
pub metadata: serde_json::Value,
}
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)]
pub struct VersionField {
pub version_id: VersionId,
pub field_id: LoaderFieldId,
pub field_name: String,
pub value: VersionFieldValue,
}
#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Eq)]
pub enum VersionFieldValue {
Integer(i32),
Text(String),
Enum(LoaderFieldEnumId, LoaderFieldEnumValue),
Boolean(bool),
ArrayInteger(Vec<i32>),
ArrayText(Vec<String>),
ArrayEnum(LoaderFieldEnumId, Vec<LoaderFieldEnumValue>),
ArrayBoolean(Vec<bool>),
}
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct QueryVersionField {
pub version_id: VersionId,
pub field_id: LoaderFieldId,
pub int_value: Option<i32>,
pub enum_value: Option<LoaderFieldEnumValue>,
pub string_value: Option<String>,
}
impl QueryVersionField {
pub fn with_int_value(mut self, int_value: i32) -> Self {
self.int_value = Some(int_value);
self
}
pub fn with_enum_value(mut self, enum_value: LoaderFieldEnumValue) -> Self {
self.enum_value = Some(enum_value);
self
}
pub fn with_string_value(mut self, string_value: String) -> Self {
self.string_value = Some(string_value);
self
}
}
#[derive(Clone, Serialize, Deserialize, Debug)]
pub struct SideType {
pub id: SideTypeId,
pub name: String,
}
impl LoaderField {
pub async fn get_field<'a, E>(
field: &str,
exec: E,
redis: &RedisPool,
) -> Result<Option<LoaderField>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let fields = Self::get_fields(exec, redis).await?;
Ok(fields.into_iter().find(|f| f.field == field))
}
// Gets all fields for a given loader
// Returns all as this there are probably relatively few fields per loader
// TODO: in the future, this should be to get all fields in relation to something
// - e.g. get all fields for a given game?
pub async fn get_fields<'a, E>(
exec: E,
redis: &RedisPool,
) -> Result<Vec<LoaderField>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let cached_fields = redis
.get_deserialized_from_json(LOADER_FIELDS_NAMESPACE, 0) // 0 => whatever we search for fields by
.await?;
if let Some(cached_fields) = cached_fields {
return Ok(cached_fields);
}
let result = sqlx::query!(
"
SELECT lf.id, lf.field, lf.field_type, lf.optional, lf.min_val, lf.max_val, lf.enum_type
FROM loader_fields lf
",
)
.fetch_many(exec)
.try_filter_map(|e| async {
Ok(e.right().and_then(|r| {
Some(LoaderField {
id: LoaderFieldId(r.id),
field_type: LoaderFieldType::build(&r.field_type, r.enum_type)?,
field: r.field,
optional: r.optional,
min_val: r.min_val,
max_val: r.max_val,
})
}))
})
.try_collect::<Vec<LoaderField>>()
.await?;
redis
.set_serialized_to_json(LOADER_FIELDS_NAMESPACE, &0, &result, None)
.await?;
Ok(result)
}
}
impl LoaderFieldEnum {
pub async fn get<'a, E>(
enum_name: &str, // Note: NOT loader field name
exec: E,
redis: &RedisPool,
) -> Result<Option<LoaderFieldEnum>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let cached_enum = redis
.get_deserialized_from_json(LOADER_FIELD_ENUMS_ID_NAMESPACE, enum_name)
.await?;
if let Some(cached_enum) = cached_enum {
return Ok(cached_enum);
}
let result = sqlx::query!(
"
SELECT lfe.id, lfe.enum_name, lfe.ordering, lfe.hidable
FROM loader_field_enums lfe
WHERE lfe.enum_name = $1
",
enum_name
)
.fetch_optional(exec)
.await?
.map(|l| LoaderFieldEnum {
id: LoaderFieldEnumId(l.id),
enum_name: l.enum_name,
ordering: l.ordering,
hidable: l.hidable,
});
redis
.set_serialized_to_json(LOADER_FIELD_ENUMS_ID_NAMESPACE, enum_name, &result, None)
.await?;
Ok(result)
}
}
impl LoaderFieldEnumValue {
pub async fn list<'a, E>(
loader_field_enum_id: LoaderFieldEnumId,
exec: E,
redis: &RedisPool,
) -> Result<Vec<LoaderFieldEnumValue>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
Ok(Self::list_many(&[loader_field_enum_id], exec, redis)
.await?
.into_iter()
.next()
.map(|x| x.1)
.unwrap_or_default())
}
pub async fn list_many_loader_fields<'a, E>(
loader_fields: &[LoaderField],
exec: E,
redis: &RedisPool,
) -> Result<HashMap<LoaderFieldId, Vec<LoaderFieldEnumValue>>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let get_enum_id = |x: &LoaderField| match x.field_type {
LoaderFieldType::Enum(id) | LoaderFieldType::ArrayEnum(id) => Some(id),
_ => None,
};
let enum_ids = loader_fields
.iter()
.filter_map(|x| get_enum_id(x))
.collect::<Vec<_>>();
let values = Self::list_many(&enum_ids, exec, redis)
.await?
.into_iter()
.collect::<HashMap<_, _>>();
let mut res = HashMap::new();
for lf in loader_fields {
if let Some(id) = get_enum_id(lf) {
res.insert(lf.id, values.get(&id).unwrap_or(&Vec::new()).to_vec());
}
}
Ok(res)
}
pub async fn list_many<'a, E>(
loader_field_enum_ids: &[LoaderFieldEnumId],
exec: E,
redis: &RedisPool,
) -> Result<Vec<(LoaderFieldEnumId, Vec<LoaderFieldEnumValue>)>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let mut found_enums = Vec::new();
let mut remaining_enums: Vec<LoaderFieldEnumId> = loader_field_enum_ids.to_vec();
if !remaining_enums.is_empty() {
let enums = redis
.multi_get::<String, _>(
LOADER_FIELD_ENUM_VALUES_NAMESPACE,
loader_field_enum_ids.iter().map(|x| x.0),
)
.await?;
for lfe in enums {
if let Some(lfe) = lfe.and_then(|x| {
serde_json::from_str::<(LoaderFieldEnumId, Vec<LoaderFieldEnumValue>)>(&x).ok()
}) {
remaining_enums.retain(|x| lfe.0 .0 != x.0);
found_enums.push(lfe.1);
continue;
}
}
}
let remaining_enums = remaining_enums.iter().map(|x| x.0).collect::<Vec<_>>();
let result = sqlx::query!(
"
SELECT id, enum_id, value, ordering, metadata, created FROM loader_field_enum_values
WHERE enum_id = ANY($1)
",
&remaining_enums
)
.fetch_many(exec)
.try_filter_map(|e| async {
Ok(e.right().map(|c| LoaderFieldEnumValue {
id: LoaderFieldEnumValueId(c.id),
enum_id: LoaderFieldEnumId(c.enum_id),
value: c.value,
ordering: c.ordering,
created: c.created,
metadata: c.metadata.unwrap_or_default(),
}))
})
.try_collect::<Vec<LoaderFieldEnumValue>>()
.await?;
// Convert from an Vec<LoaderFieldEnumValue> to a Vec<(LoaderFieldEnumId, Vec<LoaderFieldEnumValue>)>
let cachable_enum_sets: Vec<(LoaderFieldEnumId, Vec<LoaderFieldEnumValue>)> = result
.clone()
.into_iter()
.group_by(|x| x.enum_id)
.into_iter()
.map(|(k, v)| (k, v.collect::<Vec<_>>().to_vec()))
.collect();
for (k, v) in cachable_enum_sets.iter() {
redis
.set_serialized_to_json(LOADER_FIELD_ENUM_VALUES_NAMESPACE, k.0, v, None)
.await?;
}
Ok(cachable_enum_sets)
}
// Matches filter against metadata of enum values
pub async fn list_filter<'a, E>(
loader_field_enum_id: LoaderFieldEnumId,
filter: HashMap<String, serde_json::Value>,
exec: E,
redis: &RedisPool,
) -> Result<Vec<LoaderFieldEnumValue>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let result = Self::list(loader_field_enum_id, exec, redis)
.await?
.into_iter()
.filter(|x| {
let mut bool = true;
for (key, value) in filter.iter() {
if let Some(metadata_value) = x.metadata.get(key) {
bool &= metadata_value == value;
} else {
bool = false;
}
}
bool
})
.collect();
Ok(result)
}
}
impl VersionField {
pub async fn insert_many(
items: Vec<Self>,
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<(), DatabaseError> {
let mut query_version_fields = vec![];
for item in items {
let base = QueryVersionField {
version_id: item.version_id,
field_id: item.field_id,
int_value: None,
enum_value: None,
string_value: None,
};
match item.value {
VersionFieldValue::Integer(i) => {
query_version_fields.push(base.clone().with_int_value(i))
}
VersionFieldValue::Text(s) => {
query_version_fields.push(base.clone().with_string_value(s))
}
VersionFieldValue::Boolean(b) => {
query_version_fields.push(base.clone().with_int_value(if b { 1 } else { 0 }))
}
VersionFieldValue::ArrayInteger(v) => {
for i in v {
query_version_fields.push(base.clone().with_int_value(i));
}
}
VersionFieldValue::ArrayText(v) => {
for s in v {
query_version_fields.push(base.clone().with_string_value(s));
}
}
VersionFieldValue::ArrayBoolean(v) => {
for b in v {
query_version_fields.push(base.clone().with_int_value(if b {
1
} else {
0
}));
}
}
VersionFieldValue::Enum(_, v) => {
query_version_fields.push(base.clone().with_enum_value(v))
}
VersionFieldValue::ArrayEnum(_, v) => {
for ev in v {
query_version_fields.push(base.clone().with_enum_value(ev));
}
}
};
}
let (field_ids, version_ids, int_values, enum_values, string_values): (
Vec<_>,
Vec<_>,
Vec<_>,
Vec<_>,
Vec<_>,
) = query_version_fields
.iter()
.map(|l| {
(
l.field_id.0,
l.version_id.0,
l.int_value,
l.enum_value.as_ref().map(|e| e.id.0),
l.string_value.clone(),
)
})
.multiunzip();
sqlx::query!(
"
INSERT INTO version_fields (field_id, version_id, int_value, string_value, enum_value)
SELECT * FROM UNNEST($1::integer[], $2::bigint[], $3::integer[], $4::text[], $5::integer[])
",
&field_ids[..],
&version_ids[..],
&int_values[..] as &[Option<i32>],
&string_values[..] as &[Option<String>],
&enum_values[..] as &[Option<i32>]
)
.execute(&mut **transaction)
.await?;
Ok(())
}
pub fn check_parse(
version_id: VersionId,
loader_field: LoaderField,
value: serde_json::Value,
enum_variants: Vec<LoaderFieldEnumValue>,
) -> Result<VersionField, String> {
let value = VersionFieldValue::parse(&loader_field, value, enum_variants)?;
Ok(VersionField {
version_id,
field_id: loader_field.id,
field_name: loader_field.field,
value,
})
}
pub fn from_query_json(
version_id: i64,
loader_fields: Option<serde_json::Value>,
version_fields: Option<serde_json::Value>,
loader_field_enum_values: Option<serde_json::Value>,
) -> Vec<VersionField> {
#[derive(Deserialize, Debug)]
struct JsonLoaderField {
lf_id: i32,
field: String,
field_type: String,
enum_type: Option<i32>,
min_val: Option<i32>,
max_val: Option<i32>,
optional: bool,
}
#[derive(Deserialize, Debug)]
struct JsonVersionField {
field_id: i32,
int_value: Option<i32>,
enum_value: Option<i32>,
string_value: Option<String>,
}
#[derive(Deserialize, Debug)]
struct JsonLoaderFieldEnumValue {
id: i32,
enum_id: i32,
value: String,
ordering: Option<i32>,
created: DateTime<Utc>,
metadata: Option<serde_json::Value>,
}
let query_loader_fields: Vec<JsonLoaderField> = loader_fields
.and_then(|x| serde_json::from_value(x).ok())
.unwrap_or_default();
let query_version_field_combined: Vec<JsonVersionField> = version_fields
.and_then(|x| serde_json::from_value(x).ok())
.unwrap_or_default();
let query_loader_field_enum_values: Vec<JsonLoaderFieldEnumValue> =
loader_field_enum_values
.and_then(|x| serde_json::from_value(x).ok())
.unwrap_or_default();
let version_id = VersionId(version_id);
query_loader_fields
.into_iter()
.filter_map(|q| {
let loader_field_type = match LoaderFieldType::build(&q.field_type, q.enum_type) {
Some(lft) => lft,
None => return None,
};
let loader_field = LoaderField {
id: LoaderFieldId(q.lf_id),
field: q.field.clone(),
field_type: loader_field_type,
optional: q.optional,
min_val: q.min_val,
max_val: q.max_val,
};
let values = query_version_field_combined
.iter()
.filter_map(|qvf| {
if qvf.field_id == q.lf_id {
let lfev = query_loader_field_enum_values
.iter()
.find(|x| Some(x.id) == qvf.enum_value);
Some(QueryVersionField {
version_id,
field_id: LoaderFieldId(qvf.field_id),
int_value: qvf.int_value,
enum_value: lfev.map(|lfev| LoaderFieldEnumValue {
id: LoaderFieldEnumValueId(lfev.id),
enum_id: LoaderFieldEnumId(lfev.enum_id),
value: lfev.value.clone(),
ordering: lfev.ordering,
created: lfev.created,
metadata: lfev.metadata.clone().unwrap_or_default(),
}),
string_value: qvf.string_value.clone(),
})
} else {
None
}
})
.collect::<Vec<_>>();
VersionField::build(loader_field, version_id, values).ok()
})
.collect()
}
pub fn build(
loader_field: LoaderField,
version_id: VersionId,
query_version_fields: Vec<QueryVersionField>,
) -> Result<VersionField, DatabaseError> {
let value = VersionFieldValue::build(&loader_field.field_type, query_version_fields)?;
Ok(VersionField {
version_id,
field_id: loader_field.id,
field_name: loader_field.field,
value,
})
}
}
impl VersionFieldValue {
// Build from user-submitted JSON data
// value is the attempted value of the field, which will be tried to parse to the correct type
// enum_array is the list of valid enum variants for the field, if it is an enum (see LoaderFieldEnumValue::list_many_loader_fields)
pub fn parse(
loader_field: &LoaderField,
value: serde_json::Value,
enum_array: Vec<LoaderFieldEnumValue>,
) -> Result<VersionFieldValue, String> {
let field_name = &loader_field.field;
let field_type = &loader_field.field_type;
let error_value = value.clone();
let incorrect_type_error = |field_type: &str| {
format!(
"Provided value '{v}' for {field_name} could not be parsed to {field_type} ",
v = serde_json::to_string(&error_value).unwrap_or_default()
)
};
Ok(match field_type {
LoaderFieldType::Integer => VersionFieldValue::Integer(
serde_json::from_value(value).map_err(|_| incorrect_type_error("integer"))?,
),
LoaderFieldType::Text => VersionFieldValue::Text(
value
.as_str()
.ok_or_else(|| incorrect_type_error("string"))?
.to_string(),
),
LoaderFieldType::Boolean => VersionFieldValue::Boolean(
value
.as_bool()
.ok_or_else(|| incorrect_type_error("boolean"))?,
),
LoaderFieldType::ArrayInteger => VersionFieldValue::ArrayInteger({
let array_values: Vec<i32> = serde_json::from_value(value)
.map_err(|_| incorrect_type_error("array of integers"))?;
array_values.into_iter().collect()
}),
LoaderFieldType::ArrayText => VersionFieldValue::ArrayText({
let array_values: Vec<String> = serde_json::from_value(value)
.map_err(|_| incorrect_type_error("array of strings"))?;
array_values.into_iter().collect()
}),
LoaderFieldType::ArrayBoolean => VersionFieldValue::ArrayBoolean({
let array_values: Vec<i64> = serde_json::from_value(value)
.map_err(|_| incorrect_type_error("array of booleans"))?;
array_values.into_iter().map(|v| v != 0).collect()
}),
LoaderFieldType::Enum(id) => VersionFieldValue::Enum(*id, {
let enum_value = value.as_str().ok_or_else(|| incorrect_type_error("enum"))?;
if let Some(ev) = enum_array.into_iter().find(|v| v.value == enum_value) {
ev
} else {
return Err(format!(
"Provided value '{enum_value}' is not a valid variant for {field_name}"
));
}
}),
LoaderFieldType::ArrayEnum(id) => VersionFieldValue::ArrayEnum(*id, {
let array_values: Vec<String> = serde_json::from_value(value)
.map_err(|_| incorrect_type_error("array of enums"))?;
let mut enum_values = vec![];
for av in array_values {
if let Some(ev) = enum_array.iter().find(|v| v.value == av) {
enum_values.push(ev.clone());
} else {
return Err(format!(
"Provided value '{av}' is not a valid variant for {field_name}"
));
}
}
enum_values
}),
})
}
// Build from internal query data
// This encapsulates reundant behavior in db querie -> object conversions
pub fn build(
field_type: &LoaderFieldType,
qvfs: Vec<QueryVersionField>,
) -> Result<VersionFieldValue, DatabaseError> {
let field_name = field_type.to_str();
let get_first = |qvfs: Vec<QueryVersionField>| -> Result<QueryVersionField, DatabaseError> {
if qvfs.len() > 1 {
return Err(DatabaseError::SchemaError(format!(
"Multiple fields for field {}",
field_name
)));
}
qvfs.into_iter().next().ok_or_else(|| {
DatabaseError::SchemaError(format!("No version fields for field {}", field_name))
})
};
let did_not_exist_error = |field_name: &str, desired_field: &str| {
DatabaseError::SchemaError(format!(
"Field name {} for field {} in does not exist",
desired_field, field_name
))
};
Ok(match field_type {
LoaderFieldType::Integer => VersionFieldValue::Integer(
get_first(qvfs)?
.int_value
.ok_or(did_not_exist_error(field_name, "int_value"))?,
),
LoaderFieldType::Text => VersionFieldValue::Text(
get_first(qvfs)?
.string_value
.ok_or(did_not_exist_error(field_name, "string_value"))?,
),
LoaderFieldType::Boolean => VersionFieldValue::Boolean(
get_first(qvfs)?
.int_value
.ok_or(did_not_exist_error(field_name, "int_value"))?
!= 0,
),
LoaderFieldType::ArrayInteger => VersionFieldValue::ArrayInteger(
qvfs.into_iter()
.map(|qvf| {
qvf.int_value
.ok_or(did_not_exist_error(field_name, "int_value"))
})
.collect::<Result<_, _>>()?,
),
LoaderFieldType::ArrayText => VersionFieldValue::ArrayText(
qvfs.into_iter()
.map(|qvf| {
qvf.string_value
.ok_or(did_not_exist_error(field_name, "string_value"))
})
.collect::<Result<_, _>>()?,
),
LoaderFieldType::ArrayBoolean => VersionFieldValue::ArrayBoolean(
qvfs.into_iter()
.map(|qvf| {
Ok::<bool, DatabaseError>(
qvf.int_value
.ok_or(did_not_exist_error(field_name, "int_value"))?
!= 0,
)
})
.collect::<Result<_, _>>()?,
),
LoaderFieldType::Enum(id) => VersionFieldValue::Enum(
*id,
get_first(qvfs)?
.enum_value
.ok_or(did_not_exist_error(field_name, "enum_value"))?,
),
LoaderFieldType::ArrayEnum(id) => VersionFieldValue::ArrayEnum(
*id,
qvfs.into_iter()
.map(|qvf| {
qvf.enum_value
.ok_or(did_not_exist_error(field_name, "enum_value"))
})
.collect::<Result<_, _>>()?,
),
})
}
// Serialize to internal value, such as for converting to user-facing JSON
pub fn serialize_internal(&self) -> serde_json::Value {
match self {
VersionFieldValue::Integer(i) => serde_json::Value::Number((*i).into()),
VersionFieldValue::Text(s) => serde_json::Value::String(s.clone()),
VersionFieldValue::Boolean(b) => serde_json::Value::Bool(*b),
VersionFieldValue::ArrayInteger(v) => serde_json::Value::Array(
v.iter()
.map(|i| serde_json::Value::Number((*i).into()))
.collect(),
),
VersionFieldValue::ArrayText(v) => serde_json::Value::Array(
v.iter()
.map(|s| serde_json::Value::String(s.clone()))
.collect(),
),
VersionFieldValue::ArrayBoolean(v) => {
serde_json::Value::Array(v.iter().map(|b| serde_json::Value::Bool(*b)).collect())
}
VersionFieldValue::Enum(_, v) => serde_json::Value::String(v.value.clone()),
VersionFieldValue::ArrayEnum(_, v) => serde_json::Value::Array(
v.iter()
.map(|v| serde_json::Value::String(v.value.clone()))
.collect(),
),
}
}
// For conversion to an interanl string(s), such as for search facets, filtering, or direct hardcoding
// No matter the type, it will be converted to a Vec<String>, whre the non-array types will have a single element
pub fn as_strings(&self) -> Vec<String> {
match self {
VersionFieldValue::Integer(i) => vec![i.to_string()],
VersionFieldValue::Text(s) => vec![s.clone()],
VersionFieldValue::Boolean(b) => vec![b.to_string()],
VersionFieldValue::ArrayInteger(v) => v.iter().map(|i| i.to_string()).collect(),
VersionFieldValue::ArrayText(v) => v.clone(),
VersionFieldValue::ArrayBoolean(v) => v.iter().map(|b| b.to_string()).collect(),
VersionFieldValue::Enum(_, v) => vec![v.value.clone()],
VersionFieldValue::ArrayEnum(_, v) => v.iter().map(|v| v.value.clone()).collect(),
}
}
pub fn contains_json_value(&self, value: &serde_json::Value) -> bool {
match self {
VersionFieldValue::Integer(i) => value.as_i64() == Some(*i as i64),
VersionFieldValue::Text(s) => value.as_str() == Some(s),
VersionFieldValue::Boolean(b) => value.as_bool() == Some(*b),
VersionFieldValue::ArrayInteger(v) => value
.as_i64()
.map(|i| v.contains(&(i as i32)))
.unwrap_or(false),
VersionFieldValue::ArrayText(v) => value
.as_str()
.map(|s| v.contains(&s.to_string()))
.unwrap_or(false),
VersionFieldValue::ArrayBoolean(v) => {
value.as_bool().map(|b| v.contains(&b)).unwrap_or(false)
}
VersionFieldValue::Enum(_, v) => value.as_str() == Some(&v.value),
VersionFieldValue::ArrayEnum(_, v) => value
.as_str()
.map(|s| v.iter().any(|v| v.value == s))
.unwrap_or(false),
}
}
}

4
src/database/models/mod.rs
View File

@@ -5,6 +5,8 @@ pub mod collection_item;
pub mod flow_item;
pub mod ids;
pub mod image_item;
pub mod legacy_loader_fields;
pub mod loader_fields;
pub mod notification_item;
pub mod oauth_client_authorization_item;
pub mod oauth_client_item;
@@ -43,4 +45,6 @@ pub enum DatabaseError {
RedisPool(#[from] deadpool_redis::PoolError),
#[error("Error while serializing with the cache: {0}")]
SerdeCacheError(#[from] serde_json::Error),
#[error("Schema error: {0}")]
SchemaError(String),
}

120
src/database/models/project_item.rs
View File

@@ -141,7 +141,6 @@ impl ModCategory {
#[derive(Clone)]
pub struct ProjectBuilder {
pub project_id: ProjectId,
pub project_type_id: ProjectTypeId,
pub team_id: TeamId,
pub organization_id: Option<OrganizationId>,
pub title: String,
@@ -158,8 +157,6 @@ pub struct ProjectBuilder {
pub initial_versions: Vec<super::version_item::VersionBuilder>,
pub status: ProjectStatus,
pub requested_status: Option<ProjectStatus>,
pub client_side: SideTypeId,
pub server_side: SideTypeId,
pub license: String,
pub slug: Option<String>,
pub donation_urls: Vec<DonationUrl>,
@@ -175,7 +172,6 @@ impl ProjectBuilder {
) -> Result<ProjectId, DatabaseError> {
let project_struct = Project {
id: self.project_id,
project_type: self.project_type_id,
team_id: self.team_id,
organization_id: self.organization_id,
title: self.title,
@@ -200,8 +196,6 @@ impl ProjectBuilder {
wiki_url: self.wiki_url,
license_url: self.license_url,
discord_url: self.discord_url,
client_side: self.client_side,
server_side: self.server_side,
license: self.license,
slug: self.slug,
moderation_message: None,
@@ -210,7 +204,6 @@ impl ProjectBuilder {
color: self.color,
monetization_status: self.monetization_status,
loaders: vec![],
game_versions: vec![],
};
project_struct.insert(&mut *transaction).await?;
@@ -244,16 +237,12 @@ impl ProjectBuilder {
.collect_vec();
ModCategory::insert_many(mod_categories, &mut *transaction).await?;
Project::update_game_versions(self.project_id, &mut *transaction).await?;
Project::update_loaders(self.project_id, &mut *transaction).await?;
Ok(self.project_id)
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct Project {
pub id: ProjectId,
pub project_type: ProjectTypeId,
pub team_id: TeamId,
pub organization_id: Option<OrganizationId>,
pub title: String,
@@ -274,8 +263,6 @@ pub struct Project {
pub wiki_url: Option<String>,
pub license_url: Option<String>,
pub discord_url: Option<String>,
pub client_side: SideTypeId,
pub server_side: SideTypeId,
pub license: String,
pub slug: Option<String>,
pub moderation_message: Option<String>,
@@ -284,7 +271,6 @@ pub struct Project {
pub color: Option<u32>,
pub monetization_status: MonetizationStatus,
pub loaders: Vec<String>,
pub game_versions: Vec<String>,
}
impl Project {
@@ -298,15 +284,15 @@ impl Project {
id, team_id, title, description, body,
published, downloads, icon_url, issues_url,
source_url, wiki_url, status, requested_status, discord_url,
client_side, server_side, license_url, license,
slug, project_type, color, monetization_status
license_url, license,
slug, color, monetization_status
)
VALUES (
$1, $2, $3, $4, $5,
$6, $7, $8, $9,
$10, $11, $12, $13, $14,
$15, $16, $17, $18,
LOWER($19), $20, $21, $22
$15, $16,
LOWER($17), $18, $19
)
",
self.id as ProjectId,
@@ -323,12 +309,9 @@ impl Project {
self.status.as_str(),
self.requested_status.map(|x| x.as_str()),
self.discord_url.as_ref(),
self.client_side as SideTypeId,
self.server_side as SideTypeId,
self.license_url.as_ref(),
&self.license,
self.slug.as_ref(),
self.project_type as ProjectTypeId,
self.color.map(|x| x as i32),
self.monetization_status.as_str(),
)
@@ -552,7 +535,6 @@ impl Project {
.flatten()
.collect(),
);
if !project_ids.is_empty() {
let projects = redis
.multi_get::<String, _>(PROJECTS_NAMESPACE, project_ids)
@@ -571,31 +553,31 @@ impl Project {
}
}
}
if !remaining_strings.is_empty() {
let project_ids_parsed: Vec<i64> = remaining_strings
.iter()
.flat_map(|x| parse_base62(&x.to_string()).ok())
.map(|x| x as i64)
.collect();
let db_projects: Vec<QueryProject> = sqlx::query!(
"
SELECT m.id id, m.project_type project_type, m.title title, m.description description, m.downloads downloads, m.follows follows,
SELECT m.id id, m.title title, m.description description, m.downloads downloads, m.follows follows,
m.icon_url icon_url, m.body body, m.published published,
m.updated updated, m.approved approved, m.queued, m.status status, m.requested_status requested_status,
m.issues_url issues_url, m.source_url source_url, m.wiki_url wiki_url, m.discord_url discord_url, m.license_url license_url,
m.team_id team_id, m.organization_id organization_id, m.client_side client_side, m.server_side server_side, m.license license, m.slug slug, m.moderation_message moderation_message, m.moderation_message_body moderation_message_body,
cs.name client_side_type, ss.name server_side_type, pt.name project_type_name, m.webhook_sent, m.color,
t.id thread_id, m.monetization_status monetization_status, m.loaders loaders, m.game_versions game_versions,
m.team_id team_id, m.organization_id organization_id, m.license license, m.slug slug, m.moderation_message moderation_message, m.moderation_message_body moderation_message_body,
m.webhook_sent, m.color,
t.id thread_id, m.monetization_status monetization_status,
ARRAY_AGG(DISTINCT l.loader) filter (where l.loader is not null) loaders,
ARRAY_AGG(DISTINCT pt.name) filter (where pt.name is not null) project_types,
ARRAY_AGG(DISTINCT g.name) filter (where g.name is not null) games,
ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is false) categories,
ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is true) additional_categories,
JSONB_AGG(DISTINCT jsonb_build_object('id', v.id, 'date_published', v.date_published)) filter (where v.id is not null) versions,
JSONB_AGG(DISTINCT jsonb_build_object('image_url', mg.image_url, 'featured', mg.featured, 'title', mg.title, 'description', mg.description, 'created', mg.created, 'ordering', mg.ordering)) filter (where mg.image_url is not null) gallery,
JSONB_AGG(DISTINCT jsonb_build_object('platform_id', md.joining_platform_id, 'platform_short', dp.short, 'platform_name', dp.name,'url', md.url)) filter (where md.joining_platform_id is not null) donations
FROM mods m
INNER JOIN project_types pt ON pt.id = m.project_type
INNER JOIN side_types cs ON m.client_side = cs.id
INNER JOIN side_types ss ON m.server_side = ss.id
FROM mods m
INNER JOIN threads t ON t.mod_id = m.id
LEFT JOIN mods_gallery mg ON mg.mod_id = m.id
LEFT JOIN mods_donations md ON md.joining_mod_id = m.id
@@ -603,8 +585,14 @@ impl Project {
LEFT JOIN mods_categories mc ON mc.joining_mod_id = m.id
LEFT JOIN categories c ON mc.joining_category_id = c.id
LEFT JOIN versions v ON v.mod_id = m.id AND v.status = ANY($3)
LEFT JOIN loaders_versions lv ON lv.version_id = v.id
LEFT JOIN loaders l on lv.loader_id = l.id
LEFT JOIN loaders_project_types lpt ON lpt.joining_loader_id = l.id
LEFT JOIN project_types pt ON pt.id = lpt.joining_project_type_id
LEFT JOIN loaders_project_types_games lptg ON lptg.loader_id = l.id AND lptg.project_type_id = pt.id
LEFT JOIN games g ON lptg.game_id = g.id
WHERE m.id = ANY($1) OR m.slug = ANY($2)
GROUP BY pt.id, cs.id, ss.id, t.id, m.id;
GROUP BY t.id, m.id;
",
&project_ids_parsed,
&remaining_strings.into_iter().map(|x| x.to_string().to_lowercase()).collect::<Vec<_>>(),
@@ -614,11 +602,9 @@ impl Project {
.try_filter_map(|e| async {
Ok(e.right().map(|m| {
let id = m.id;
QueryProject {
inner: Project {
id: ProjectId(id),
project_type: ProjectTypeId(m.project_type),
team_id: TeamId(m.team_id),
organization_id: m.organization_id.map(OrganizationId),
title: m.title.clone(),
@@ -633,14 +619,12 @@ impl Project {
wiki_url: m.wiki_url.clone(),
license_url: m.license_url.clone(),
discord_url: m.discord_url.clone(),
client_side: SideTypeId(m.client_side),
status: ProjectStatus::from_string(
&m.status,
),
requested_status: m.requested_status.map(|x| ProjectStatus::from_string(
&x,
)),
server_side: SideTypeId(m.server_side),
license: m.license.clone(),
slug: m.slug.clone(),
body: m.body.clone(),
@@ -654,12 +638,12 @@ impl Project {
monetization_status: MonetizationStatus::from_string(
&m.monetization_status,
),
loaders: m.loaders,
game_versions: m.game_versions,
loaders: m.loaders.unwrap_or_default(),
},
project_type: m.project_type_name,
categories: m.categories.unwrap_or_default(),
additional_categories: m.additional_categories.unwrap_or_default(),
project_types: m.project_types.unwrap_or_default(),
games: m.games.unwrap_or_default(),
versions: {
#[derive(Deserialize)]
struct Version {
@@ -674,7 +658,6 @@ impl Project {
.unwrap_or_default();
versions.sort_by(|a, b| a.date_published.cmp(&b.date_published));
versions.into_iter().map(|x| x.id).collect()
},
gallery_items: {
@@ -689,8 +672,6 @@ impl Project {
donation_urls: serde_json::from_value(
m.donations.unwrap_or_default(),
).ok().unwrap_or_default(),
client_side: crate::models::projects::SideType::from_string(&m.client_side_type),
server_side: crate::models::projects::SideType::from_string(&m.server_side_type),
thread_id: ThreadId(m.thread_id),
}}))
})
@@ -768,56 +749,6 @@ impl Project {
Ok(dependencies)
}
pub async fn update_game_versions(
id: ProjectId,
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<(), sqlx::error::Error> {
sqlx::query!(
"
UPDATE mods
SET game_versions = (
SELECT COALESCE(ARRAY_AGG(DISTINCT gv.version) filter (where gv.version is not null), array[]::varchar[])
FROM versions v
INNER JOIN game_versions_versions gvv ON v.id = gvv.joining_version_id
INNER JOIN game_versions gv on gvv.game_version_id = gv.id
WHERE v.mod_id = mods.id AND v.status != ALL($2)
)
WHERE id = $1
",
id as ProjectId,
&*crate::models::projects::VersionStatus::iterator().filter(|x| x.is_hidden()).map(|x| x.to_string()).collect::<Vec<String>>()
)
.execute(&mut **transaction)
.await?;
Ok(())
}
pub async fn update_loaders(
id: ProjectId,
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<(), sqlx::error::Error> {
sqlx::query!(
"
UPDATE mods
SET loaders = (
SELECT COALESCE(ARRAY_AGG(DISTINCT l.loader) filter (where l.loader is not null), array[]::varchar[])
FROM versions v
INNER JOIN loaders_versions lv ON lv.version_id = v.id
INNER JOIN loaders l on lv.loader_id = l.id
WHERE v.mod_id = mods.id AND v.status != ALL($2)
)
WHERE id = $1
",
id as ProjectId,
&*crate::models::projects::VersionStatus::iterator().filter(|x| x.is_hidden()).map(|x| x.to_string()).collect::<Vec<String>>()
)
.execute(&mut **transaction)
.await?;
Ok(())
}
pub async fn clear_cache(
id: ProjectId,
slug: Option<String>,
@@ -845,13 +776,12 @@ impl Project {
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct QueryProject {
pub inner: Project,
pub project_type: String,
pub categories: Vec<String>,
pub additional_categories: Vec<String>,
pub versions: Vec<VersionId>,
pub project_types: Vec<String>,
pub games: Vec<String>,
pub donation_urls: Vec<DonationUrl>,
pub gallery_items: Vec<GalleryItem>,
pub client_side: crate::models::projects::SideType,
pub server_side: crate::models::projects::SideType,
pub thread_id: ThreadId,
}

132
src/database/models/version_item.rs
View File

@@ -1,4 +1,5 @@
use super::ids::*;
use super::loader_fields::VersionField;
use super::DatabaseError;
use crate::database::redis::RedisPool;
use crate::models::projects::{FileType, VersionStatus};
@@ -9,7 +10,7 @@ use std::cmp::Ordering;
use std::collections::HashMap;
use std::iter;
const VERSIONS_NAMESPACE: &str = "versions";
pub const VERSIONS_NAMESPACE: &str = "versions";
const VERSION_FILES_NAMESPACE: &str = "versions_files";
#[derive(Clone)]
@@ -22,8 +23,8 @@ pub struct VersionBuilder {
pub changelog: String,
pub files: Vec<VersionFileBuilder>,
pub dependencies: Vec<DependencyBuilder>,
pub game_versions: Vec<GameVersionId>,
pub loaders: Vec<LoaderId>,
pub version_fields: Vec<VersionField>,
pub version_type: String,
pub featured: bool,
pub status: VersionStatus,
@@ -234,7 +235,6 @@ impl VersionBuilder {
let VersionBuilder {
dependencies,
loaders,
game_versions,
files,
version_id,
..
@@ -249,17 +249,13 @@ impl VersionBuilder {
.collect_vec();
LoaderVersion::insert_many(loader_versions, transaction).await?;
let game_version_versions = game_versions
.iter()
.map(|v| VersionVersion::new(*v, version_id))
.collect_vec();
VersionVersion::insert_many(game_version_versions, transaction).await?;
VersionField::insert_many(self.version_fields, transaction).await?;
Ok(self.version_id)
}
}
#[derive(derive_new::new)]
#[derive(derive_new::new, Serialize, Deserialize)]
pub struct LoaderVersion {
pub loader_id: LoaderId,
pub version_id: VersionId,
@@ -289,36 +285,6 @@ impl LoaderVersion {
}
}
#[derive(derive_new::new)]
pub struct VersionVersion {
pub game_version_id: GameVersionId,
pub joining_version_id: VersionId,
}
impl VersionVersion {
pub async fn insert_many(
items: Vec<Self>,
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<(), DatabaseError> {
let (game_version_ids, version_ids): (Vec<_>, Vec<_>) = items
.into_iter()
.map(|i| (i.game_version_id.0, i.joining_version_id.0))
.unzip();
sqlx::query!(
"
INSERT INTO game_versions_versions (game_version_id, joining_version_id)
SELECT * FROM UNNEST($1::integer[], $2::bigint[])
",
&game_version_ids[..],
&version_ids[..],
)
.execute(&mut **transaction)
.await?;
Ok(())
}
}
#[derive(Clone, Deserialize, Serialize, PartialEq, Eq)]
pub struct Version {
pub id: VersionId,
@@ -401,8 +367,8 @@ impl Version {
sqlx::query!(
"
DELETE FROM game_versions_versions gvv
WHERE gvv.joining_version_id = $1
DELETE FROM version_fields vf
WHERE vf.version_id = $1
",
id as VersionId,
)
@@ -494,14 +460,11 @@ impl Version {
.execute(&mut **transaction)
.await?;
crate::database::models::Project::update_game_versions(
crate::database::models::Project::clear_cache(
ProjectId(project_id.mod_id),
&mut *transaction,
)
.await?;
crate::database::models::Project::update_loaders(
ProjectId(project_id.mod_id),
&mut *transaction,
None,
None,
redis,
)
.await?;
@@ -559,19 +522,59 @@ impl Version {
SELECT v.id id, v.mod_id mod_id, v.author_id author_id, v.name version_name, v.version_number version_number,
v.changelog changelog, v.date_published date_published, v.downloads downloads,
v.version_type version_type, v.featured featured, v.status status, v.requested_status requested_status, v.ordering ordering,
JSONB_AGG(DISTINCT jsonb_build_object('version', gv.version, 'created', gv.created)) filter (where gv.version is not null) game_versions,
ARRAY_AGG(DISTINCT l.loader) filter (where l.loader is not null) loaders,
ARRAY_AGG(DISTINCT pt.name) filter (where pt.name is not null) project_types,
ARRAY_AGG(DISTINCT g.name) filter (where g.name is not null) games,
JSONB_AGG(DISTINCT jsonb_build_object('id', f.id, 'url', f.url, 'filename', f.filename, 'primary', f.is_primary, 'size', f.size, 'file_type', f.file_type)) filter (where f.id is not null) files,
JSONB_AGG(DISTINCT jsonb_build_object('algorithm', h.algorithm, 'hash', encode(h.hash, 'escape'), 'file_id', h.file_id)) filter (where h.hash is not null) hashes,
JSONB_AGG(DISTINCT jsonb_build_object('project_id', d.mod_dependency_id, 'version_id', d.dependency_id, 'dependency_type', d.dependency_type,'file_name', dependency_file_name)) filter (where d.dependency_type is not null) dependencies
JSONB_AGG(DISTINCT jsonb_build_object('project_id', d.mod_dependency_id, 'version_id', d.dependency_id, 'dependency_type', d.dependency_type,'file_name', dependency_file_name)) filter (where d.dependency_type is not null) dependencies,
JSONB_AGG(
DISTINCT jsonb_build_object(
'field_id', vf.field_id,
'int_value', vf.int_value,
'enum_value', vf.enum_value,
'string_value', vf.string_value
)
) filter (where vf.field_id is not null) version_fields,
JSONB_AGG(
DISTINCT jsonb_build_object(
'lf_id', lf.id,
'loader_name', l.loader,
'field', lf.field,
'field_type', lf.field_type,
'enum_type', lf.enum_type,
'min_val', lf.min_val,
'max_val', lf.max_val,
'optional', lf.optional
)
) filter (where lf.id is not null) loader_fields,
JSONB_AGG(
DISTINCT jsonb_build_object(
'id', lfev.id,
'enum_id', lfev.enum_id,
'value', lfev.value,
'ordering', lfev.ordering,
'created', lfev.created,
'metadata', lfev.metadata
)
) filter (where lfev.id is not null) loader_field_enum_values
FROM versions v
LEFT OUTER JOIN game_versions_versions gvv on v.id = gvv.joining_version_id
LEFT OUTER JOIN game_versions gv on gvv.game_version_id = gv.id
LEFT OUTER JOIN loaders_versions lv on v.id = lv.version_id
LEFT OUTER JOIN loaders l on lv.loader_id = l.id
LEFT OUTER JOIN loaders_project_types lpt on l.id = lpt.joining_loader_id
LEFT JOIN project_types pt on lpt.joining_project_type_id = pt.id
LEFT OUTER JOIN loaders_project_types_games lptg on l.id = lptg.loader_id AND pt.id = lptg.project_type_id
LEFT JOIN games g on lptg.game_id = g.id
LEFT OUTER JOIN files f on v.id = f.version_id
LEFT OUTER JOIN hashes h on f.id = h.file_id
LEFT OUTER JOIN dependencies d on v.id = d.dependent_id
LEFT OUTER JOIN version_fields vf on v.id = vf.version_id
LEFT OUTER JOIN loader_fields lf on vf.field_id = lf.id
LEFT OUTER JOIN loader_field_enums lfe on lf.enum_type = lfe.id
LEFT OUTER JOIN loader_field_enum_values lfev on lfe.id = lfev.enum_id
WHERE v.id = ANY($1)
GROUP BY v.id
ORDER BY v.ordering ASC NULLS LAST, v.date_published ASC;
@@ -664,24 +667,10 @@ impl Version {
files
},
game_versions: {
#[derive(Deserialize)]
struct GameVersion {
pub version: String,
pub created: DateTime<Utc>,
}
let mut game_versions: Vec<GameVersion> = serde_json::from_value(
v.game_versions.unwrap_or_default(),
)
.ok()
.unwrap_or_default();
game_versions.sort_by(|a, b| a.created.cmp(&b.created));
game_versions.into_iter().map(|x| x.version).collect()
},
version_fields: VersionField::from_query_json(v.id, v.loader_fields, v.version_fields, v.loader_field_enum_values),
loaders: v.loaders.unwrap_or_default(),
project_types: v.project_types.unwrap_or_default(),
games: v.games.unwrap_or_default(),
dependencies: serde_json::from_value(
v.dependencies.unwrap_or_default(),
)
@@ -751,7 +740,6 @@ impl Version {
.collect::<Vec<_>>(),
)
.await?;
for file in files {
if let Some(mut file) =
file.and_then(|x| serde_json::from_str::<Vec<SingleFile>>(&x).ok())
@@ -861,8 +849,10 @@ pub struct QueryVersion {
pub inner: Version,
pub files: Vec<QueryFile>,
pub game_versions: Vec<String>,
pub version_fields: Vec<VersionField>,
pub loaders: Vec<String>,
pub project_types: Vec<String>,
pub games: Vec<String>,
pub dependencies: Vec<QueryDependency>,
}