Search test + v3 (#731)

* search patch for accurate loader/gv filtering

* backup

* basic search test

* finished test

* incomplete commit; backing up

* Working multipat reroute backup

* working rough draft v3

* most tests passing

* works

* search v2 conversion

* added some tags.rs v2 conversions

* Worked through warnings, unwraps, prints

* refactors

* new search test

* version files changes fixes

* redesign to revs

* removed old caches

* removed games

* fmt clippy

* merge conflicts

* fmt, prepare

* moved v2 routes over to v3

* fixes; tests passing

* project type changes

* moved files over

* fmt, clippy, prepare, etc

* loaders to loader_fields, added tests

* fmt, clippy, prepare

* fixed sorting bug

* reversed back- wrong order for consistency

* fmt; clippy; prepare

---------

Co-authored-by: Jai A <jaiagr+gpg@pm.me>

src/models/mod.rs

@@ -1,16 +1,19 @@
-pub mod analytics;
-pub mod collections;
-pub mod error;
-pub mod ids;
-pub mod images;
-pub mod notifications;
-pub mod oauth_clients;
-pub mod organizations;
-pub mod pack;
-pub mod pats;
-pub mod projects;
-pub mod reports;
-pub mod sessions;
-pub mod teams;
-pub mod threads;
-pub mod users;
+pub mod v2;
+pub mod v3;
+
+pub use v3::analytics;
+pub use v3::collections;
+pub use v3::error;
+pub use v3::ids;
+pub use v3::images;
+pub use v3::notifications;
+pub use v3::oauth_clients;
+pub use v3::organizations;
+pub use v3::pack;
+pub use v3::pats;
+pub use v3::projects;
+pub use v3::reports;
+pub use v3::sessions;
+pub use v3::teams;
+pub use v3::threads;
+pub use v3::users;

src/models/v2/mod.rs

@@ -0,0 +1,2 @@
+// Legacy models from V2, where its useful to keep the struct for rerouting/conversion
+pub mod projects;

src/models/v2/projects.rs

@@ -0,0 +1,307 @@
+use super::super::ids::OrganizationId;
+use super::super::teams::TeamId;
+use super::super::users::UserId;
+use crate::database::models::legacy_loader_fields::MinecraftGameVersion;
+use crate::database::models::{version_item, DatabaseError};
+use crate::database::redis::RedisPool;
+use crate::models::ids::{ProjectId, VersionId};
+use crate::models::projects::{
+    Dependency, DonationLink, GalleryItem, License, Loader, ModeratorMessage, MonetizationStatus,
+    Project, ProjectStatus, Version, VersionFile, VersionStatus, VersionType,
+};
+use crate::models::threads::ThreadId;
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+
+/// A project returned from the API
+#[derive(Serialize, Deserialize, Clone)]
+pub struct LegacyProject {
+    /// Relevant V2 fields- these were removed or modfified in V3,
+    /// and are now part of the dynamic fields system
+    /// The support range for the client project*
+    pub client_side: LegacySideType,
+    /// The support range for the server project
+    pub server_side: LegacySideType,
+    /// A list of game versions this project supports
+    pub game_versions: Vec<String>,
+
+    // All other fields are the same as V3
+    // If they change, or their constituent types change, we may need to
+    // add a new struct for them here.
+    pub id: ProjectId,
+    pub slug: Option<String>,
+    pub project_type: String,
+    pub team: TeamId,
+    pub organization: Option<OrganizationId>,
+    pub title: String,
+    pub description: String,
+    pub body: String,
+    pub body_url: Option<String>,
+    pub published: DateTime<Utc>,
+    pub updated: DateTime<Utc>,
+    pub approved: Option<DateTime<Utc>>,
+    pub queued: Option<DateTime<Utc>>,
+    pub status: ProjectStatus,
+    pub requested_status: Option<ProjectStatus>,
+    pub moderator_message: Option<ModeratorMessage>,
+    pub license: License,
+    pub downloads: u32,
+    pub followers: u32,
+    pub categories: Vec<String>,
+    pub additional_categories: Vec<String>,
+    pub loaders: Vec<String>,
+    pub versions: Vec<VersionId>,
+    pub icon_url: Option<String>,
+    pub issues_url: Option<String>,
+    pub source_url: Option<String>,
+    pub wiki_url: Option<String>,
+    pub discord_url: Option<String>,
+    pub donation_urls: Option<Vec<DonationLink>>,
+    pub gallery: Vec<GalleryItem>,
+    pub color: Option<u32>,
+    pub thread_id: ThreadId,
+    pub monetization_status: MonetizationStatus,
+}
+
+impl LegacyProject {
+    // Convert from a standard V3 project to a V2 project
+    // Requires any queried versions to be passed in, to get access to certain version fields contained within.
+    // - This can be any version, because the fields are ones that used to be on the project itself.
+    // - Its conceivable that certain V3 projects that have many different ones may not have the same fields on all of them.
+    // TODO: Should this return an error instead for v2 users?
+    // It's safe to use a db version_item for this as the only info is side types, game versions, and loader fields (for loaders), which used to be public on project anyway.
+    pub fn from(data: Project, versions_item: Option<version_item::QueryVersion>) -> Self {
+        let mut client_side = LegacySideType::Unknown;
+        let mut server_side = LegacySideType::Unknown;
+        let mut game_versions = Vec::new();
+
+        // V2 versions only have one project type- v3 versions can rarely have multiple.
+        // We'll just use the first one.
+        let mut project_type = data.project_types.get(0).cloned().unwrap_or_default();
+        let mut loaders = data.loaders;
+
+        if let Some(versions_item) = versions_item {
+            client_side = versions_item
+                .version_fields
+                .iter()
+                .find(|f| f.field_name == "client_side")
+                .and_then(|f| {
+                    Some(LegacySideType::from_string(
+                        f.value.serialize_internal().as_str()?,
+                    ))
+                })
+                .unwrap_or(LegacySideType::Unknown);
+            server_side = versions_item
+                .version_fields
+                .iter()
+                .find(|f| f.field_name == "server_side")
+                .and_then(|f| {
+                    Some(LegacySideType::from_string(
+                        f.value.serialize_internal().as_str()?,
+                    ))
+                })
+                .unwrap_or(LegacySideType::Unknown);
+            game_versions = versions_item
+                .version_fields
+                .iter()
+                .find(|f| f.field_name == "game_versions")
+                .and_then(|f| MinecraftGameVersion::try_from_version_field(f).ok())
+                .map(|v| v.into_iter().map(|v| v.version).collect())
+                .unwrap_or(Vec::new());
+
+            // - if loader is mrpack, this is a modpack
+            // the loaders are whatever the corresponding loader fields are
+            if versions_item.loaders == vec!["mrpack".to_string()] {
+                project_type = "modpack".to_string();
+                if let Some(mrpack_loaders) = versions_item
+                    .version_fields
+                    .iter()
+                    .find(|f| f.field_name == "mrpack_loaders")
+                {
+                    loaders = mrpack_loaders.value.as_strings();
+                }
+            }
+        }
+
+        Self {
+            id: data.id,
+            slug: data.slug,
+            project_type,
+            team: data.team,
+            organization: data.organization,
+            title: data.title,
+            description: data.description,
+            body: data.body,
+            body_url: data.body_url,
+            published: data.published,
+            updated: data.updated,
+            approved: data.approved,
+            queued: data.queued,
+            status: data.status,
+            requested_status: data.requested_status,
+            moderator_message: data.moderator_message,
+            license: data.license,
+            downloads: data.downloads,
+            followers: data.followers,
+            categories: data.categories,
+            additional_categories: data.additional_categories,
+            loaders,
+            versions: data.versions,
+            icon_url: data.icon_url,
+            issues_url: data.issues_url,
+            source_url: data.source_url,
+            wiki_url: data.wiki_url,
+            discord_url: data.discord_url,
+            donation_urls: data.donation_urls,
+            gallery: data.gallery,
+            color: data.color,
+            thread_id: data.thread_id,
+            monetization_status: data.monetization_status,
+            client_side,
+            server_side,
+            game_versions,
+        }
+    }
+
+    // Because from needs a version_item, this is a helper function to get many from one db query.
+    pub async fn from_many<'a, E>(
+        data: Vec<Project>,
+        exec: E,
+        redis: &RedisPool,
+    ) -> Result<Vec<Self>, DatabaseError>
+    where
+        E: sqlx::Executor<'a, Database = sqlx::Postgres>,
+    {
+        let version_ids: Vec<_> = data
+            .iter()
+            .filter_map(|p| p.versions.get(0).map(|i| (*i).into()))
+            .collect();
+        let example_versions = version_item::Version::get_many(&version_ids, exec, redis).await?;
+        let mut legacy_projects = Vec::new();
+        for project in data {
+            let version_item = example_versions
+                .iter()
+                .find(|v| v.inner.project_id == project.id.into())
+                .cloned();
+            let project = LegacyProject::from(project, version_item);
+            legacy_projects.push(project);
+        }
+        Ok(legacy_projects)
+    }
+}
+
+#[derive(Serialize, Deserialize, Clone, Debug, Eq, PartialEq)]
+#[serde(rename_all = "kebab-case")]
+pub enum LegacySideType {
+    Required,
+    Optional,
+    Unsupported,
+    Unknown,
+}
+
+impl std::fmt::Display for LegacySideType {
+    fn fmt(&self, fmt: &mut std::fmt::Formatter) -> std::fmt::Result {
+        write!(fmt, "{}", self.as_str())
+    }
+}
+
+impl LegacySideType {
+    // These are constant, so this can remove unneccessary allocations (`to_string`)
+    pub fn as_str(&self) -> &'static str {
+        match self {
+            LegacySideType::Required => "required",
+            LegacySideType::Optional => "optional",
+            LegacySideType::Unsupported => "unsupported",
+            LegacySideType::Unknown => "unknown",
+        }
+    }
+
+    pub fn from_string(string: &str) -> LegacySideType {
+        match string {
+            "required" => LegacySideType::Required,
+            "optional" => LegacySideType::Optional,
+            "unsupported" => LegacySideType::Unsupported,
+            _ => LegacySideType::Unknown,
+        }
+    }
+}
+
+/// A specific version of a project
+#[derive(Serialize, Deserialize, Clone)]
+pub struct LegacyVersion {
+    /// Relevant V2 fields- these were removed or modfified in V3,
+    /// and are now part of the dynamic fields system
+    /// A list of game versions this project supports
+    pub game_versions: Vec<String>,
+    /// A list of loaders this project supports
+    pub loaders: Vec<Loader>,
+
+    // TODO: remove this once we have v3 testing, as this is a v3 field and tests for it should be isolated to v3
+    pub ordering: Option<i32>,
+
+    pub id: VersionId,
+    pub project_id: ProjectId,
+    pub author_id: UserId,
+    pub featured: bool,
+    pub name: String,
+    pub version_number: String,
+    pub changelog: String,
+    pub changelog_url: Option<String>,
+    pub date_published: DateTime<Utc>,
+    pub downloads: u32,
+    pub version_type: VersionType,
+    pub status: VersionStatus,
+    pub requested_status: Option<VersionStatus>,
+    pub files: Vec<VersionFile>,
+    pub dependencies: Vec<Dependency>,
+}
+
+impl From<Version> for LegacyVersion {
+    fn from(data: Version) -> Self {
+        let mut game_versions = Vec::new();
+        if let Some(value) = data.fields.get("game_versions").and_then(|v| v.as_array()) {
+            for gv in value {
+                if let Some(game_version) = gv.as_str() {
+                    game_versions.push(game_version.to_string());
+                }
+            }
+        }
+
+        // - if loader is mrpack, this is a modpack
+        // the v2 loaders are whatever the corresponding loader fields are
+        let mut loaders = data.loaders.into_iter().map(|l| l.0).collect::<Vec<_>>();
+        if loaders == vec!["mrpack".to_string()] {
+            if let Some((_, mrpack_loaders)) = data
+                .fields
+                .into_iter()
+                .find(|(key, _)| key == "mrpack_loaders")
+            {
+                if let Ok(mrpack_loaders) = serde_json::from_value(mrpack_loaders) {
+                    loaders = mrpack_loaders;
+                }
+            }
+        }
+        let loaders = loaders.into_iter().map(Loader).collect::<Vec<_>>();
+
+        Self {
+            id: data.id,
+            project_id: data.project_id,
+            author_id: data.author_id,
+            featured: data.featured,
+            name: data.name,
+            version_number: data.version_number,
+            changelog: data.changelog,
+            changelog_url: data.changelog_url,
+            date_published: data.date_published,
+            downloads: data.downloads,
+            version_type: data.version_type,
+            status: data.status,
+            requested_status: data.requested_status,
+            files: data.files,
+            dependencies: data.dependencies,
+            game_versions,
+            ordering: data.ordering,
+            loaders,
+        }
+    }
+}

src/models/v3/mod.rs

@@ -0,0 +1,16 @@
+pub mod analytics;
+pub mod collections;
+pub mod error;
+pub mod ids;
+pub mod images;
+pub mod notifications;
+pub mod oauth_clients;
+pub mod organizations;
+pub mod pack;
+pub mod pats;
+pub mod projects;
+pub mod reports;
+pub mod sessions;
+pub mod teams;
+pub mod threads;
+pub mod users;

src/models/v3/pats.rs

@@ -132,9 +132,7 @@ impl Scopes {
     }
 
     pub fn parse_from_oauth_scopes(scopes: &str) -> Result<Scopes, bitflags::parser::ParseError> {
-        let scopes = scopes
-            .replace(['+', ' '], "|")
-            .replace("%20", "|");
+        let scopes = scopes.replace(['+', ' '], "|").replace("%20", "|");
         bitflags::parser::from_str(&scopes)
     }
 

src/models/v3/projects.rs

@@ -1,3 +1,5 @@
+use std::collections::HashMap;
+
 use super::ids::{Base62Id, OrganizationId};
 use super::teams::TeamId;
 use super::users::UserId;
@@ -27,8 +29,10 @@ pub struct Project {
     pub id: ProjectId,
     /// The slug of a project, used for vanity URLs
     pub slug: Option<String>,
-    /// The project type of the project
-    pub project_type: String,
+    /// The aggregated project typs of the versions of this project
+    pub project_types: Vec<String>,
+    /// The aggregated games of the versions of this project
+    pub games: Vec<String>,
     /// The team of people that has ownership of this project.
     pub team: TeamId,
     /// The optional organization of people that have ownership of this project.
@@ -66,11 +70,6 @@ pub struct Project {
     /// The license of this project
     pub license: License,
 
-    /// The support range for the client project*
-    pub client_side: SideType,
-    /// The support range for the server project
-    pub server_side: SideType,
-
     /// The total number of downloads the project has had.
     pub downloads: u32,
     /// The total number of followers this project has accumulated
@@ -81,8 +80,6 @@ pub struct Project {
 
     /// A list of the categories that the project is in.
     pub additional_categories: Vec<String>,
-    /// A list of game versions this project supports
-    pub game_versions: Vec<String>,
     /// A list of loaders this project supports
     pub loaders: Vec<String>,
 
@@ -120,7 +117,8 @@ impl From<QueryProject> for Project {
         Self {
             id: m.id.into(),
             slug: m.slug,
-            project_type: data.project_type,
+            project_types: data.project_types,
+            games: data.games,
             team: m.team_id.into(),
             organization: m.organization_id.map(|i| i.into()),
             title: m.title,
@@ -162,13 +160,10 @@ impl From<QueryProject> for Project {
                 },
                 url: m.license_url,
             },
-            client_side: data.client_side,
-            server_side: data.server_side,
             downloads: m.downloads as u32,
             followers: m.follows as u32,
             categories: data.categories,
             additional_categories: data.additional_categories,
-            game_versions: m.game_versions,
             loaders: m.loaders,
             versions: data.versions.into_iter().map(|v| v.into()).collect(),
             icon_url: m.icon_url,
@@ -462,11 +457,14 @@ pub struct Version {
     pub author_id: UserId,
     /// Whether the version is featured or not
     pub featured: bool,
-
     /// The name of this version
     pub name: String,
     /// The version number. Ideally will follow semantic versioning
     pub version_number: String,
+    /// Project types for which this version is compatible with, extracted from Loader
+    pub project_types: Vec<String>,
+    /// Games for which this version is compatible with, extracted from Loader/Project types
+    pub games: Vec<String>,
     /// The changelog for this version of the project.
     pub changelog: String,
     /// A link to the changelog for this version of the project. Deprecated, always None
@@ -487,26 +485,40 @@ pub struct Version {
     pub files: Vec<VersionFile>,
     /// A list of projects that this version depends on.
     pub dependencies: Vec<Dependency>,
-    /// A list of versions of Minecraft that this version of the project supports.
-    pub game_versions: Vec<GameVersion>,
+
     /// The loaders that this version works on
     pub loaders: Vec<Loader>,
     /// Ordering override, lower is returned first
     pub ordering: Option<i32>,
+
+    // All other fields are loader-specific VersionFields
+    // These are flattened during serialization
+    #[serde(deserialize_with = "skip_nulls")]
+    #[serde(flatten)]
+    pub fields: HashMap<String, serde_json::Value>,
+}
+
+pub fn skip_nulls<'de, D>(deserializer: D) -> Result<HashMap<String, serde_json::Value>, D::Error>
+where
+    D: serde::Deserializer<'de>,
+{
+    let mut map = HashMap::deserialize(deserializer)?;
+    map.retain(|_, v: &mut serde_json::Value| !v.is_null());
+    Ok(map)
 }
 
 impl From<QueryVersion> for Version {
     fn from(data: QueryVersion) -> Version {
         let v = data.inner;
-
         Version {
             id: v.id.into(),
             project_id: v.project_id.into(),
             author_id: v.author_id.into(),
-
             featured: v.featured,
             name: v.name,
             version_number: v.version_number,
+            project_types: data.project_types,
+            games: data.games,
             changelog: v.changelog,
             changelog_url: None,
             date_published: v.date_published,
@@ -543,8 +555,14 @@ impl From<QueryVersion> for Version {
                     dependency_type: DependencyType::from_string(d.dependency_type.as_str()),
                 })
                 .collect(),
-            game_versions: data.game_versions.into_iter().map(GameVersion).collect(),
             loaders: data.loaders.into_iter().map(Loader).collect(),
+            // Only add the internal component of the field for display
+            // "ie": "game_versions",["1.2.3"] instead of "game_versions",ArrayEnum(...)
+            fields: data
+                .version_fields
+                .into_iter()
+                .map(|vf| (vf.field_name, vf.value.serialize_internal()))
+                .collect(),
         }
     }
 }
@@ -658,7 +676,7 @@ pub struct VersionFile {
 
 /// A dendency which describes what versions are required, break support, or are optional to the
 /// version's functionality
-#[derive(Serialize, Deserialize, Clone)]
+#[derive(Serialize, Deserialize, Clone, Debug)]
 pub struct Dependency {
     /// The specific version id that the dependency uses
     pub version_id: Option<VersionId>,
@@ -670,7 +688,7 @@ pub struct Dependency {
     pub dependency_type: DependencyType,
 }
 
-#[derive(Serialize, Deserialize, Copy, Clone, Eq, PartialEq)]
+#[derive(Serialize, Deserialize, Copy, Clone, Eq, PartialEq, Debug)]
 #[serde(rename_all = "lowercase")]
 pub enum VersionType {
     Release,
@@ -695,7 +713,7 @@ impl VersionType {
     }
 }
 
-#[derive(Serialize, Deserialize, Copy, Clone)]
+#[derive(Serialize, Deserialize, Copy, Clone, Debug)]
 #[serde(rename_all = "lowercase")]
 pub enum DependencyType {
     Required,
@@ -766,19 +784,14 @@ impl FileType {
     }
 }
 
-/// A specific version of Minecraft
-#[derive(Serialize, Deserialize, Clone, PartialEq, Eq)]
-#[serde(transparent)]
-pub struct GameVersion(pub String);
-
 /// A project loader
-#[derive(Serialize, Deserialize, Clone)]
+#[derive(Serialize, Deserialize, Clone, Debug, PartialEq, Eq)]
 #[serde(transparent)]
 pub struct Loader(pub String);
 
 // These fields must always succeed parsing; deserialize errors aren't
 // processed correctly (don't return JSON errors)
-#[derive(Serialize, Deserialize)]
+#[derive(Serialize, Deserialize, Debug)]
 pub struct SearchRequest {
     pub query: Option<String>,
     pub offset: Option<String>,
@@ -787,7 +800,7 @@ pub struct SearchRequest {
 
     pub new_filters: Option<String>,
 
-    // Deprecated values below. WILL BE REMOVED V3!
+    // TODO: Deprecated values below. WILL BE REMOVED V3!
     pub facets: Option<String>,
     pub filters: Option<String>,
     pub version: Option<String>,