From d2a66bb2b0ca9f8648ce3ee8a57e11422a9084e7 Mon Sep 17 00:00:00 2001 From: aecsocket Date: Wed, 10 Jun 2026 18:30:03 +0100 Subject: [PATCH] Allow searching by project dependencies (#6350) * Allow searching by project dependencies * change field name * use query macro --- ...6e84e9238dee422cc2b400cea0048a71e4c27.json | 47 ++++++++++++++ apps/labrinth/AGENTS.md | 1 + .../src/search/backend/typesense/mod.rs | 22 +++++-- apps/labrinth/src/search/indexing.rs | 64 +++++++++++++++++-- apps/labrinth/src/search/mod.rs | 19 ++++++ apps/labrinth/src/test/search.rs | 21 +++++- apps/labrinth/tests/search.rs | 42 +++++++++++- 7 files changed, 202 insertions(+), 14 deletions(-) create mode 100644 apps/labrinth/.sqlx/query-20cb8a3d45911a126aa17451b1e6e84e9238dee422cc2b400cea0048a71e4c27.json diff --git a/apps/labrinth/.sqlx/query-20cb8a3d45911a126aa17451b1e6e84e9238dee422cc2b400cea0048a71e4c27.json b/apps/labrinth/.sqlx/query-20cb8a3d45911a126aa17451b1e6e84e9238dee422cc2b400cea0048a71e4c27.json new file mode 100644 index 000000000..5d7941bfe --- /dev/null +++ b/apps/labrinth/.sqlx/query-20cb8a3d45911a126aa17451b1e6e84e9238dee422cc2b400cea0048a71e4c27.json @@ -0,0 +1,47 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT DISTINCT v.mod_id dependent_project_id, d.mod_dependency_id dependency_project_id,\n m.name dependency_name, m.slug dependency_slug, m.icon_url dependency_icon_url\n FROM versions v\n INNER JOIN dependencies d ON d.dependent_id = v.id\n INNER JOIN mods m ON m.id = d.mod_dependency_id\n WHERE v.mod_id = ANY($1)\n AND d.mod_dependency_id IS NOT NULL\n AND m.status = ANY($2)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "dependent_project_id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "dependency_project_id", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "dependency_name", + "type_info": "Varchar" + }, + { + "ordinal": 3, + "name": "dependency_slug", + "type_info": "Varchar" + }, + { + "ordinal": 4, + "name": "dependency_icon_url", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Int8Array", + "TextArray" + ] + }, + "nullable": [ + false, + true, + false, + true, + true + ] + }, + "hash": "20cb8a3d45911a126aa17451b1e6e84e9238dee422cc2b400cea0048a71e4c27" +} diff --git a/apps/labrinth/AGENTS.md b/apps/labrinth/AGENTS.md index dd00fb4cb..fd050808c 100644 --- a/apps/labrinth/AGENTS.md +++ b/apps/labrinth/AGENTS.md @@ -24,3 +24,4 @@ - `Authorization: Bearer mra_user` for a regular user - `Modrinth-Admin: feedbeef` as admin key - If some steps require you to create a project/mod or version for testing, ask the user to go into the web frontend and manually create a project/version +- When using `sqlx::query` etc. always use the macro form like `sqlx::query!` or `sqlx::query_scalar!` - never the plain function form. Avoid using `query_as!`. diff --git a/apps/labrinth/src/search/backend/typesense/mod.rs b/apps/labrinth/src/search/backend/typesense/mod.rs index f5c44b51a..5959d9578 100644 --- a/apps/labrinth/src/search/backend/typesense/mod.rs +++ b/apps/labrinth/src/search/backend/typesense/mod.rs @@ -325,16 +325,16 @@ impl TypesenseClient { filter_by: &str, ) -> Result<()> { let resp = self - .request( - Method::DELETE, - &format!( + .request( + Method::DELETE, + &format!( "/collections/{collection}/documents?filter_by={}&batch_size=1000", urlencoding::encode(filter_by) ), - ) - .send() - .await - .wrap_err("failed to DELETE Typesense documents by filter")?; + ) + .send() + .await + .wrap_err("failed to DELETE Typesense documents by filter")?; if resp.status() == reqwest::StatusCode::NOT_FOUND { return Ok(()); } @@ -478,6 +478,13 @@ impl SearchField { sort: false, optional: true, }, + SearchField::DependencyProjectIds => TypesenseFieldSpec { + path: "dependency_project_ids", + ty: "string[]", + facet: true, + sort: false, + optional: true, + }, } } } @@ -526,6 +533,7 @@ impl Typesense { json!({"name": "minecraft_java_server.verified_plays_2w", "type": "int64", "sort": true, "optional": true}), json!({"name": "minecraft_java_server.is_online", "type": "bool", "sort": true, "optional": true}), json!({"name": "minecraft_java_server.ping.data.players_online", "type": "int32", "sort": true, "optional": true}), + json!({"name": "dependencies", "type": "object[]", "optional": true}), ]; fields.extend(TYPESENSE_SEARCH_FIELDS.iter().cloned()); diff --git a/apps/labrinth/src/search/indexing.rs b/apps/labrinth/src/search/indexing.rs index ff2d73436..314f841f9 100644 --- a/apps/labrinth/src/search/indexing.rs +++ b/apps/labrinth/src/search/indexing.rs @@ -24,7 +24,7 @@ use crate::models::ids::ProjectId; use crate::models::projects::from_duplicate_version_fields; use crate::models::v2::projects::LegacyProject; use crate::routes::v2_reroute; -use crate::search::UploadSearchProject; +use crate::search::{SearchProjectDependency, UploadSearchProject}; use crate::util::error::Context; fn normalize_for_search(s: &str) -> String { @@ -65,6 +65,12 @@ pub async fn index_local( components: exp::ProjectSerial, } + let searchable_statuses = + crate::models::projects::ProjectStatus::iterator() + .filter(|x| x.is_searchable()) + .map(|x| x.to_string()) + .collect::>(); + let db_projects = sqlx::query!( r#" SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows, @@ -76,10 +82,7 @@ pub async fn index_local( ORDER BY m.id ASC LIMIT $2; "#, - &*crate::models::projects::ProjectStatus::iterator() - .filter(|x| x.is_searchable()) - .map(|x| x.to_string()) - .collect::>(), + &searchable_statuses, limit, cursor, ) @@ -118,6 +121,47 @@ pub async fn index_local( return Ok((vec![], i64::MAX)); }; + info!("Indexing local dependencies!"); + + let dependencies: DashMap> = + sqlx::query!( + " + SELECT DISTINCT v.mod_id dependent_project_id, d.mod_dependency_id dependency_project_id, + m.name dependency_name, m.slug dependency_slug, m.icon_url dependency_icon_url + FROM versions v + INNER JOIN dependencies d ON d.dependent_id = v.id + INNER JOIN mods m ON m.id = d.mod_dependency_id + WHERE v.mod_id = ANY($1) + AND d.mod_dependency_id IS NOT NULL + AND m.status = ANY($2) + ", + &project_ids, + &searchable_statuses, + ) + .fetch(pool) + .try_fold( + DashMap::new(), + |acc: DashMap>, m| { + if let Some(dependency_project_id) = m.dependency_project_id { + acc.entry(DBProjectId(m.dependent_project_id)) + .or_default() + .push(SearchProjectDependency { + project_id: ProjectId::from(DBProjectId( + dependency_project_id, + )) + .to_string(), + name: m.dependency_name, + slug: m.dependency_slug, + icon_url: m.dependency_icon_url, + }); + } + + async move { Ok(acc) } + }, + ) + .await + .wrap_err("failed to fetch project dependencies")?; + struct PartialGallery { url: String, featured: bool, @@ -346,6 +390,14 @@ pub async fn index_local( } else { (vec![], vec![]) }; + let dependencies = dependencies + .get(&project.id) + .map(|x| x.clone()) + .unwrap_or_default(); + let dependency_project_ids = dependencies + .iter() + .map(|dependency| dependency.project_id.clone()) + .collect::>(); if let Some(versions) = versions.remove(&project.id) { // Aggregated project loader fields @@ -486,6 +538,8 @@ pub async fn index_local( featured_gallery: featured_gallery.clone(), open_source, color: project.color.map(|x| x as u32), + dependency_project_ids: dependency_project_ids.clone(), + dependencies: dependencies.clone(), loader_fields, project_loader_fields: project_loader_fields.clone(), // 'loaders' is aggregate of all versions' loaders diff --git a/apps/labrinth/src/search/mod.rs b/apps/labrinth/src/search/mod.rs index 1fd208a8e..90c0f9e3c 100644 --- a/apps/labrinth/src/search/mod.rs +++ b/apps/labrinth/src/search/mod.rs @@ -195,6 +195,7 @@ pub enum SearchField { MinecraftJavaServerContentKind, MinecraftJavaServerContentSupportedGameVersions, MinecraftJavaServerPingData, + DependencyProjectIds, } #[derive(Debug, Error)] @@ -248,6 +249,10 @@ pub struct UploadSearchProject { pub version_published_timestamp: i64, pub open_source: bool, pub color: Option, + #[serde(default)] + pub dependency_project_ids: Vec, + #[serde(default)] + pub dependencies: Vec, // Hidden fields to get the Project model out of the search results. pub loaders: Vec, // Search uses loaders as categories- this is purely for the Project model. @@ -259,6 +264,14 @@ pub struct UploadSearchProject { pub loader_fields: HashMap>, } +#[derive(Serialize, Deserialize, Debug, Clone)] +pub struct SearchProjectDependency { + pub project_id: String, + pub name: String, + pub slug: Option, + pub icon_url: Option, +} + #[derive(Serialize, Deserialize, Debug)] pub struct SearchResults { pub hits: Vec, @@ -295,6 +308,10 @@ pub struct ResultSearchProject { pub gallery: Vec, pub featured_gallery: Option, pub color: Option, + #[serde(default)] + pub dependency_project_ids: Vec, + #[serde(default)] + pub dependencies: Vec, // Hidden fields to get the Project model out of the search results. pub loaders: Vec, // Search uses loaders as categories- this is purely for the Project model. @@ -332,6 +349,8 @@ impl From for ResultSearchProject { gallery: source.gallery, featured_gallery: source.featured_gallery, color: source.color, + dependency_project_ids: source.dependency_project_ids, + dependencies: source.dependencies, loaders: source.loaders, project_loader_fields: source.project_loader_fields, components: source.components, diff --git a/apps/labrinth/src/test/search.rs b/apps/labrinth/src/test/search.rs index 45ce34f7f..9159a5a14 100644 --- a/apps/labrinth/src/test/search.rs +++ b/apps/labrinth/src/test/search.rs @@ -214,12 +214,31 @@ pub async fn setup_search_projects( USER_USER_PAT, ) .await; + let project_1 = api + .get_project_deserialized_common( + &format!("{test_name}-searchable-project-1"), + USER_USER_PAT, + ) + .await; + let modify_json = serde_json::from_value(json!([ + { + "op": "add", + "path": "/dependencies", + "value": [ + { + "project_id": project_1.id, + "dependency_type": "required" + } + ] + } + ])) + .unwrap(); api.add_public_version( project_7.id, "1.0.0", TestFile::build_random_jar(), None, - None, + Some(modify_json), USER_USER_PAT, ) .await; diff --git a/apps/labrinth/tests/search.rs b/apps/labrinth/tests/search.rs index 81a761a54..9bcba177c 100644 --- a/apps/labrinth/tests/search.rs +++ b/apps/labrinth/tests/search.rs @@ -4,7 +4,7 @@ use common::database::*; use common::dummy_data::DUMMY_CATEGORIES; -use ariadne::ids::base62_impl::parse_base62; +use ariadne::ids::base62_impl::{parse_base62, to_base62}; use common::environment::TestEnvironment; use common::environment::with_test_environment; use common::search::setup_search_projects; @@ -29,6 +29,12 @@ async fn search_projects() { let api = &test_env.api; let test_name = test_env.db.database_name.clone(); + let dependency_project_id = id_conversion + .iter() + .find_map(|(project_id, test_id)| { + (*test_id == 1).then_some(to_base62(*project_id)) + }) + .unwrap(); // Pairs of: // 1. vec of search facets @@ -83,6 +89,12 @@ async fn search_projects() { json!([["categories:fabric"], ["project_types:modpack"]]), vec![4], ), + ( + json!([[format!( + "dependency_project_ids:{dependency_project_id}" + )]]), + vec![7], + ), ]; // TODO: versions, game versions // Untested: @@ -123,6 +135,34 @@ async fn search_projects() { } }) .await; + + let projects = api + .search_deserialized( + Some(&format!("&{test_name}")), + Some(json!([[format!( + "dependency_project_ids:{dependency_project_id}" + )]])), + USER_USER_PAT, + ) + .await; + assert_eq!(projects.total_hits, 1); + assert_eq!(projects.hits[0].dependency_project_ids.len(), 1); + assert_eq!( + projects.hits[0].dependency_project_ids[0], + dependency_project_id + ); + assert_eq!(projects.hits[0].dependencies.len(), 1); + assert_eq!( + projects.hits[0].dependencies[0].project_id, + dependency_project_id + ); + assert!( + projects.hits[0].dependencies[0] + .slug + .as_ref() + .unwrap() + .contains("searchable-project-1") + ); }, ) .await;