Allow searching by project dependencies (#6350)

* Allow searching by project dependencies

* change field name

* use query macro

apps/labrinth/.sqlx/query-20cb8a3d45911a126aa17451b1e6e84e9238dee422cc2b400cea0048a71e4c27.json

@@ -0,0 +1,47 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n            SELECT DISTINCT v.mod_id dependent_project_id, d.mod_dependency_id dependency_project_id,\n                m.name dependency_name, m.slug dependency_slug, m.icon_url dependency_icon_url\n            FROM versions v\n            INNER JOIN dependencies d ON d.dependent_id = v.id\n            INNER JOIN mods m ON m.id = d.mod_dependency_id\n            WHERE v.mod_id = ANY($1)\n                AND d.mod_dependency_id IS NOT NULL\n                AND m.status = ANY($2)\n            ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "dependent_project_id",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 1,
+        "name": "dependency_project_id",
+        "type_info": "Int8"
+      },
+      {
+        "ordinal": 2,
+        "name": "dependency_name",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 3,
+        "name": "dependency_slug",
+        "type_info": "Varchar"
+      },
+      {
+        "ordinal": 4,
+        "name": "dependency_icon_url",
+        "type_info": "Varchar"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8Array",
+        "TextArray"
+      ]
+    },
+    "nullable": [
+      false,
+      true,
+      false,
+      true,
+      true
+    ]
+  },
+  "hash": "20cb8a3d45911a126aa17451b1e6e84e9238dee422cc2b400cea0048a71e4c27"
+}

apps/labrinth/AGENTS.md

@@ -24,3 +24,4 @@
   - `Authorization: Bearer mra_user` for a regular user
   - `Modrinth-Admin: feedbeef` as admin key
 - If some steps require you to create a project/mod or version for testing, ask the user to go into the web frontend and manually create a project/version
+- When using `sqlx::query` etc. always use the macro form like `sqlx::query!` or `sqlx::query_scalar!` - never the plain function form. Avoid using `query_as!`.

apps/labrinth/src/search/backend/typesense/mod.rs

@@ -325,16 +325,16 @@ impl TypesenseClient {
         filter_by: &str,
     ) -> Result<()> {
         let resp = self
-			.request(
+            .request(
-				Method::DELETE,
+                Method::DELETE,
-				&format!(
+                &format!(
 					"/collections/{collection}/documents?filter_by={}&batch_size=1000",
 					urlencoding::encode(filter_by)
 				),
-			)
+            )
-			.send()
+            .send()
-			.await
+            .await
-			.wrap_err("failed to DELETE Typesense documents by filter")?;
+            .wrap_err("failed to DELETE Typesense documents by filter")?;
         if resp.status() == reqwest::StatusCode::NOT_FOUND {
             return Ok(());
         }
@@ -478,6 +478,13 @@ impl SearchField {
                 sort: false,
                 optional: true,
             },
+            SearchField::DependencyProjectIds => TypesenseFieldSpec {
+                path: "dependency_project_ids",
+                ty: "string[]",
+                facet: true,
+                sort: false,
+                optional: true,
+            },
         }
     }
 }
@@ -526,6 +533,7 @@ impl Typesense {
             json!({"name": "minecraft_java_server.verified_plays_2w", "type": "int64", "sort": true, "optional": true}),
             json!({"name": "minecraft_java_server.is_online", "type": "bool", "sort": true, "optional": true}),
             json!({"name": "minecraft_java_server.ping.data.players_online", "type": "int32", "sort": true, "optional": true}),
+            json!({"name": "dependencies", "type": "object[]", "optional": true}),
         ];
         fields.extend(TYPESENSE_SEARCH_FIELDS.iter().cloned());
 

apps/labrinth/src/search/indexing.rs

@@ -24,7 +24,7 @@ use crate::models::ids::ProjectId;
 use crate::models::projects::from_duplicate_version_fields;
 use crate::models::v2::projects::LegacyProject;
 use crate::routes::v2_reroute;
-use crate::search::UploadSearchProject;
+use crate::search::{SearchProjectDependency, UploadSearchProject};
 use crate::util::error::Context;
 
 fn normalize_for_search(s: &str) -> String {
@@ -65,6 +65,12 @@ pub async fn index_local(
         components: exp::ProjectSerial,
     }
 
+    let searchable_statuses =
+        crate::models::projects::ProjectStatus::iterator()
+            .filter(|x| x.is_searchable())
+            .map(|x| x.to_string())
+            .collect::<Vec<String>>();
+
     let db_projects = sqlx::query!(
         r#"
         SELECT m.id id, m.name name, m.summary summary, m.downloads downloads, m.follows follows,
@@ -76,10 +82,7 @@ pub async fn index_local(
         ORDER BY m.id ASC
         LIMIT $2;
         "#,
-        &*crate::models::projects::ProjectStatus::iterator()
+        &searchable_statuses,
-        .filter(|x| x.is_searchable())
-        .map(|x| x.to_string())
-        .collect::<Vec<String>>(),
         limit,
         cursor,
     )
@@ -118,6 +121,47 @@ pub async fn index_local(
         return Ok((vec![], i64::MAX));
     };
 
+    info!("Indexing local dependencies!");
+
+    let dependencies: DashMap<DBProjectId, Vec<SearchProjectDependency>> =
+		sqlx::query!(
+			"
+            SELECT DISTINCT v.mod_id dependent_project_id, d.mod_dependency_id dependency_project_id,
+                m.name dependency_name, m.slug dependency_slug, m.icon_url dependency_icon_url
+            FROM versions v
+            INNER JOIN dependencies d ON d.dependent_id = v.id
+            INNER JOIN mods m ON m.id = d.mod_dependency_id
+            WHERE v.mod_id = ANY($1)
+                AND d.mod_dependency_id IS NOT NULL
+                AND m.status = ANY($2)
+            ",
+			&project_ids,
+			&searchable_statuses,
+		)
+		.fetch(pool)
+		.try_fold(
+			DashMap::new(),
+			|acc: DashMap<DBProjectId, Vec<SearchProjectDependency>>, m| {
+				if let Some(dependency_project_id) = m.dependency_project_id {
+					acc.entry(DBProjectId(m.dependent_project_id))
+						.or_default()
+						.push(SearchProjectDependency {
+							project_id: ProjectId::from(DBProjectId(
+								dependency_project_id,
+							))
+							.to_string(),
+							name: m.dependency_name,
+							slug: m.dependency_slug,
+							icon_url: m.dependency_icon_url,
+						});
+				}
+
+				async move { Ok(acc) }
+			},
+		)
+		.await
+		.wrap_err("failed to fetch project dependencies")?;
+
     struct PartialGallery {
         url: String,
         featured: bool,
@@ -346,6 +390,14 @@ pub async fn index_local(
             } else {
                 (vec![], vec![])
             };
+        let dependencies = dependencies
+            .get(&project.id)
+            .map(|x| x.clone())
+            .unwrap_or_default();
+        let dependency_project_ids = dependencies
+            .iter()
+            .map(|dependency| dependency.project_id.clone())
+            .collect::<Vec<_>>();
 
         if let Some(versions) = versions.remove(&project.id) {
             // Aggregated project loader fields
@@ -486,6 +538,8 @@ pub async fn index_local(
                     featured_gallery: featured_gallery.clone(),
                     open_source,
                     color: project.color.map(|x| x as u32),
+                    dependency_project_ids: dependency_project_ids.clone(),
+                    dependencies: dependencies.clone(),
                     loader_fields,
                     project_loader_fields: project_loader_fields.clone(),
                     // 'loaders' is aggregate of all versions' loaders

apps/labrinth/src/search/mod.rs

@@ -195,6 +195,7 @@ pub enum SearchField {
     MinecraftJavaServerContentKind,
     MinecraftJavaServerContentSupportedGameVersions,
     MinecraftJavaServerPingData,
+    DependencyProjectIds,
 }
 
 #[derive(Debug, Error)]
@@ -248,6 +249,10 @@ pub struct UploadSearchProject {
     pub version_published_timestamp: i64,
     pub open_source: bool,
     pub color: Option<u32>,
+    #[serde(default)]
+    pub dependency_project_ids: Vec<String>,
+    #[serde(default)]
+    pub dependencies: Vec<SearchProjectDependency>,
 
     // Hidden fields to get the Project model out of the search results.
     pub loaders: Vec<String>, // Search uses loaders as categories- this is purely for the Project model.
@@ -259,6 +264,14 @@ pub struct UploadSearchProject {
     pub loader_fields: HashMap<String, Vec<serde_json::Value>>,
 }
 
+#[derive(Serialize, Deserialize, Debug, Clone)]
+pub struct SearchProjectDependency {
+    pub project_id: String,
+    pub name: String,
+    pub slug: Option<String>,
+    pub icon_url: Option<String>,
+}
+
 #[derive(Serialize, Deserialize, Debug)]
 pub struct SearchResults {
     pub hits: Vec<ResultSearchProject>,
@@ -295,6 +308,10 @@ pub struct ResultSearchProject {
     pub gallery: Vec<String>,
     pub featured_gallery: Option<String>,
     pub color: Option<u32>,
+    #[serde(default)]
+    pub dependency_project_ids: Vec<String>,
+    #[serde(default)]
+    pub dependencies: Vec<SearchProjectDependency>,
 
     // Hidden fields to get the Project model out of the search results.
     pub loaders: Vec<String>, // Search uses loaders as categories- this is purely for the Project model.
@@ -332,6 +349,8 @@ impl From<UploadSearchProject> for ResultSearchProject {
             gallery: source.gallery,
             featured_gallery: source.featured_gallery,
             color: source.color,
+            dependency_project_ids: source.dependency_project_ids,
+            dependencies: source.dependencies,
             loaders: source.loaders,
             project_loader_fields: source.project_loader_fields,
             components: source.components,

apps/labrinth/src/test/search.rs

@@ -214,12 +214,31 @@ pub async fn setup_search_projects(
             USER_USER_PAT,
         )
         .await;
+    let project_1 = api
+        .get_project_deserialized_common(
+            &format!("{test_name}-searchable-project-1"),
+            USER_USER_PAT,
+        )
+        .await;
+    let modify_json = serde_json::from_value(json!([
+        {
+            "op": "add",
+            "path": "/dependencies",
+            "value": [
+                {
+                    "project_id": project_1.id,
+                    "dependency_type": "required"
+                }
+            ]
+        }
+    ]))
+    .unwrap();
     api.add_public_version(
         project_7.id,
         "1.0.0",
         TestFile::build_random_jar(),
         None,
-        None,
+        Some(modify_json),
         USER_USER_PAT,
     )
     .await;

apps/labrinth/tests/search.rs

@@ -4,7 +4,7 @@ use common::database::*;
 
 use common::dummy_data::DUMMY_CATEGORIES;
 
-use ariadne::ids::base62_impl::parse_base62;
+use ariadne::ids::base62_impl::{parse_base62, to_base62};
 use common::environment::TestEnvironment;
 use common::environment::with_test_environment;
 use common::search::setup_search_projects;
@@ -29,6 +29,12 @@ async fn search_projects() {
 
             let api = &test_env.api;
             let test_name = test_env.db.database_name.clone();
+            let dependency_project_id = id_conversion
+                .iter()
+                .find_map(|(project_id, test_id)| {
+                    (*test_id == 1).then_some(to_base62(*project_id))
+                })
+                .unwrap();
 
             // Pairs of:
             // 1. vec of search facets
@@ -83,6 +89,12 @@ async fn search_projects() {
                     json!([["categories:fabric"], ["project_types:modpack"]]),
                     vec![4],
                 ),
+                (
+                    json!([[format!(
+                        "dependency_project_ids:{dependency_project_id}"
+                    )]]),
+                    vec![7],
+                ),
             ];
             // TODO: versions, game versions
             // Untested:
@@ -123,6 +135,34 @@ async fn search_projects() {
                     }
                 })
                 .await;
+
+            let projects = api
+                .search_deserialized(
+                    Some(&format!("&{test_name}")),
+                    Some(json!([[format!(
+                        "dependency_project_ids:{dependency_project_id}"
+                    )]])),
+                    USER_USER_PAT,
+                )
+                .await;
+            assert_eq!(projects.total_hits, 1);
+            assert_eq!(projects.hits[0].dependency_project_ids.len(), 1);
+            assert_eq!(
+                projects.hits[0].dependency_project_ids[0],
+                dependency_project_id
+            );
+            assert_eq!(projects.hits[0].dependencies.len(), 1);
+            assert_eq!(
+                projects.hits[0].dependencies[0].project_id,
+                dependency_project_id
+            );
+            assert!(
+                projects.hits[0].dependencies[0]
+                    .slug
+                    .as_ref()
+                    .unwrap()
+                    .contains("searchable-project-1")
+            );
         },
     )
     .await;