eightequal/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Search overhaul (#771)

Browse Source 
* started work; switching context

* working!

* fmt clippy prepare

* fixes

* fixes

* revs

* merge fixes

* changed comments

* merge issues
This commit is contained in:
Wyatt Verchere
2023-12-03 06:27:12 -08:00
committed by GitHub
parent a70df067bc
commit b2be4a7d67
18 changed files with 882 additions and 585 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
tests/common/api_v3/project.rs
View File

@@ -8,7 +8,10 @@ use actix_web::{
use async_trait::async_trait;
use bytes::Bytes;
use chrono::{DateTime, Utc};
use labrinth::{models::projects::Project, search::SearchResults, util::actix::AppendsMultipart};
use labrinth::{
models::projects::Project, routes::v3::projects::ReturnSearchResults,
util::actix::AppendsMultipart,
};
use rust_decimal::Decimal;
use serde_json::json;
@@ -222,7 +225,7 @@ impl ApiV3 {
query: Option<&str>,
facets: Option<serde_json::Value>,
pat: &str,
) -> SearchResults {
) -> ReturnSearchResults {
let query_field = if let Some(query) = query {
format!("&query={}", urlencoding::encode(query))
} else {

1
tests/common/mod.rs
View File

@@ -12,6 +12,7 @@ pub mod environment;
pub mod pats;
pub mod permissions;
pub mod scopes;
pub mod search;
// Testing equivalent to 'setup' function, producing a LabrinthConfig
// If making a test, you should probably use environment::TestEnvironment::build() (which calls this)

212
tests/common/search.rs Normal file
View File

@@ -0,0 +1,212 @@
#![allow(dead_code)]
use std::{collections::HashMap, sync::Arc};
use serde_json::json;
use crate::common::{
api_common::{Api, ApiProject, ApiVersion},
database::{FRIEND_USER_PAT, MOD_USER_PAT, USER_USER_PAT},
dummy_data::{TestFile, DUMMY_CATEGORIES},
};
use super::{api_v3::ApiV3, environment::TestEnvironment};
pub async fn setup_search_projects(test_env: &TestEnvironment<ApiV3>) -> Arc<HashMap<u64, u64>> {
// Test setup and dummy data
let api = &test_env.api;
let test_name = test_env.db.database_name.clone();
// Add dummy projects of various categories for searchability
let mut project_creation_futures = vec![];
let create_async_future =
|id: u64, pat: &'static str, is_modpack: bool, modify_json: Option<json_patch::Patch>| {
let slug = format!("{test_name}-searchable-project-{id}");
let jar = if is_modpack {
TestFile::build_random_mrpack()
} else {
TestFile::build_random_jar()
};
async move {
// Add a project- simple, should work.
let req = api.add_public_project(&slug, Some(jar), modify_json, pat);
let (project, _) = req.await;
// Approve, so that the project is searchable
let resp = api
.edit_project(
&project.id.to_string(),
json!({
"status": "approved"
}),
MOD_USER_PAT,
)
.await;
assert_eq!(resp.status(), 204);
(project.id.0, id)
}
};
// Test project 0
let id = 0;
let modify_json = serde_json::from_value(json!([
{ "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[4..6] },
{ "op": "add", "path": "/initial_versions/0/server_only", "value": true },
{ "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" },
]))
.unwrap();
project_creation_futures.push(create_async_future(
id,
USER_USER_PAT,
false,
Some(modify_json),
));
// Test project 1
let id = 1;
let modify_json = serde_json::from_value(json!([
{ "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[0..2] },
{ "op": "add", "path": "/initial_versions/0/client_only", "value": false },
]))
.unwrap();
project_creation_futures.push(create_async_future(
id,
USER_USER_PAT,
false,
Some(modify_json),
));
// Test project 2
let id = 2;
let modify_json = serde_json::from_value(json!([
{ "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[0..2] },
{ "op": "add", "path": "/initial_versions/0/server_only", "value": true },
{ "op": "add", "path": "/name", "value": "Mysterious Project" },
]))
.unwrap();
project_creation_futures.push(create_async_future(
id,
USER_USER_PAT,
false,
Some(modify_json),
));
// Test project 3
let id = 3;
let modify_json = serde_json::from_value(json!([
{ "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[0..3] },
{ "op": "add", "path": "/initial_versions/0/server_only", "value": true },
{ "op": "add", "path": "/initial_versions/0/game_versions", "value": ["1.20.4"] },
{ "op": "add", "path": "/name", "value": "Mysterious Project" },
{ "op": "add", "path": "/license_id", "value": "LicenseRef-All-Rights-Reserved" },
]))
.unwrap();
project_creation_futures.push(create_async_future(
id,
FRIEND_USER_PAT,
false,
Some(modify_json),
));
// Test project 4
let id = 4;
let modify_json = serde_json::from_value(json!([
{ "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[0..3] },
{ "op": "add", "path": "/initial_versions/0/client_only", "value": false },
{ "op": "add", "path": "/initial_versions/0/game_versions", "value": ["1.20.5"] },
]))
.unwrap();
project_creation_futures.push(create_async_future(
id,
USER_USER_PAT,
true,
Some(modify_json),
));
// Test project 5
let id = 5;
let modify_json = serde_json::from_value(json!([
{ "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[5..6] },
{ "op": "add", "path": "/initial_versions/0/client_only", "value": false },
{ "op": "add", "path": "/initial_versions/0/game_versions", "value": ["1.20.5"] },
{ "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" },
]))
.unwrap();
project_creation_futures.push(create_async_future(
id,
USER_USER_PAT,
false,
Some(modify_json),
));
// Test project 6
let id = 6;
let modify_json = serde_json::from_value(json!([
{ "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[5..6] },
{ "op": "add", "path": "/initial_versions/0/client_only", "value": false },
{ "op": "add", "path": "/initial_versions/0/server_only", "value": true },
{ "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" },
]))
.unwrap();
project_creation_futures.push(create_async_future(
id,
FRIEND_USER_PAT,
false,
Some(modify_json),
));
// Test project 7 (testing the search bug)
// This project has an initial private forge version that is 1.20.3, and a fabric 1.20.5 version.
// This means that a search for fabric + 1.20.3 or forge + 1.20.5 should not return this project.
let id = 7;
let modify_json = serde_json::from_value(json!([
{ "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[5..6] },
{ "op": "add", "path": "/initial_versions/0/client_only", "value": false },
{ "op": "add", "path": "/initial_versions/0/server_only", "value": true },
{ "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" },
{ "op": "add", "path": "/initial_versions/0/loaders", "value": ["forge"] },
{ "op": "add", "path": "/initial_versions/0/game_versions", "value": ["1.20.2"] },
]))
.unwrap();
project_creation_futures.push(create_async_future(
id,
USER_USER_PAT,
false,
Some(modify_json),
));
// Await all project creation
// Returns a mapping of:
// project id -> test id
let id_conversion: Arc<HashMap<u64, u64>> = Arc::new(
futures::future::join_all(project_creation_futures)
.await
.into_iter()
.collect(),
);
// Create a second version for project 7
let project_7 = api
.get_project_deserialized_common(
&format!("{test_name}-searchable-project-7"),
USER_USER_PAT,
)
.await;
api.add_public_version(
project_7.id,
"1.0.0",
TestFile::build_random_jar(),
None,
None,
USER_USER_PAT,
)
.await;
// Forcibly reset the search index
let resp = api.reset_search_index().await;
assert_eq!(resp.status(), 204);
id_conversion
}

46
tests/project.rs
View File

@@ -1,3 +1,5 @@
use std::collections::HashMap;
use actix_http::StatusCode;
use actix_web::test;
use bytes::Bytes;
@@ -9,10 +11,11 @@ use common::dummy_data::DUMMY_CATEGORIES;
use common::environment::{with_test_environment, with_test_environment_all, TestEnvironment};
use common::permissions::{PermissionsTest, PermissionsTestContext};
use common::search::setup_search_projects;
use futures::StreamExt;
use labrinth::database::models::project_item::{PROJECTS_NAMESPACE, PROJECTS_SLUGS_NAMESPACE};
use labrinth::models::ids::base62_impl::parse_base62;
use labrinth::models::projects::ProjectId;
use labrinth::models::projects::{Project, ProjectId};
use labrinth::models::teams::ProjectPermissions;
use labrinth::util::actix::{AppendsMultipart, MultipartSegment, MultipartSegmentData};
use serde_json::json;
@@ -1093,6 +1096,47 @@ async fn project_permissions_consistency_test() {
.await;
}
#[actix_rt::test]
async fn align_search_projects() {
// Test setup and dummy data
with_test_environment(Some(10), |test_env: TestEnvironment<ApiV3>| async move {
setup_search_projects(&test_env).await;
let api = &test_env.api;
let test_name = test_env.db.database_name.clone();
let projects = api
.search_deserialized(
Some(&test_name),
Some(json!([["categories:fabric"]])),
USER_USER_PAT,
)
.await;
for mut project in projects.hits {
let project_model = api
.get_project(&project.id.to_string(), USER_USER_PAT)
.await;
let mut project_model: Project = test::read_body_json(project_model).await;
// Body/description is huge- don't store it in search, so it's OK if they differ here
// (Search should return "")
project_model.description = "".into();
// Aggregate project loader fields will not match exactly,
// because the search will only return the matching version, whereas the project returns the aggregate.
// So, we remove them from both.
project_model.fields = HashMap::new();
project.fields = HashMap::new();
let project_model = serde_json::to_value(project_model).unwrap();
let searched_project_serialized = serde_json::to_value(project).unwrap();
assert_eq!(project_model, searched_project_serialized);
}
})
.await
}
// Route tests:
// TODO: Missing routes on projects
// TODO: using permissions/scopes, can we SEE projects existence that we are not allowed to? (ie 401 instead of 404)

209
tests/search.rs
View File

@@ -1,19 +1,13 @@
use common::api_v3::ApiV3;
use common::database::*;
use common::dummy_data::TestFile;
use common::dummy_data::DUMMY_CATEGORIES;
use common::environment::with_test_environment;
use common::environment::TestEnvironment;
use common::search::setup_search_projects;
use futures::stream::StreamExt;
use labrinth::models::ids::base62_impl::parse_base62;
use serde_json::json;
use std::collections::HashMap;
use std::sync::Arc;
use crate::common::api_common::Api;
use crate::common::api_common::ApiProject;
use crate::common::api_common::ApiVersion;
mod common;
@@ -24,199 +18,11 @@ mod common;
async fn search_projects() {
// Test setup and dummy data
with_test_environment(Some(10), |test_env: TestEnvironment<ApiV3>| async move {
let id_conversion = setup_search_projects(&test_env).await;
let api = &test_env.api;
let test_name = test_env.db.database_name.clone();
// Add dummy projects of various categories for searchability
let mut project_creation_futures = vec![];
let create_async_future =
|id: u64,
pat: &'static str,
is_modpack: bool,
modify_json: Option<json_patch::Patch>| {
let slug = format!("{test_name}-searchable-project-{id}");
let jar = if is_modpack {
TestFile::build_random_mrpack()
} else {
TestFile::build_random_jar()
};
async move {
// Add a project- simple, should work.
let req = api.add_public_project(&slug, Some(jar), modify_json, pat);
let (project, _) = req.await;
// Approve, so that the project is searchable
let resp = api
.edit_project(
&project.id.to_string(),
json!({
"status": "approved"
}),
MOD_USER_PAT,
)
.await;
assert_eq!(resp.status(), 204);
(project.id.0, id)
}
};
// Test project 0
let id = 0;
let modify_json = serde_json::from_value(json!([
{ "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[4..6] },
{ "op": "add", "path": "/initial_versions/0/server_only", "value": true },
{ "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" },
]))
.unwrap();
project_creation_futures.push(create_async_future(
id,
USER_USER_PAT,
false,
Some(modify_json),
));
// Test project 1
let id = 1;
let modify_json = serde_json::from_value(json!([
{ "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[0..2] },
{ "op": "add", "path": "/initial_versions/0/client_only", "value": false },
]))
.unwrap();
project_creation_futures.push(create_async_future(
id,
USER_USER_PAT,
false,
Some(modify_json),
));
// Test project 2
let id = 2;
let modify_json = serde_json::from_value(json!([
{ "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[0..2] },
{ "op": "add", "path": "/initial_versions/0/server_only", "value": true },
{ "op": "add", "path": "/name", "value": "Mysterious Project" },
]))
.unwrap();
project_creation_futures.push(create_async_future(
id,
USER_USER_PAT,
false,
Some(modify_json),
));
// Test project 3
let id = 3;
let modify_json = serde_json::from_value(json!([
{ "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[0..3] },
{ "op": "add", "path": "/initial_versions/0/server_only", "value": true },
{ "op": "add", "path": "/initial_versions/0/game_versions", "value": ["1.20.4"] },
{ "op": "add", "path": "/name", "value": "Mysterious Project" },
{ "op": "add", "path": "/license_id", "value": "LicenseRef-All-Rights-Reserved" },
]))
.unwrap();
project_creation_futures.push(create_async_future(
id,
FRIEND_USER_PAT,
false,
Some(modify_json),
));
// Test project 4
let id = 4;
let modify_json = serde_json::from_value(json!([
{ "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[0..3] },
{ "op": "add", "path": "/initial_versions/0/client_only", "value": false },
{ "op": "add", "path": "/initial_versions/0/game_versions", "value": ["1.20.5"] },
]))
.unwrap();
project_creation_futures.push(create_async_future(
id,
USER_USER_PAT,
true,
Some(modify_json),
));
// Test project 5
let id = 5;
let modify_json = serde_json::from_value(json!([
{ "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[5..6] },
{ "op": "add", "path": "/initial_versions/0/client_only", "value": false },
{ "op": "add", "path": "/initial_versions/0/game_versions", "value": ["1.20.5"] },
{ "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" },
]))
.unwrap();
project_creation_futures.push(create_async_future(
id,
USER_USER_PAT,
false,
Some(modify_json),
));
// Test project 6
let id = 6;
let modify_json = serde_json::from_value(json!([
{ "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[5..6] },
{ "op": "add", "path": "/initial_versions/0/client_only", "value": false },
{ "op": "add", "path": "/initial_versions/0/server_only", "value": true },
{ "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" },
]))
.unwrap();
project_creation_futures.push(create_async_future(
id,
FRIEND_USER_PAT,
false,
Some(modify_json),
));
// Test project 7 (testing the search bug)
// This project has an initial private forge version that is 1.20.3, and a fabric 1.20.5 version.
// This means that a search for fabric + 1.20.3 or forge + 1.20.5 should not return this project.
let id = 7;
let modify_json = serde_json::from_value(json!([
{ "op": "add", "path": "/categories", "value": DUMMY_CATEGORIES[5..6] },
{ "op": "add", "path": "/initial_versions/0/client_only", "value": false },
{ "op": "add", "path": "/initial_versions/0/server_only", "value": true },
{ "op": "add", "path": "/license_id", "value": "LGPL-3.0-or-later" },
{ "op": "add", "path": "/initial_versions/0/loaders", "value": ["forge"] },
{ "op": "add", "path": "/initial_versions/0/game_versions", "value": ["1.20.2"] },
]))
.unwrap();
project_creation_futures.push(create_async_future(
id,
USER_USER_PAT,
false,
Some(modify_json),
));
// Await all project creation
// Returns a mapping of:
// project id -> test id
let id_conversion: Arc<HashMap<u64, u64>> = Arc::new(
futures::future::join_all(project_creation_futures)
.await
.into_iter()
.collect(),
);
// Create a second version for project 7
let project_7 = api
.get_project_deserialized_common(
&format!("{test_name}-searchable-project-7"),
USER_USER_PAT,
)
.await;
api.add_public_version(
project_7.id,
"1.0.0",
TestFile::build_random_jar(),
None,
None,
USER_USER_PAT,
)
.await;
// Pairs of:
// 1. vec of search facets
// 2. expected project ids to be returned by this search
@@ -277,10 +83,6 @@ async fn search_projects() {
// - modified_timestamp (not varied)
// TODO: multiple different project types test
// Forcibly reset the search index
let resp = api.reset_search_index().await;
assert_eq!(resp.status(), 204);
// Test searches
let stream = futures::stream::iter(pairs);
stream
@@ -294,10 +96,11 @@ async fn search_projects() {
let mut found_project_ids: Vec<u64> = projects
.hits
.into_iter()
.map(|p| id_conversion[&parse_base62(&p.project_id).unwrap()])
.map(|p| id_conversion[&p.id.0])
.collect();
expected_project_ids.sort();
found_project_ids.sort();
println!("Facets: {:?}", facets);
assert_eq!(found_project_ids, expected_project_ids);
}
})

45
tests/v2/search.rs
View File

@@ -305,6 +305,51 @@ async fn search_projects() {
}
})
.await;
// A couple additional tests for the saerch type returned, making sure it is properly translated back
let client_side_required = api
.search_deserialized(
Some(&test_name),
Some(json!([["client_side:required"]])),
USER_USER_PAT,
)
.await;
for hit in client_side_required.hits {
assert_eq!(hit.client_side, "required".to_string());
}
let server_side_required = api
.search_deserialized(
Some(&test_name),
Some(json!([["server_side:required"]])),
USER_USER_PAT,
)
.await;
for hit in server_side_required.hits {
assert_eq!(hit.server_side, "required".to_string());
}
let client_side_unsupported = api
.search_deserialized(
Some(&test_name),
Some(json!([["client_side:unsupported"]])),
USER_USER_PAT,
)
.await;
for hit in client_side_unsupported.hits {
assert_eq!(hit.client_side, "unsupported".to_string());
}
let game_versions = api
.search_deserialized(
Some(&test_name),
Some(json!([["versions:1.20.5"]])),
USER_USER_PAT,
)
.await;
for hit in game_versions.hits {
assert_eq!(hit.versions, vec!["1.20.5".to_string()]);
}
})
.await;
}