eightequal/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Fixes missing plugin/datapack in search (#829)

Browse Source 
* fixes datapack/plugin issue

* fixes level

* server side searching; org projects

* total hits

* total hits fixes

---------

Co-authored-by: Geometrically <18202329+Geometrically@users.noreply.github.com>
This commit is contained in:
Wyatt Verchere
2024-01-07 13:36:26 -08:00
committed by GitHub
parent aee9b6a951
commit c3448033de
13 changed files with 217 additions and 337 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
src/models/v2/projects.rs
View File

@@ -70,6 +70,33 @@ pub struct LegacyProject {
}
impl LegacyProject {
// Returns visible v2 project_type and also 'og' selected project type
// These are often identical, but we want to display 'mod' for datapacks and plugins
// The latter can be used for further processing, such as determining side types of plugins
pub fn get_project_type(project_types: &[String]) -> (String, String) {
// V2 versions only have one project type- v3 versions can rarely have multiple.
// We'll prioritize 'modpack' first, and if neither are found, use the first one.
// If there are no project types, default to 'project'
let mut project_types = project_types.to_vec();
if project_types.contains(&"modpack".to_string()) {
project_types = vec!["modpack".to_string()];
}
let og_project_type = project_types
.first()
.cloned()
.unwrap_or("project".to_string()); // Default to 'project' if none are found
let project_type = if og_project_type == "datapack" || og_project_type == "plugin" {
// These are not supported in V2, so we'll just use 'mod' instead
"mod".to_string()
} else {
og_project_type.clone()
};
(project_type, og_project_type)
}
// Convert from a standard V3 project to a V2 project
// Requires any queried versions to be passed in, to get access to certain version fields contained within.
// - This can be any version, because the fields are ones that used to be on the project itself.
@@ -83,22 +110,8 @@ impl LegacyProject {
// V2 versions only have one project type- v3 versions can rarely have multiple.
// We'll prioritize 'modpack' first, and if neither are found, use the first one.
// If there are no project types, default to 'project'
let mut project_types = data.project_types;
if project_types.contains(&"modpack".to_string()) {
project_types = vec!["modpack".to_string()];
}
let og_project_type = project_types
.first()
.cloned()
.unwrap_or("project".to_string()); // Default to 'project' if none are found
let mut project_type = if og_project_type == "datapack" || og_project_type == "plugin" {
// These are not supported in V2, so we'll just use 'mod' instead
"mod".to_string()
} else {
og_project_type.clone()
};
let project_types = data.project_types;
let (mut project_type, og_project_type) = Self::get_project_type(&project_types);
let mut loaders = data.loaders;

6
src/models/v2/search.rs
View File

@@ -156,14 +156,16 @@ impl LegacyResultSearchProject {
impl LegacySearchResults {
pub fn from(search_results: crate::search::SearchResults) -> Self {
let limit = search_results.hits_per_page;
let offset = (search_results.page - 1) * limit;
Self {
hits: search_results
.hits
.into_iter()
.map(LegacyResultSearchProject::from)
.collect(),
offset: search_results.offset,
limit: search_results.limit,
offset,
limit,
total_hits: search_results.total_hits,
}
}

6
src/routes/v2/projects.rs
View File

@@ -78,8 +78,10 @@ pub async fn project_search(
})
.collect_vec();
// We will now convert side_types to their new boolean format
let facets = v2_reroute::convert_side_type_facets_v3(facets);
// These loaders speciically used to be combined with 'mod' to be a plugin, but now
// they are their own loader type. We will convert 'mod' to 'mod' OR 'plugin'
// as it essentially was before.
let facets = v2_reroute::convert_plugin_loaders_v3(facets);
Some(
facets

266
src/routes/v2_reroute.rs
View File

@@ -8,7 +8,6 @@ use actix_multipart::Multipart;
use actix_web::http::header::{HeaderMap, TryIntoHeaderPair};
use actix_web::HttpResponse;
use futures::{stream, Future, StreamExt};
use itertools::Itertools;
use serde_json::{json, Value};
pub async fn extract_ok_json<T>(response: HttpResponse) -> Result<T, HttpResponse>
@@ -152,90 +151,24 @@ pub fn convert_side_types_v3(
fields
}
// Convert search facets from V2 to V3
// Less trivial as we need to handle the case where one side is set and the other is not, which does not convert cleanly
pub fn convert_side_type_facets_v3(facets: Vec<Vec<Vec<String>>>) -> Vec<Vec<Vec<String>>> {
use LegacySideType::{Optional, Required, Unsupported};
let possible_side_types = [Required, Optional, Unsupported]; // Should not include Unknown
let mut v3_facets = vec![];
// Outer facets are joined by AND
for inner_facets in facets {
// Inner facets are joined by OR
// These may change as the inner facets are converted
// ie:
// for A v B v C, if A is converted to X^Y v Y^Z, then the new facets are X^Y v Y^Z v B v C
let mut new_inner_facets = vec![];
for inner_inner_facets in inner_facets {
// Inner inner facets are joined by AND
let mut client_side = None;
let mut server_side = None;
// Extract client_side and server_side facets, and remove them from the list
let inner_inner_facets = inner_inner_facets
.into_iter()
.filter_map(|facet| {
let val = match facet.split(':').nth(1) {
Some(val) => val,
None => return Some(facet.to_string()),
};
if facet.starts_with("client_side:") {
client_side = Some(LegacySideType::from_string(val));
None
} else if facet.starts_with("server_side:") {
server_side = Some(LegacySideType::from_string(val));
None
} else {
Some(facet.to_string())
}
})
.collect_vec();
// Depending on whether client_side and server_side are set, we can convert the facets to the new loader fields differently
let mut new_possibilities = match (client_side, server_side) {
// Both set or unset is a trivial case
(Some(client_side), Some(server_side)) => {
vec![convert_side_types_v3(client_side, server_side)
.into_iter()
.map(|(k, v)| format!("{}:{}", k, v))
.collect()]
}
(None, None) => vec![vec![]],
(Some(client_side), None) => possible_side_types
.iter()
.map(|server_side| {
convert_side_types_v3(client_side, *server_side)
.into_iter()
.map(|(k, v)| format!("{}:{}", k, v))
.unique()
.collect::<Vec<_>>()
})
.collect::<Vec<_>>(),
(None, Some(server_side)) => possible_side_types
.iter()
.map(|client_side| {
convert_side_types_v3(*client_side, server_side)
.into_iter()
.map(|(k, v)| format!("{}:{}", k, v))
.unique()
.collect::<Vec<_>>()
})
.collect::<Vec<_>>(),
};
// Add the new possibilities to the list
for new_possibility in &mut new_possibilities {
new_possibility.extend(inner_inner_facets.clone());
// Converts plugin loaders from v2 to v3
// Within every 1st and 2nd level (the ones allowed in v2), we convert every instance of:
// "project_type:mod" to "project_type:plugin" OR "project_type:mod"
pub fn convert_plugin_loaders_v3(facets: Vec<Vec<Vec<String>>>) -> Vec<Vec<Vec<String>>> {
facets
.into_iter()
.map(|inner_facets| {
if inner_facets == [["project_type:mod"]] {
vec![
vec!["project_type:plugin".to_string()],
vec!["project_type:datapack".to_string()],
vec!["project_type:mod".to_string()],
]
} else {
inner_facets
}
new_inner_facets.extend(new_possibilities);
}
v3_facets.push(new_inner_facets);
}
v3_facets
})
.collect::<Vec<_>>()
}
// Convert search facets from V3 back to v2
@@ -347,169 +280,4 @@ mod tests {
}
}
}
#[test]
fn convert_facets() {
let pre_facets = vec![
// Test combinations of both sides being set
vec![vec![
"client_side:required".to_string(),
"server_side:required".to_string(),
]],
vec![vec![
"client_side:required".to_string(),
"server_side:optional".to_string(),
]],
vec![vec![
"client_side:required".to_string(),
"server_side:unsupported".to_string(),
]],
vec![vec![
"client_side:optional".to_string(),
"server_side:required".to_string(),
]],
vec![vec![
"client_side:optional".to_string(),
"server_side:optional".to_string(),
]],
// Test multiple inner facets
vec![
vec![
"client_side:required".to_string(),
"server_side:required".to_string(),
],
vec![
"client_side:required".to_string(),
"server_side:optional".to_string(),
],
],
// Test additional fields
vec![
vec![
"random_field_test_1".to_string(),
"client_side:required".to_string(),
"server_side:required".to_string(),
],
vec![
"random_field_test_2".to_string(),
"client_side:required".to_string(),
"server_side:optional".to_string(),
],
],
// Test only one facet being set
vec![vec!["client_side:required".to_string()]],
];
let converted_facets = convert_side_type_facets_v3(pre_facets)
.into_iter()
.map(|x| {
x.into_iter()
.map(|mut y| {
y.sort();
y
})
.collect::<Vec<_>>()
})
.collect::<Vec<_>>();
let post_facets = vec![
vec![vec![
"singleplayer:true".to_string(),
"client_and_server:true".to_string(),
"client_only:false".to_string(),
"server_only:false".to_string(),
]],
vec![vec![
"singleplayer:true".to_string(),
"client_and_server:true".to_string(),
"client_only:true".to_string(),
"server_only:false".to_string(),
]],
vec![vec![
"singleplayer:true".to_string(),
"client_and_server:true".to_string(),
"client_only:true".to_string(),
"server_only:false".to_string(),
]],
vec![vec![
"singleplayer:true".to_string(),
"client_and_server:true".to_string(),
"client_only:false".to_string(),
"server_only:true".to_string(),
]],
vec![vec![
"singleplayer:true".to_string(),
"client_and_server:true".to_string(),
"client_only:true".to_string(),
"server_only:true".to_string(),
]],
vec![
vec![
"singleplayer:true".to_string(),
"client_and_server:true".to_string(),
"client_only:false".to_string(),
"server_only:false".to_string(),
],
vec![
"singleplayer:true".to_string(),
"client_and_server:true".to_string(),
"client_only:true".to_string(),
"server_only:false".to_string(),
],
],
vec![
vec![
"random_field_test_1".to_string(),
"singleplayer:true".to_string(),
"client_and_server:true".to_string(),
"client_only:false".to_string(),
"server_only:false".to_string(),
],
vec![
"random_field_test_2".to_string(),
"singleplayer:true".to_string(),
"client_and_server:true".to_string(),
"client_only:true".to_string(),
"server_only:false".to_string(),
],
],
// Test only one facet being set
// Iterates over all possible side types
vec![
// C: Required, S: Required
vec![
"singleplayer:true".to_string(),
"client_and_server:true".to_string(),
"client_only:false".to_string(),
"server_only:false".to_string(),
],
// C: Required, S: Optional
vec![
"singleplayer:true".to_string(),
"client_and_server:true".to_string(),
"client_only:true".to_string(),
"server_only:false".to_string(),
],
// C: Required, S: Unsupported
vec![
"singleplayer:true".to_string(),
"client_and_server:true".to_string(),
"client_only:true".to_string(),
"server_only:false".to_string(),
],
],
]
.into_iter()
.map(|x| {
x.into_iter()
.map(|mut y| {
y.sort();
y
})
.collect::<Vec<_>>()
})
.collect::<Vec<_>>();
assert_eq!(converted_facets, post_facets);
}
}

8
src/routes/v3/projects.rs
View File

@@ -896,8 +896,8 @@ pub async fn edit_project_categories(
#[derive(Serialize, Deserialize)]
pub struct ReturnSearchResults {
pub hits: Vec<Project>,
pub offset: usize,
pub limit: usize,
pub page: usize,
pub hits_per_page: usize,
pub total_hits: usize,
}
@@ -913,8 +913,8 @@ pub async fn project_search(
.into_iter()
.filter_map(Project::from_search)
.collect::<Vec<_>>(),
offset: results.offset,
limit: results.limit,
page: results.page,
hits_per_page: results.hits_per_page,
total_hits: results.total_hits,
};

45
src/search/indexing/local_import.rs
View File

@@ -6,22 +6,29 @@ use super::IndexingError;
use crate::database::models::{project_item, version_item, ProjectId, VersionId};
use crate::database::redis::RedisPool;
use crate::models;
use crate::models::v2::projects::LegacyProject;
use crate::routes::v2_reroute;
use crate::search::UploadSearchProject;
use sqlx::postgres::PgPool;
pub async fn get_all_ids(
pool: PgPool,
) -> Result<Vec<(VersionId, ProjectId, String)>, IndexingError> {
// TODO: Currently org owner is set to be considered owner. It may be worth considering
// adding a new facetable 'organization' field to the search index, and using that instead,
// and making owner to be optional.
let all_visible_ids: Vec<(VersionId, ProjectId, String)> = sqlx::query!(
"
SELECT v.id id, m.id mod_id, u.username owner_username
SELECT v.id id, m.id mod_id, COALESCE(u.username, ou.username) owner_username
FROM versions v
INNER JOIN mods m ON v.mod_id = m.id AND m.status = ANY($2)
INNER JOIN team_members tm ON tm.team_id = m.team_id AND tm.is_owner = TRUE AND tm.accepted = TRUE
INNER JOIN users u ON tm.user_id = u.id
LEFT JOIN team_members tm ON tm.team_id = m.team_id AND tm.is_owner = TRUE AND tm.accepted = TRUE
LEFT JOIN users u ON tm.user_id = u.id
LEFT JOIN organizations o ON o.id = m.organization_id
LEFT JOIN team_members otm ON otm.team_id = o.team_id AND otm.is_owner = TRUE AND otm.accepted = TRUE
LEFT JOIN users ou ON otm.user_id = ou.id
WHERE v.status != ANY($1)
GROUP BY v.id, m.id, u.id
GROUP BY v.id, m.id, u.username, ou.username
ORDER BY m.id DESC;
",
&*crate::models::projects::VersionStatus::iterator()
@@ -38,7 +45,8 @@ pub async fn get_all_ids(
Ok(e.right().map(|m| {
let project_id: ProjectId = ProjectId(m.mod_id);
let version_id: VersionId = VersionId(m.id);
(version_id, project_id, m.owner_username)
let owner_username = m.owner_username.unwrap_or_default();
(version_id, project_id, owner_username)
}))
})
.try_collect::<Vec<_>>()
@@ -114,7 +122,12 @@ pub async fn index_local(
categories.append(&mut additional_categories);
let version_fields = v.version_fields.clone();
let loader_fields = models::projects::from_duplicate_version_fields(version_fields);
let unvectorized_loader_fields = v
.version_fields
.iter()
.map(|vf| (vf.field_name.clone(), vf.value.serialize_internal()))
.collect();
let mut loader_fields = models::projects::from_duplicate_version_fields(version_fields);
let license = match m.inner.license.split(' ').next() {
Some(license) => license.to_string(),
None => m.inner.license.clone(),
@@ -158,6 +171,24 @@ pub async fn index_local(
categories.retain(|x| *x != "mrpack");
}
// SPECIAL BEHAVIOUR:
// For consitency with v2 searching, we manually input the
// client_side and server_side fields from the loader fields into
// separate loader fields.
// 'client_side' and 'server_side' remain supported by meilisearch even though they are no longer v3 fields.
let (_, v2_og_project_type) = LegacyProject::get_project_type(&v.project_types);
let (client_side, server_side) = v2_reroute::convert_side_types_v2(
&unvectorized_loader_fields,
Some(&v2_og_project_type),
);
if let Ok(client_side) = serde_json::to_value(client_side) {
loader_fields.insert("client_side".to_string(), vec![client_side]);
}
if let Ok(server_side) = serde_json::to_value(server_side) {
loader_fields.insert("server_side".to_string(), vec![server_side]);
}
let gallery = m
.gallery_items
.iter()

6
src/search/indexing/mod.rs
View File

@@ -382,6 +382,9 @@ const DEFAULT_DISPLAYED_ATTRIBUTES: &[&str] = &[
"singleplayer",
"client_and_server",
"mrpack_loaders",
// V2 legacy fields for logical consistency
"client_side",
"server_side",
// Non-searchable fields for filling out the Project model.
"license_url",
"monetization_status",
@@ -424,6 +427,9 @@ const DEFAULT_ATTRIBUTES_FOR_FACETING: &[&str] = &[
"singleplayer",
"client_and_server",
"mrpack_loaders",
// V2 legacy fields for logical consistency
"client_side",
"server_side",
];
const DEFAULT_SORTABLE_ATTRIBUTES: &[&str] =

21
src/search/mod.rs
View File

@@ -142,8 +142,8 @@ pub struct UploadSearchProject {
#[derive(Serialize, Deserialize, Debug)]
pub struct SearchResults {
pub hits: Vec<ResultSearchProject>,
pub offset: usize,
pub limit: usize,
pub page: usize,
pub hits_per_page: usize,
pub total_hits: usize,
}
@@ -212,7 +212,7 @@ pub async fn search_for_project(
) -> Result<SearchResults, SearchError> {
let client = Client::new(&*config.address, Some(&*config.key));
let offset = info.offset.as_deref().unwrap_or("0").parse()?;
let offset: usize = info.offset.as_deref().unwrap_or("0").parse()?;
let index = info.index.as_deref().unwrap_or("relevance");
let limit = info.limit.as_deref().unwrap_or("10").parse()?;
@@ -221,12 +221,15 @@ pub async fn search_for_project(
let mut filter_string = String::new();
// Convert offset and limit to page and hits_per_page
let hits_per_page = limit;
let page = offset / limit + 1;
let results = {
let mut query = meilisearch_index.search();
query
.with_limit(min(100, limit))
.with_offset(offset)
.with_page(page)
.with_hits_per_page(hits_per_page)
.with_query(info.query.as_deref().unwrap_or_default())
.with_sort(&sort.1);
@@ -312,8 +315,8 @@ pub async fn search_for_project(
Ok(SearchResults {
hits: results.hits.into_iter().map(|r| r.result).collect(),
offset: results.offset.unwrap_or_default(),
limit: results.limit.unwrap_or_default(),
total_hits: results.estimated_total_hits.unwrap_or_default(),
page: results.page.unwrap_or_default(),
hits_per_page: results.hits_per_page.unwrap_or_default(),
total_hits: results.total_hits.unwrap_or_default(),
})
}