eightequal/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Fix clippy errors + lint, use turbo CI

Browse Source
This commit is contained in:
Jai A
2024-10-18 16:07:35 -07:00
parent 663ab83b08
commit 8dd955563e
186 changed files with 10615 additions and 6433 deletions
Download Patch File Download Diff File Expand all files Collapse all files

142
apps/labrinth/src/search/indexing/local_import.rs
View File

@@ -7,10 +7,12 @@ use std::collections::HashMap;
use super::IndexingError;
use crate::database::models::loader_fields::{
QueryLoaderField, QueryLoaderFieldEnumValue, QueryVersionField, VersionField,
QueryLoaderField, QueryLoaderFieldEnumValue, QueryVersionField,
VersionField,
};
use crate::database::models::{
LoaderFieldEnumId, LoaderFieldEnumValueId, LoaderFieldId, ProjectId, VersionId,
LoaderFieldEnumId, LoaderFieldEnumValueId, LoaderFieldId, ProjectId,
VersionId,
};
use crate::models::projects::from_duplicate_version_fields;
use crate::models::v2::projects::LegacyProject;
@@ -18,7 +20,9 @@ use crate::routes::v2_reroute;
use crate::search::UploadSearchProject;
use sqlx::postgres::PgPool;
pub async fn index_local(pool: &PgPool) -> Result<Vec<UploadSearchProject>, IndexingError> {
pub async fn index_local(
pool: &PgPool,
) -> Result<Vec<UploadSearchProject>, IndexingError> {
info!("Indexing local projects!");
// todo: loaders, project type, game versions
@@ -190,24 +194,25 @@ pub async fn index_local(pool: &PgPool) -> Result<Vec<UploadSearchProject>, Inde
info!("Getting all loader field enum values!");
let loader_field_enum_values: Vec<QueryLoaderFieldEnumValue> = sqlx::query!(
"
let loader_field_enum_values: Vec<QueryLoaderFieldEnumValue> =
sqlx::query!(
"
SELECT DISTINCT id, enum_id, value, ordering, created, metadata
FROM loader_field_enum_values lfev
ORDER BY enum_id, ordering, created DESC
"
)
.fetch(pool)
.map_ok(|m| QueryLoaderFieldEnumValue {
id: LoaderFieldEnumValueId(m.id),
enum_id: LoaderFieldEnumId(m.enum_id),
value: m.value,
ordering: m.ordering,
created: m.created,
metadata: m.metadata,
})
.try_collect()
.await?;
)
.fetch(pool)
.map_ok(|m| QueryLoaderFieldEnumValue {
id: LoaderFieldEnumValueId(m.id),
enum_id: LoaderFieldEnumId(m.enum_id),
value: m.value,
ordering: m.ordering,
created: m.created,
metadata: m.metadata,
})
.try_collect()
.await?;
info!("Indexing loaders, project types!");
let mut uploads = Vec::new();
@@ -218,17 +223,20 @@ pub async fn index_local(pool: &PgPool) -> Result<Vec<UploadSearchProject>, Inde
count += 1;
info!("projects index prog: {count}/{total_len}");
let owner = if let Some((_, org_owner)) = mods_org_owners.remove(&project.id) {
org_owner
} else if let Some((_, team_owner)) = mods_team_owners.remove(&project.id) {
team_owner
} else {
println!(
"org owner not found for project {} id: {}!",
project.name, project.id.0
);
continue;
};
let owner =
if let Some((_, org_owner)) = mods_org_owners.remove(&project.id) {
org_owner
} else if let Some((_, team_owner)) =
mods_team_owners.remove(&project.id)
{
team_owner
} else {
println!(
"org owner not found for project {} id: {}!",
project.name, project.id.0
);
continue;
};
let license = match project.license.split(' ').next() {
Some(license) => license.to_string(),
@@ -291,7 +299,8 @@ pub async fn index_local(pool: &PgPool) -> Result<Vec<UploadSearchProject>, Inde
&loader_field_enum_values,
true,
);
let project_loader_fields = from_duplicate_version_fields(aggregated_version_fields);
let project_loader_fields =
from_duplicate_version_fields(aggregated_version_fields);
// aggregated project loaders
let project_loaders = versions
@@ -308,9 +317,12 @@ pub async fn index_local(pool: &PgPool) -> Result<Vec<UploadSearchProject>, Inde
);
let unvectorized_loader_fields = version_fields
.iter()
.map(|vf| (vf.field_name.clone(), vf.value.serialize_internal()))
.map(|vf| {
(vf.field_name.clone(), vf.value.serialize_internal())
})
.collect();
let mut loader_fields = from_duplicate_version_fields(version_fields);
let mut loader_fields =
from_duplicate_version_fields(version_fields);
let project_types = version.project_types;
let mut version_loaders = version.loaders;
@@ -347,22 +359,28 @@ pub async fn index_local(pool: &PgPool) -> Result<Vec<UploadSearchProject>, Inde
// client_side and server_side fields from the loader fields into
// separate loader fields.
// 'client_side' and 'server_side' remain supported by meilisearch even though they are no longer v3 fields.
let (_, v2_og_project_type) = LegacyProject::get_project_type(&project_types);
let (client_side, server_side) = v2_reroute::convert_side_types_v2(
&unvectorized_loader_fields,
Some(&v2_og_project_type),
);
let (_, v2_og_project_type) =
LegacyProject::get_project_type(&project_types);
let (client_side, server_side) =
v2_reroute::convert_side_types_v2(
&unvectorized_loader_fields,
Some(&v2_og_project_type),
);
if let Ok(client_side) = serde_json::to_value(client_side) {
loader_fields.insert("client_side".to_string(), vec![client_side]);
loader_fields
.insert("client_side".to_string(), vec![client_side]);
}
if let Ok(server_side) = serde_json::to_value(server_side) {
loader_fields.insert("server_side".to_string(), vec![server_side]);
loader_fields
.insert("server_side".to_string(), vec![server_side]);
}
let usp = UploadSearchProject {
version_id: crate::models::ids::VersionId::from(version.id).to_string(),
project_id: crate::models::ids::ProjectId::from(project.id).to_string(),
version_id: crate::models::ids::VersionId::from(version.id)
.to_string(),
project_id: crate::models::ids::ProjectId::from(project.id)
.to_string(),
name: project.name.clone(),
summary: project.summary.clone(),
categories: categories.clone(),
@@ -470,34 +488,36 @@ async fn index_versions(
.await?;
// Get version fields
let version_fields: DashMap<VersionId, Vec<QueryVersionField>> = sqlx::query!(
"
let version_fields: DashMap<VersionId, Vec<QueryVersionField>> =
sqlx::query!(
"
SELECT version_id, field_id, int_value, enum_value, string_value
FROM version_fields
WHERE version_id = ANY($1)
",
&all_version_ids,
)
.fetch(pool)
.try_fold(
DashMap::new(),
|acc: DashMap<VersionId, Vec<QueryVersionField>>, m| {
let qvf = QueryVersionField {
version_id: VersionId(m.version_id),
field_id: LoaderFieldId(m.field_id),
int_value: m.int_value,
enum_value: m.enum_value.map(LoaderFieldEnumValueId),
string_value: m.string_value,
};
&all_version_ids,
)
.fetch(pool)
.try_fold(
DashMap::new(),
|acc: DashMap<VersionId, Vec<QueryVersionField>>, m| {
let qvf = QueryVersionField {
version_id: VersionId(m.version_id),
field_id: LoaderFieldId(m.field_id),
int_value: m.int_value,
enum_value: m.enum_value.map(LoaderFieldEnumValueId),
string_value: m.string_value,
};
acc.entry(VersionId(m.version_id)).or_default().push(qvf);
async move { Ok(acc) }
},
)
.await?;
acc.entry(VersionId(m.version_id)).or_default().push(qvf);
async move { Ok(acc) }
},
)
.await?;
// Convert to partial versions
let mut res_versions: HashMap<ProjectId, Vec<PartialVersion>> = HashMap::new();
let mut res_versions: HashMap<ProjectId, Vec<PartialVersion>> =
HashMap::new();
for (project_id, version_ids) in versions.iter() {
for version_id in version_ids {
// Extract version-specific data fetched

36
apps/labrinth/src/search/indexing/mod.rs
View File

@@ -44,7 +44,9 @@ pub async fn remove_documents(
for index in indexes {
index
.delete_documents(&ids.iter().map(|x| to_base62(x.0)).collect::<Vec<_>>())
.delete_documents(
&ids.iter().map(|x| to_base62(x.0)).collect::<Vec<_>>(),
)
.await?;
}
@@ -70,11 +72,13 @@ pub async fn index_projects(
let indices = get_indexes_for_indexing(config, true).await?;
let all_loader_fields =
crate::database::models::loader_fields::LoaderField::get_fields_all(&pool, &redis)
.await?
.into_iter()
.map(|x| x.field)
.collect::<Vec<_>>();
crate::database::models::loader_fields::LoaderField::get_fields_all(
&pool, &redis,
)
.await?
.into_iter()
.map(|x| x.field)
.collect::<Vec<_>>();
let uploads = index_local(&pool).await?;
add_projects(&indices, uploads, all_loader_fields.clone(), config).await?;
@@ -92,7 +96,10 @@ pub async fn index_projects(
Ok(())
}
pub async fn swap_index(config: &SearchConfig, index_name: &str) -> Result<(), IndexingError> {
pub async fn swap_index(
config: &SearchConfig,
index_name: &str,
) -> Result<(), IndexingError> {
let client = config.make_client();
let index_name_next = config.get_index_name(index_name, true);
let index_name = config.get_index_name(index_name, false);
@@ -114,7 +121,8 @@ pub async fn get_indexes_for_indexing(
) -> Result<Vec<Index>, meilisearch_sdk::errors::Error> {
let client = config.make_client();
let project_name = config.get_index_name("projects", next);
let project_filtered_name = config.get_index_name("projects_filtered", next);
let project_filtered_name =
config.get_index_name("projects_filtered", next);
let projects_index = create_or_update_index(
&client,
&project_name,
@@ -214,7 +222,11 @@ async fn add_to_index(
index
.add_or_replace(chunk, Some("version_id"))
.await?
.wait_for_completion(client, None, Some(std::time::Duration::from_secs(3600)))
.wait_for_completion(
client,
None,
Some(std::time::Duration::from_secs(3600)),
)
.await?;
info!("Added chunk of {} projects to index", chunk.len());
}
@@ -275,7 +287,8 @@ pub async fn add_projects(
) -> Result<(), IndexingError> {
let client = config.make_client();
for index in indices {
update_and_add_to_index(&client, index, &projects, &additional_fields).await?;
update_and_add_to_index(&client, index, &projects, &additional_fields)
.await?;
}
Ok(())
@@ -342,7 +355,8 @@ const DEFAULT_DISPLAYED_ATTRIBUTES: &[&str] = &[
"project_loader_fields",
];
const DEFAULT_SEARCHABLE_ATTRIBUTES: &[&str] = &["name", "summary", "author", "slug"];
const DEFAULT_SEARCHABLE_ATTRIBUTES: &[&str] =
&["name", "summary", "author", "slug"];
const DEFAULT_ATTRIBUTES_FOR_FACETING: &[&str] = &[
"categories",

41
apps/labrinth/src/search/mod.rs
View File

@@ -68,8 +68,10 @@ impl SearchConfig {
// Panics if the environment variables are not set,
// but these are already checked for on startup.
pub fn new(meta_namespace: Option<String>) -> Self {
let address = dotenvy::var("MEILISEARCH_ADDR").expect("MEILISEARCH_ADDR not set");
let key = dotenvy::var("MEILISEARCH_KEY").expect("MEILISEARCH_KEY not set");
let address =
dotenvy::var("MEILISEARCH_ADDR").expect("MEILISEARCH_ADDR not set");
let key =
dotenvy::var("MEILISEARCH_KEY").expect("MEILISEARCH_KEY not set");
Self {
address,
@@ -172,7 +174,8 @@ pub fn get_sort_index(
index: &str,
) -> Result<(String, [&'static str; 1]), SearchError> {
let projects_name = config.get_index_name("projects", false);
let projects_filtered_name = config.get_index_name("projects_filtered", false);
let projects_filtered_name =
config.get_index_name("projects_filtered", false);
Ok(match index {
"relevance" => (projects_name, ["downloads:desc"]),
"downloads" => (projects_filtered_name, ["downloads:desc"]),
@@ -224,12 +227,13 @@ pub async fn search_for_project(
None
};
let filters: Cow<_> = match (info.filters.as_deref(), info.version.as_deref()) {
(Some(f), Some(v)) => format!("({f}) AND ({v})").into(),
(Some(f), None) => f.into(),
(None, Some(v)) => v.into(),
(None, None) => "".into(),
};
let filters: Cow<_> =
match (info.filters.as_deref(), info.version.as_deref()) {
(Some(f), Some(v)) => format!("({f}) AND ({v})").into(),
(Some(f), None) => f.into(),
(None, Some(v)) => v.into(),
(None, None) => "".into(),
};
if let Some(facets) = facets {
// Search can now *optionally* have a third inner array: So Vec(AND)<Vec(OR)<Vec(AND)< _ >>>
@@ -242,10 +246,13 @@ pub async fn search_for_project(
.into_iter()
.map(|facet| {
if facet.is_array() {
serde_json::from_value::<Vec<String>>(facet).unwrap_or_default()
serde_json::from_value::<Vec<String>>(facet)
.unwrap_or_default()
} else {
vec![serde_json::from_value::<String>(facet)
.unwrap_or_default()]
vec![serde_json::from_value::<String>(
facet,
)
.unwrap_or_default()]
}
})
.collect_vec()
@@ -256,12 +263,16 @@ pub async fn search_for_project(
for (index, facet_outer_list) in facets.iter().enumerate() {
filter_string.push('(');
for (facet_outer_index, facet_inner_list) in facet_outer_list.iter().enumerate()
for (facet_outer_index, facet_inner_list) in
facet_outer_list.iter().enumerate()
{
filter_string.push('(');
for (facet_inner_index, facet) in facet_inner_list.iter().enumerate() {
for (facet_inner_index, facet) in
facet_inner_list.iter().enumerate()
{
filter_string.push_str(&facet.replace(':', " = "));
if facet_inner_index != (facet_inner_list.len() - 1) {
if facet_inner_index != (facet_inner_list.len() - 1)
{
filter_string.push_str(" AND ")
}
}