mtd/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Testing search prod (#791)

Browse Source 
* testing push

* lowers it

* removed unwrap

* reduced to 500

* Really took down time

* reorders chunking

* rebuild docker

* reverted most changes

* cargo fmt

* reduced meilisearch limit

* added logs, removed deletion of index

* one client creation

* changes

* reverted gallery cahnge

* testing re-splitting again

* Remove chunking + index deletion

* Bring back chunking

* Update chunk size

---------

Co-authored-by: Jai A <jaiagr+gpg@pm.me>
Co-authored-by: Geometrically <18202329+Geometrically@users.noreply.github.com>
This commit is contained in:
Wyatt Verchere
2023-12-11 20:01:15 -08:00
committed by GitHub
parent 6217523cc8
commit 90954dac49
15 changed files with 274 additions and 315 deletions
Download Patch File Download Diff File Expand all files Collapse all files

58
src/database/models/project_item.rs
View File

@@ -550,7 +550,7 @@ impl Project {
.collect();
let slugs = remaining_strings
.into_iter()
.map(|x| x.to_string().to_lowercase())
.map(|x| x.to_lowercase())
.collect::<Vec<_>>();
let all_version_ids = DashSet::new();
@@ -569,15 +569,18 @@ impl Project {
.collect::<Vec<String>>()
)
.fetch(&mut *exec)
.try_fold(DashMap::new(), |acc : DashMap<ProjectId, Vec<(VersionId, DateTime<Utc>)>>, m| {
let version_id = VersionId(m.id);
let date_published = m.date_published;
all_version_ids.insert(version_id);
acc.entry(ProjectId(m.mod_id))
.or_default()
.push((version_id, date_published));
async move { Ok(acc) }
})
.try_fold(
DashMap::new(),
|acc: DashMap<ProjectId, Vec<(VersionId, DateTime<Utc>)>>, m| {
let version_id = VersionId(m.id);
let date_published = m.date_published;
all_version_ids.insert(version_id);
acc.entry(ProjectId(m.mod_id))
.or_default()
.push((version_id, date_published));
async move { Ok(acc) }
},
)
.await?;
let loader_field_ids = DashSet::new();
@@ -592,25 +595,26 @@ impl Project {
&all_version_ids.iter().map(|x| x.0).collect::<Vec<_>>()
)
.fetch(&mut *exec)
.try_fold(DashMap::new(), |acc : DashMap<ProjectId, Vec<QueryVersionField>>, m| {
let qvf = QueryVersionField {
version_id: VersionId(m.version_id),
field_id: LoaderFieldId(m.field_id),
int_value: m.int_value,
enum_value: m.enum_value.map(LoaderFieldEnumValueId),
string_value: m.string_value,
};
.try_fold(
DashMap::new(),
|acc: DashMap<ProjectId, Vec<QueryVersionField>>, m| {
let qvf = QueryVersionField {
version_id: VersionId(m.version_id),
field_id: LoaderFieldId(m.field_id),
int_value: m.int_value,
enum_value: m.enum_value.map(LoaderFieldEnumValueId),
string_value: m.string_value,
};
loader_field_ids.insert(LoaderFieldId(m.field_id));
if let Some(enum_value) = m.enum_value {
loader_field_enum_value_ids.insert(LoaderFieldEnumValueId(enum_value));
}
loader_field_ids.insert(LoaderFieldId(m.field_id));
if let Some(enum_value) = m.enum_value {
loader_field_enum_value_ids.insert(LoaderFieldEnumValueId(enum_value));
}
acc.entry(ProjectId(m.mod_id))
.or_default()
.push(qvf);
async move { Ok(acc) }
})
acc.entry(ProjectId(m.mod_id)).or_default().push(qvf);
async move { Ok(acc) }
},
)
.await?;
let loader_fields: Vec<QueryLoaderField> = sqlx::query!(

44
src/database/models/user_item.rs
View File

@@ -434,7 +434,6 @@ impl User {
pub async fn remove(
id: UserId,
full: bool,
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
redis: &RedisPool,
) -> Result<Option<()>, DatabaseError> {
@@ -445,38 +444,17 @@ impl User {
let deleted_user: UserId = crate::models::users::DELETED_USER.into();
if full {
let projects: Vec<ProjectId> = sqlx::query!(
"
SELECT m.id FROM mods m
INNER JOIN team_members tm ON tm.team_id = m.team_id
WHERE tm.user_id = $1 AND tm.is_owner = TRUE
",
id as UserId,
)
.fetch_many(&mut **transaction)
.try_filter_map(|e| async { Ok(e.right().map(|m| ProjectId(m.id))) })
.try_collect::<Vec<ProjectId>>()
.await?;
for project_id in projects {
let _result =
super::project_item::Project::remove(project_id, transaction, redis)
.await?;
}
} else {
sqlx::query!(
"
UPDATE team_members
SET user_id = $1
WHERE (user_id = $2 AND is_owner = TRUE)
",
deleted_user as UserId,
id as UserId,
)
.execute(&mut **transaction)
.await?;
}
sqlx::query!(
"
UPDATE team_members
SET user_id = $1
WHERE (user_id = $2 AND is_owner = TRUE)
",
deleted_user as UserId,
id as UserId,
)
.execute(&mut **transaction)
.await?;
sqlx::query!(
"

43
src/database/models/version_item.rs
View File

@@ -540,25 +540,26 @@ impl Version {
&version_ids_parsed
)
.fetch(&mut *exec)
.try_fold(DashMap::new(), |acc : DashMap<VersionId, Vec<QueryVersionField>>, m| {
let qvf = QueryVersionField {
version_id: VersionId(m.version_id),
field_id: LoaderFieldId(m.field_id),
int_value: m.int_value,
enum_value: m.enum_value.map(LoaderFieldEnumValueId),
string_value: m.string_value,
};
.try_fold(
DashMap::new(),
|acc: DashMap<VersionId, Vec<QueryVersionField>>, m| {
let qvf = QueryVersionField {
version_id: VersionId(m.version_id),
field_id: LoaderFieldId(m.field_id),
int_value: m.int_value,
enum_value: m.enum_value.map(LoaderFieldEnumValueId),
string_value: m.string_value,
};
loader_field_ids.insert(LoaderFieldId(m.field_id));
if let Some(enum_value) = m.enum_value {
loader_field_enum_value_ids.insert(LoaderFieldEnumValueId(enum_value));
}
loader_field_ids.insert(LoaderFieldId(m.field_id));
if let Some(enum_value) = m.enum_value {
loader_field_enum_value_ids.insert(LoaderFieldEnumValueId(enum_value));
}
acc.entry(VersionId(m.version_id))
.or_default()
.push(qvf);
async move { Ok(acc) }
})
acc.entry(VersionId(m.version_id)).or_default().push(qvf);
async move { Ok(acc) }
},
)
.await?;
let loader_fields: Vec<QueryLoaderField> = sqlx::query!(
@@ -692,7 +693,7 @@ impl Version {
&file_ids.iter().map(|x| x.0).collect::<Vec<_>>()
)
.fetch(&mut *exec)
.try_fold(DashMap::new(), |acc : DashMap<VersionId, Vec<Hash>>, m| {
.try_fold(DashMap::new(), |acc: DashMap<VersionId, Vec<Hash>>, m| {
if let Some(found_hash) = m.hash {
let hash = Hash {
file_id: FileId(m.file_id),
@@ -700,9 +701,9 @@ impl Version {
hash: found_hash,
};
let version_id = *reverse_file_map.get(&FileId(m.file_id)).unwrap();
acc.entry(version_id).or_default().push(hash);
if let Some(version_id) = reverse_file_map.get(&FileId(m.file_id)) {
acc.entry(*version_id).or_default().push(hash);
}
}
async move { Ok(acc) }
})