Pernoise/Rocketmc
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Switch to ARRAY_AGG for database aggregations to improve peformance + fix gallery images not showing up (#242)

Browse Source
This commit is contained in:
Geometrically
2021-08-31 15:29:51 -07:00
committed by GitHub
parent f37e267a5e
commit efb82847cb
6 changed files with 923 additions and 923 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1738
sqlx-data.json
View File

File diff suppressed because it is too large Load Diff

4
src/database/models/categories.rs
View File

@@ -303,7 +303,7 @@ impl Loader {
let result = sqlx::query!(
"
SELECT l.id id, l.loader loader, l.icon icon,
STRING_AGG(DISTINCT pt.name, ',') project_types
ARRAY_AGG(DISTINCT pt.name) project_types
FROM loaders l
LEFT OUTER JOIN loaders_project_types lpt ON joining_loader_id = l.id
LEFT OUTER JOIN project_types pt ON lpt.joining_project_type_id = pt.id
@@ -319,7 +319,7 @@ impl Loader {
supported_project_types: x
.project_types
.unwrap_or_default()
.split(',')
.iter()
.map(|x| x.to_string())
.collect(),
}))

12
src/database/models/notification_item.rs
View File

@@ -123,7 +123,7 @@ impl Notification {
let result = sqlx::query!(
"
SELECT n.user_id, n.title, n.text, n.link, n.created, n.read, n.type notification_type,
STRING_AGG(DISTINCT na.id || ', ' || na.title || ', ' || na.action_route || ', ' || na.action_route_method, ' ,') actions
ARRAY_AGG(DISTINCT na.id || ', ' || na.title || ', ' || na.action_route || ', ' || na.action_route_method) actions
FROM notifications n
LEFT OUTER JOIN notifications_actions na on n.id = na.notification_id
WHERE n.id = $1
@@ -137,7 +137,7 @@ impl Notification {
if let Some(row) = result {
let mut actions: Vec<NotificationAction> = Vec::new();
row.actions.unwrap_or_default().split(" ,").for_each(|x| {
row.actions.unwrap_or_default().iter().for_each(|x| {
let action: Vec<&str> = x.split(", ").collect();
if action.len() >= 3 {
@@ -180,7 +180,7 @@ impl Notification {
sqlx::query!(
"
SELECT n.id, n.user_id, n.title, n.text, n.link, n.created, n.read, n.type notification_type,
STRING_AGG(DISTINCT na.id || ', ' || na.title || ', ' || na.action_route || ', ' || na.action_route_method, ' ,') actions
ARRAY_AGG(DISTINCT na.id || ', ' || na.title || ', ' || na.action_route || ', ' || na.action_route_method) actions
FROM notifications n
LEFT OUTER JOIN notifications_actions na on n.id = na.notification_id
WHERE n.id = ANY($1)
@@ -195,7 +195,7 @@ impl Notification {
let id = NotificationId(row.id);
let mut actions: Vec<NotificationAction> = Vec::new();
row.actions.unwrap_or_default().split(" ,").for_each(|x| {
row.actions.unwrap_or_default().iter().for_each(|x| {
let action: Vec<&str> = x.split(", ").collect();
if action.len() >= 3 {
@@ -238,7 +238,7 @@ impl Notification {
sqlx::query!(
"
SELECT n.id, n.user_id, n.title, n.text, n.link, n.created, n.read, n.type notification_type,
STRING_AGG(DISTINCT na.id || ', ' || na.title || ', ' || na.action_route || ', ' || na.action_route_method, ' ,') actions
ARRAY_AGG(DISTINCT na.id || ', ' || na.title || ', ' || na.action_route || ', ' || na.action_route_method) actions
FROM notifications n
LEFT OUTER JOIN notifications_actions na on n.id = na.notification_id
WHERE n.user_id = $1
@@ -252,7 +252,7 @@ impl Notification {
let id = NotificationId(row.id);
let mut actions: Vec<NotificationAction> = Vec::new();
row.actions.unwrap_or_default().split(" ,").for_each(|x| {
row.actions.unwrap_or_default().iter().for_each(|x| {
let action: Vec<&str> = x.split(", ").collect();
if action.len() >= 3 {

32
src/database/models/project_item.rs
View File

@@ -599,9 +599,9 @@ impl Project {
m.issues_url issues_url, m.source_url source_url, m.wiki_url wiki_url, m.discord_url discord_url, m.license_url license_url,
m.team_id team_id, m.client_side client_side, m.server_side server_side, m.license license, m.slug slug, m.moderation_message moderation_message, m.moderation_message_body moderation_message_body,
s.status status_name, cs.name client_side_type, ss.name server_side_type, l.short short, l.name license_name, pt.name project_type_name,
STRING_AGG(DISTINCT c.category, ',') categories, STRING_AGG(DISTINCT v.id::text, ',') versions,
STRING_AGG(DISTINCT mg.image_url || ', ' || mg.featured || ', ' || COALESCE(mg.title, ' ') || ', ' || COALESCE(mg.description, ' ') || ', ' || mg.created, ' ,') gallery,
STRING_AGG(DISTINCT md.joining_platform_id || ', ' || md.url || ', ' || dp.short || ', ' || dp.name, ' ,') donations
ARRAY_AGG(DISTINCT c.category) categories, ARRAY_AGG(DISTINCT v.id::text) versions,
ARRAY_AGG(DISTINCT mg.image_url || ', ' || mg.featured || ', ' || COALESCE(mg.title, '') || ', ' || COALESCE(mg.description, '') || ', ' || mg.created) gallery,
ARRAY_AGG(DISTINCT md.joining_platform_id || ', ' || md.url || ', ' || dp.short || ', ' || dp.name) donations
FROM mods m
INNER JOIN project_types pt ON pt.id = m.project_type
INNER JOIN statuses s ON s.id = m.status
@@ -653,12 +653,12 @@ impl Project {
project_type: m.project_type_name,
categories: m
.categories
.map(|x| x.split(',').map(|x| x.to_string()).collect())
.map(|x| x.iter().map(|x| x.to_string()).collect())
.unwrap_or_default(),
versions: m
.versions
.map(|x| {
x.split(',')
x.iter()
.map(|x| VersionId(x.parse().unwrap_or_default()))
.collect()
})
@@ -666,7 +666,7 @@ impl Project {
donation_urls: m
.donations
.unwrap_or_default()
.split(" ,")
.iter()
.map(|d| {
let strings: Vec<&str> = d.split(", ").collect();
@@ -687,7 +687,7 @@ impl Project {
gallery_items: m
.gallery
.unwrap_or_default()
.split(" ,")
.iter()
.map(|d| {
let strings: Vec<&str> = d.split(", ").collect();
@@ -696,12 +696,12 @@ impl Project {
project_id: id,
image_url: strings[0].to_string(),
featured: strings[1].parse().unwrap_or(false),
title: if strings[2] == " " {
title: if strings[2] == "" {
None
} else {
Some(strings[2].to_string())
},
description: if strings[3] == " " {
description: if strings[3] == "" {
None
} else {
Some(strings[3].to_string())
@@ -745,9 +745,9 @@ impl Project {
m.issues_url issues_url, m.source_url source_url, m.wiki_url wiki_url, m.discord_url discord_url, m.license_url license_url,
m.team_id team_id, m.client_side client_side, m.server_side server_side, m.license license, m.slug slug, m.moderation_message moderation_message, m.moderation_message_body moderation_message_body,
s.status status_name, cs.name client_side_type, ss.name server_side_type, l.short short, l.name license_name, pt.name project_type_name,
STRING_AGG(DISTINCT c.category, ',') categories, STRING_AGG(DISTINCT v.id::text, ',') versions,
STRING_AGG(DISTINCT mg.image_url || ', ' || mg.featured || ', ' || COALESCE(mg.title, ' ') || ', ' || COALESCE(mg.description, ' ') || ', ' || mg.created, ' ,') gallery,
STRING_AGG(DISTINCT md.joining_platform_id || ', ' || md.url || ', ' || dp.short || ', ' || dp.name, ' ,') donations
ARRAY_AGG(DISTINCT c.category) categories, ARRAY_AGG(DISTINCT v.id::text) versions,
ARRAY_AGG(DISTINCT mg.image_url || ', ' || mg.featured || ', ' || COALESCE(mg.title, '') || ', ' || COALESCE(mg.description, '') || ', ' || mg.created) gallery,
ARRAY_AGG(DISTINCT md.joining_platform_id || ', ' || md.url || ', ' || dp.short || ', ' || dp.name) donations
FROM mods m
INNER JOIN project_types pt ON pt.id = m.project_type
INNER JOIN statuses s ON s.id = m.status
@@ -797,12 +797,12 @@ impl Project {
moderation_message_body: m.moderation_message_body,
},
project_type: m.project_type_name,
categories: m.categories.map(|x| x.split(',').map(|x| x.to_string()).collect()).unwrap_or_default(),
versions: m.versions.map(|x| x.split(',').map(|x| VersionId(x.parse().unwrap_or_default())).collect()).unwrap_or_default(),
categories: m.categories.map(|x| x.iter().map(|x| x.to_string()).collect()).unwrap_or_default(),
versions: m.versions.map(|x| x.iter().map(|x| VersionId(x.parse().unwrap_or_default())).collect()).unwrap_or_default(),
gallery_items: m
.gallery
.unwrap_or_default()
.split(" ,")
.iter()
.map(|d| {
let strings: Vec<&str> = d.split(", ").collect();
@@ -824,7 +824,7 @@ impl Project {
donation_urls: m
.donations
.unwrap_or_default()
.split(" ,")
.iter()
.map(|d| {
let strings: Vec<&str> = d.split(", ").collect();

36
src/database/models/version_item.rs
View File

@@ -603,10 +603,10 @@ impl Version {
SELECT v.id id, v.mod_id mod_id, v.author_id author_id, v.name version_name, v.version_number version_number,
v.changelog changelog, v.changelog_url changelog_url, v.date_published date_published, v.downloads downloads,
v.version_type version_type, v.featured featured,
STRING_AGG(DISTINCT gv.version, ',') game_versions, STRING_AGG(DISTINCT l.loader, ',') loaders,
STRING_AGG(DISTINCT f.id || ', ' || f.filename || ', ' || f.is_primary || ', ' || f.url, ' ,') files,
STRING_AGG(DISTINCT h.algorithm || ', ' || encode(h.hash, 'escape') || ', ' || h.file_id, ' ,') hashes,
STRING_AGG(DISTINCT COALESCE(d.dependency_id, 0) || ', ' || COALESCE(d.mod_dependency_id, 0) || ', ' || d.dependency_type, ' ,') dependencies
ARRAY_AGG(DISTINCT gv.version) game_versions, ARRAY_AGG(DISTINCT l.loader) loaders,
ARRAY_AGG(DISTINCT f.id || ', ' || f.filename || ', ' || f.is_primary || ', ' || f.url) files,
ARRAY_AGG(DISTINCT h.algorithm || ', ' || encode(h.hash, 'escape') || ', ' || h.file_id) hashes,
ARRAY_AGG(DISTINCT COALESCE(d.dependency_id, 0) || ', ' || COALESCE(d.mod_dependency_id, 0) || ', ' || d.dependency_type) dependencies
FROM versions v
LEFT OUTER JOIN game_versions_versions gvv on v.id = gvv.joining_version_id
LEFT OUTER JOIN game_versions gv on gvv.game_version_id = gv.id
@@ -627,7 +627,7 @@ impl Version {
let hashes: Vec<(FileId, String, Vec<u8>)> = v
.hashes
.unwrap_or_default()
.split(" ,")
.iter()
.map(|f| {
let hash: Vec<&str> = f.split(", ").collect();
@@ -657,7 +657,7 @@ impl Version {
files: v
.files
.unwrap_or_default()
.split(" ,")
.iter()
.map(|f| {
let file: Vec<&str> = f.split(", ").collect();
@@ -687,20 +687,20 @@ impl Version {
game_versions: v
.game_versions
.unwrap_or_default()
.split(',')
.iter()
.map(|x| x.to_string())
.collect(),
loaders: v
.loaders
.unwrap_or_default()
.split(',')
.iter()
.map(|x| x.to_string())
.collect(),
featured: v.featured,
dependencies: v
.dependencies
.unwrap_or_default()
.split(" ,")
.iter()
.map(|f| {
let dependency: Vec<&str> = f.split(", ").collect();
@@ -750,10 +750,10 @@ impl Version {
SELECT v.id id, v.mod_id mod_id, v.author_id author_id, v.name version_name, v.version_number version_number,
v.changelog changelog, v.changelog_url changelog_url, v.date_published date_published, v.downloads downloads,
v.version_type version_type, v.featured featured,
STRING_AGG(DISTINCT gv.version, ',') game_versions, STRING_AGG(DISTINCT l.loader, ',') loaders,
STRING_AGG(DISTINCT f.id || ', ' || f.filename || ', ' || f.is_primary || ', ' || f.url, ' ,') files,
STRING_AGG(DISTINCT h.algorithm || ', ' || encode(h.hash, 'escape') || ', ' || h.file_id, ' ,') hashes,
STRING_AGG(DISTINCT COALESCE(d.dependency_id, 0) || ', ' || COALESCE(d.mod_dependency_id, 0) || ', ' || d.dependency_type, ' ,') dependencies
ARRAY_AGG(DISTINCT gv.version) game_versions, ARRAY_AGG(DISTINCT l.loader) loaders,
ARRAY_AGG(DISTINCT f.id || ', ' || f.filename || ', ' || f.is_primary || ', ' || f.url) files,
ARRAY_AGG(DISTINCT h.algorithm || ', ' || encode(h.hash, 'escape') || ', ' || h.file_id) hashes,
ARRAY_AGG(DISTINCT COALESCE(d.dependency_id, 0) || ', ' || COALESCE(d.mod_dependency_id, 0) || ', ' || d.dependency_type) dependencies
FROM versions v
LEFT OUTER JOIN game_versions_versions gvv on v.id = gvv.joining_version_id
LEFT OUTER JOIN game_versions gv on gvv.game_version_id = gv.id
@@ -771,7 +771,7 @@ impl Version {
.fetch_many(exec)
.try_filter_map(|e| async {
Ok(e.right().map(|v| {
let hashes: Vec<(FileId, String, Vec<u8>)> = v.hashes.unwrap_or_default().split(" ,").map(|f| {
let hashes: Vec<(FileId, String, Vec<u8>)> = v.hashes.unwrap_or_default().iter().map(|f| {
let hash: Vec<&str> = f.split(", ").collect();
if hash.len() >= 3 {
@@ -795,7 +795,7 @@ impl Version {
changelog_url: v.changelog_url,
date_published: v.date_published,
downloads: v.downloads,
files: v.files.unwrap_or_default().split(" ,").map(|f| {
files: v.files.unwrap_or_default().iter().map(|f| {
let file: Vec<&str> = f.split(", ").collect();
if file.len() >= 4 {
@@ -819,12 +819,12 @@ impl Version {
None
}
}).flatten().collect(),
game_versions: v.game_versions.unwrap_or_default().split(',').map(|x| x.to_string()).collect(),
loaders: v.loaders.unwrap_or_default().split(',').map(|x| x.to_string()).collect(),
game_versions: v.game_versions.unwrap_or_default().iter().map(|x| x.to_string()).collect(),
loaders: v.loaders.unwrap_or_default().iter().map(|x| x.to_string()).collect(),
featured: v.featured,
dependencies: v.dependencies
.unwrap_or_default()
.split(" ,")
.iter()
.map(|f| {
let dependency: Vec<&str> = f.split(", ").collect();

24
src/search/indexing/local_import.rs
View File

@@ -18,8 +18,8 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchProject>, Index
m.updated updated,
m.team_id team_id, m.license license, m.slug slug,
s.status status_name, cs.name client_side_type, ss.name server_side_type, l.short short, pt.name project_type_name, u.username username,
STRING_AGG(DISTINCT c.category, ',') categories, STRING_AGG(DISTINCT lo.loader, ',') loaders, STRING_AGG(DISTINCT gv.version, ',') versions,
STRING_AGG(DISTINCT mg.image_url, ',') gallery
ARRAY_AGG(DISTINCT c.category) categories, ARRAY_AGG(DISTINCT lo.loader) loaders, ARRAY_AGG(DISTINCT gv.version) versions,
ARRAY_AGG(DISTINCT mg.image_url) gallery
FROM mods m
LEFT OUTER JOIN mods_categories mc ON joining_mod_id = m.id
LEFT OUTER JOIN categories c ON mc.joining_category_id = c.id
@@ -45,10 +45,10 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchProject>, Index
.fetch_many(&pool)
.try_filter_map(|e| async {
Ok(e.right().map(|m| {
let mut categories = m.categories.map(|x| x.split(',').map(|x| x.to_string()).collect::<Vec<String>>()).unwrap_or_default();
categories.append(&mut m.loaders.map(|x| x.split(',').map(|x| x.to_string()).collect::<Vec<String>>()).unwrap_or_default());
let mut categories = m.categories.map(|x| x.iter().map(|x| x.to_string()).collect::<Vec<String>>()).unwrap_or_default();
categories.append(&mut m.loaders.map(|x| x.iter().map(|x| x.to_string()).collect::<Vec<String>>()).unwrap_or_default());
let versions : Vec<String> = m.versions.map(|x| x.split(',').map(|x| x.to_string()).collect()).unwrap_or_default();
let versions : Vec<String> = m.versions.map(|x| x.iter().map(|x| x.to_string()).collect()).unwrap_or_default();
let project_id : crate::models::projects::ProjectId = ProjectId(m.id).into();
@@ -72,7 +72,7 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchProject>, Index
server_side: m.server_side_type,
slug: m.slug,
project_type: m.project_type_name,
gallery: m.gallery.map(|x| x.split(',').map(|x| x.to_string()).collect()).unwrap_or_default()
gallery: m.gallery.map(|x| x.iter().map(|x| x.to_string()).collect()).unwrap_or_default()
}
}))
})
@@ -92,8 +92,8 @@ pub async fn query_one(
m.updated updated,
m.team_id team_id, m.license license, m.slug slug,
s.status status_name, cs.name client_side_type, ss.name server_side_type, l.short short, pt.name project_type_name, u.username username,
STRING_AGG(DISTINCT c.category, ',') categories, STRING_AGG(DISTINCT lo.loader, ',') loaders, STRING_AGG(DISTINCT gv.version, ',') versions,
STRING_AGG(DISTINCT mg.image_url, ',') gallery
ARRAY_AGG(DISTINCT c.category) categories, ARRAY_AGG(DISTINCT lo.loader) loaders, ARRAY_AGG(DISTINCT gv.version) versions,
ARRAY_AGG(DISTINCT mg.image_url) gallery
FROM mods m
LEFT OUTER JOIN mods_categories mc ON joining_mod_id = m.id
LEFT OUTER JOIN categories c ON mc.joining_category_id = c.id
@@ -121,18 +121,18 @@ pub async fn query_one(
let mut categories = m
.categories
.map(|x| x.split(',').map(|x| x.to_string()).collect::<Vec<String>>())
.map(|x| x.iter().map(|x| x.to_string()).collect::<Vec<String>>())
.unwrap_or_default();
categories.append(
&mut m
.loaders
.map(|x| x.split(',').map(|x| x.to_string()).collect::<Vec<String>>())
.map(|x| x.iter().map(|x| x.to_string()).collect::<Vec<String>>())
.unwrap_or_default(),
);
let versions: Vec<String> = m
.versions
.map(|x| x.split(',').map(|x| x.to_string()).collect())
.map(|x| x.iter().map(|x| x.to_string()).collect())
.unwrap_or_default();
let project_id: crate::models::projects::ProjectId = ProjectId(m.id).into();
@@ -162,7 +162,7 @@ pub async fn query_one(
project_type: m.project_type_name,
gallery: m
.gallery
.map(|x| x.split(',').map(|x| x.to_string()).collect())
.map(|x| x.iter().map(|x| x.to_string()).collect())
.unwrap_or_default(),
})
}