tomasalias/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Chunking searches (#787)

Browse Source 
* new attempt

* revised searching CTEs

* prepare fix

* fix tests

* fixes

* restructured project_item to use queries

* search changes! fmt clippy prepare

* small changes
This commit is contained in:
Wyatt Verchere
2023-12-08 22:14:17 -08:00
committed by GitHub
parent 945e5a2dc3
commit 235f4f10ef
28 changed files with 1784 additions and 1026 deletions
Download Patch File Download Diff File Expand all files Collapse all files

39
src/search/indexing/local_import.rs
View File

@@ -12,14 +12,10 @@ use crate::models;
use crate::search::UploadSearchProject;
use sqlx::postgres::PgPool;
pub async fn index_local(
pub async fn get_all_ids(
pool: PgPool,
redis: &RedisPool,
) -> Result<(Vec<UploadSearchProject>, Vec<String>), IndexingError> {
info!("Indexing local projects!");
let loader_field_keys: Arc<DashSet<String>> = Arc::new(DashSet::new());
let all_visible_ids: HashMap<VersionId, (ProjectId, String)> = sqlx::query!(
) -> Result<Vec<(VersionId, ProjectId, String)>, IndexingError> {
let all_visible_ids: Vec<(VersionId, ProjectId, String)> = sqlx::query!(
"
SELECT v.id id, m.id mod_id, u.username owner_username
@@ -45,33 +41,48 @@ pub async fn index_local(
Ok(e.right().map(|m| {
let project_id: ProjectId = ProjectId(m.mod_id);
let version_id: VersionId = VersionId(m.id);
(version_id, (project_id, m.owner_username))
(version_id, project_id, m.owner_username)
}))
})
.try_collect::<HashMap<_, _>>()
.try_collect::<Vec<_>>()
.await?;
let project_ids = all_visible_ids
Ok(all_visible_ids)
}
pub async fn index_local(
pool: &PgPool,
redis: &RedisPool,
visible_ids: HashMap<VersionId, (ProjectId, String)>,
) -> Result<(Vec<UploadSearchProject>, Vec<String>), IndexingError> {
info!("Indexing local projects!");
let loader_field_keys: Arc<DashSet<String>> = Arc::new(DashSet::new());
let project_ids = visible_ids
.values()
.map(|(project_id, _)| project_id)
.cloned()
.collect::<Vec<_>>();
let projects: HashMap<_, _> = project_item::Project::get_many_ids(&project_ids, &pool, redis)
let projects: HashMap<_, _> = project_item::Project::get_many_ids(&project_ids, pool, redis)
.await?
.into_iter()
.map(|p| (p.inner.id, p))
.collect();
let version_ids = all_visible_ids.keys().cloned().collect::<Vec<_>>();
let versions: HashMap<_, _> = version_item::Version::get_many(&version_ids, &pool, redis)
info!("Fetched local projects!");
let version_ids = visible_ids.keys().cloned().collect::<Vec<_>>();
let versions: HashMap<_, _> = version_item::Version::get_many(&version_ids, pool, redis)
.await?
.into_iter()
.map(|v| (v.inner.id, v))
.collect();
info!("Fetched local versions!");
let mut uploads = Vec::new();
// TODO: could possibly clone less here?
for (version_id, (project_id, owner_username)) in all_visible_ids {
for (version_id, (project_id, owner_username)) in visible_ids {
let m = projects.get(&project_id);
let v = versions.get(&version_id);