eightequal/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Optimize and fix some bugs in indexing (#98)

Browse Source 
* Improve curseforge and local indexing

This should make curseforge indexing more efficient, and reuses
some of the normal local indexing for the queued indexing of
recently created mods.

* Unify impls for single and multiple routes for mods and versions

This uses the same backend for the single and multiple query
routes so that they no longer return inconsistent information.

* Cache valid curseforge mod ids to reduce request load

This caches the ids of minecraft mods and reuses them on indexing
to reduce the amount of unused addons that are returned.
This commit is contained in:
Aeledfyr
2020-11-03 18:55:50 -06:00
committed by GitHub
parent da79386cc3
commit d477874535
15 changed files with 745 additions and 343 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
src/routes/mod_creation.rs
View File

@@ -5,7 +5,7 @@ use crate::models::error::ApiError;
use crate::models::mods::{ModId, ModStatus, VersionId};
use crate::models::users::UserId;
use crate::routes::version_creation::InitialVersionData;
use crate::search::indexing::queue::CreationQueue;
use crate::search::indexing::{queue::CreationQueue, IndexingError};
use actix_multipart::{Field, Multipart};
use actix_web::http::StatusCode;
use actix_web::web::Data;
@@ -13,7 +13,6 @@ use actix_web::{post, HttpRequest, HttpResponse};
use futures::stream::StreamExt;
use serde::{Deserialize, Serialize};
use sqlx::postgres::PgPool;
use std::borrow::Cow;
use std::sync::Arc;
use thiserror::Error;
@@ -25,6 +24,8 @@ pub enum CreateError {
SqlxDatabaseError(#[from] sqlx::Error),
#[error("Database Error: {0}")]
DatabaseError(#[from] models::DatabaseError),
#[error("Indexing Error: {0}")]
IndexingError(#[from] IndexingError),
#[error("Error while parsing multipart payload")]
MultipartError(actix_multipart::MultipartError),
#[error("Error while parsing JSON: {0}")]
@@ -55,6 +56,7 @@ impl actix_web::ResponseError for CreateError {
CreateError::EnvError(..) => StatusCode::INTERNAL_SERVER_ERROR,
CreateError::SqlxDatabaseError(..) => StatusCode::INTERNAL_SERVER_ERROR,
CreateError::DatabaseError(..) => StatusCode::INTERNAL_SERVER_ERROR,
CreateError::IndexingError(..) => StatusCode::INTERNAL_SERVER_ERROR,
CreateError::FileHostingError(..) => StatusCode::INTERNAL_SERVER_ERROR,
CreateError::SerDeError(..) => StatusCode::BAD_REQUEST,
CreateError::MultipartError(..) => StatusCode::BAD_REQUEST,
@@ -75,6 +77,7 @@ impl actix_web::ResponseError for CreateError {
CreateError::EnvError(..) => "environment_error",
CreateError::SqlxDatabaseError(..) => "database_error",
CreateError::DatabaseError(..) => "database_error",
CreateError::IndexingError(..) => "indexing_error",
CreateError::FileHostingError(..) => "file_hosting_error",
CreateError::SerDeError(..) => "invalid_input",
CreateError::MultipartError(..) => "invalid_input",
@@ -460,40 +463,7 @@ async fn mod_create_inner(
status: status_id,
};
let versions_list = mod_create_data
.initial_versions
.iter()
.flat_map(|v| v.game_versions.iter().map(|name| name.0.clone()))
.collect::<std::collections::HashSet<String>>()
.into_iter()
.collect::<Vec<_>>();
let now = chrono::Utc::now();
let timestamp = now.timestamp();
let index_mod = crate::search::UploadSearchMod {
mod_id: format!("local-{}", mod_id),
title: mod_builder.title.clone(),
description: mod_builder.description.clone(),
categories: mod_create_data.categories.clone(),
versions: versions_list,
page_url: format!("https://modrinth.com/mod/{}", mod_id),
// This should really be optional in the index
icon_url: mod_builder.icon_url.clone().unwrap_or_else(String::new),
author: current_user.username.clone(),
author_url: format!("https://modrinth.com/user/{}", current_user.id),
// TODO: latest version info
latest_version: String::new(),
downloads: 0,
date_created: now,
created_timestamp: timestamp,
date_modified: now,
modified_timestamp: timestamp,
host: Cow::Borrowed("modrinth"),
empty: Cow::Borrowed("{}{}{}"),
};
indexing_queue.add(index_mod);
let response = crate::models::mods::Mod {
id: mod_id,
@@ -505,7 +475,7 @@ async fn mod_create_inner(
updated: now,
status,
downloads: 0,
categories: mod_create_data.categories.clone(),
categories: mod_create_data.categories,
versions: mod_builder
.initial_versions
.iter()
@@ -519,6 +489,11 @@ async fn mod_create_inner(
let _mod_id = mod_builder.insert(&mut *transaction).await?;
let index_mod =
crate::search::indexing::local_import::query_one(mod_id.into(), &mut *transaction)
.await?;
indexing_queue.add(index_mod);
Ok(HttpResponse::Ok().json(response))
}
}

96
src/routes/mods.rs
View File

@@ -33,43 +33,15 @@ pub async fn mods_get(
.map(|x| x.into())
.collect();
let mods_data = database::models::Mod::get_many(mod_ids, &**pool)
let mods_data = database::models::Mod::get_many_full(mod_ids, &**pool)
.await
.map_err(|e| ApiError::DatabaseError(e.into()))?;
let mut mods: Vec<models::mods::Mod> = Vec::new();
for m in mods_data {
let status = sqlx::query!(
"
SELECT status FROM statuses
WHERE id = $1
",
m.status.0,
)
.fetch_one(&**pool)
.await
.map_err(|e| ApiError::DatabaseError(e.into()))?
.status;
mods.push(models::mods::Mod {
id: m.id.into(),
team: m.team_id.into(),
title: m.title,
description: m.description,
body_url: m.body_url,
published: m.published,
updated: m.updated,
status: models::mods::ModStatus::from_str(&*status),
downloads: m.downloads as u32,
categories: vec![],
versions: vec![],
icon_url: m.icon_url,
issues_url: m.issues_url,
source_url: m.source_url,
wiki_url: m.wiki_url,
})
}
let mods = mods_data
.into_iter()
.filter_map(|m| m)
.map(convert_mod)
.collect::<Vec<_>>();
Ok(HttpResponse::Ok().json(mods))
}
@@ -85,44 +57,34 @@ pub async fn mod_get(
.map_err(|e| ApiError::DatabaseError(e.into()))?;
if let Some(data) = mod_data {
let m = data.inner;
let status = sqlx::query!(
"
SELECT status FROM statuses
WHERE id = $1
",
m.status.0,
)
.fetch_one(&**pool)
.await
.map_err(|e| ApiError::DatabaseError(e.into()))?
.status;
let response = models::mods::Mod {
id: m.id.into(),
team: m.team_id.into(),
title: m.title,
description: m.description,
body_url: m.body_url,
published: m.published,
updated: m.updated,
status: models::mods::ModStatus::from_str(&*status),
downloads: m.downloads as u32,
categories: data.categories,
versions: data.versions.into_iter().map(|v| v.into()).collect(),
icon_url: m.icon_url,
issues_url: m.issues_url,
source_url: m.source_url,
wiki_url: m.wiki_url,
};
Ok(HttpResponse::Ok().json(response))
Ok(HttpResponse::Ok().json(convert_mod(data)))
} else {
Ok(HttpResponse::NotFound().body(""))
}
}
fn convert_mod(data: database::models::mod_item::QueryMod) -> models::mods::Mod {
let m = data.inner;
models::mods::Mod {
id: m.id.into(),
team: m.team_id.into(),
title: m.title,
description: m.description,
body_url: m.body_url,
published: m.published,
updated: m.updated,
status: data.status,
downloads: m.downloads as u32,
categories: data.categories,
versions: data.versions.into_iter().map(|v| v.into()).collect(),
icon_url: m.icon_url,
issues_url: m.issues_url,
source_url: m.source_url,
wiki_url: m.wiki_url,
}
}
#[delete("{id}")]
pub async fn mod_delete(
req: HttpRequest,

2
src/routes/not_found.rs
View File

@@ -4,7 +4,7 @@ use actix_web::{HttpResponse, Responder};
pub async fn not_found() -> impl Responder {
let data = ApiError {
error: "not_found",
description: "the route you called is not (yet) implemented",
description: "the requested route does not exist",
};
HttpResponse::NotFound().json(data)

126
src/routes/versions.rs
View File

@@ -48,7 +48,6 @@ pub struct VersionIds {
pub ids: String,
}
// TODO: Make this return the versions mod struct
#[get("versions")]
pub async fn versions_get(
web::Query(ids): web::Query<VersionIds>,
@@ -58,30 +57,14 @@ pub async fn versions_get(
.into_iter()
.map(|x| x.into())
.collect();
let versions_data = database::models::Version::get_many(version_ids, &**pool)
let versions_data = database::models::Version::get_many_full(version_ids, &**pool)
.await
.map_err(|e| ApiError::DatabaseError(e.into()))?;
use models::mods::VersionType;
let versions: Vec<models::mods::Version> = versions_data
.into_iter()
.map(|data| models::mods::Version {
id: data.id.into(),
mod_id: data.mod_id.into(),
author_id: data.author_id.into(),
name: data.name,
version_number: data.version_number,
changelog_url: data.changelog_url,
date_published: data.date_published,
downloads: data.downloads as u32,
version_type: VersionType::Release,
files: vec![],
dependencies: Vec::new(), // TODO: dependencies
game_versions: vec![],
loaders: vec![],
})
.filter_map(|v| v)
.map(convert_version)
.collect();
Ok(HttpResponse::Ok().json(versions))
@@ -98,61 +81,64 @@ pub async fn version_get(
.map_err(|e| ApiError::DatabaseError(e.into()))?;
if let Some(data) = version_data {
use models::mods::VersionType;
let response = models::mods::Version {
id: data.id.into(),
mod_id: data.mod_id.into(),
author_id: data.author_id.into(),
name: data.name,
version_number: data.version_number,
changelog_url: data.changelog_url,
date_published: data.date_published,
downloads: data.downloads as u32,
version_type: match data.release_channel.as_str() {
"release" => VersionType::Release,
"beta" => VersionType::Beta,
"alpha" => VersionType::Alpha,
_ => VersionType::Alpha,
},
files: data
.files
.into_iter()
.map(|f| {
models::mods::VersionFile {
url: f.url,
filename: f.filename,
// FIXME: Hashes are currently stored as an ascii byte slice instead
// of as an actual byte array in the database
hashes: f
.hashes
.into_iter()
.map(|(k, v)| Some((k, String::from_utf8(v).ok()?)))
.collect::<Option<_>>()
.unwrap_or_else(Default::default),
}
})
.collect(),
dependencies: Vec::new(), // TODO: dependencies
game_versions: data
.game_versions
.into_iter()
.map(models::mods::GameVersion)
.collect(),
loaders: data
.loaders
.into_iter()
.map(models::mods::ModLoader)
.collect(),
};
Ok(HttpResponse::Ok().json(response))
Ok(HttpResponse::Ok().json(convert_version(data)))
} else {
Ok(HttpResponse::NotFound().body(""))
}
}
fn convert_version(data: database::models::version_item::QueryVersion) -> models::mods::Version {
use models::mods::VersionType;
models::mods::Version {
id: data.id.into(),
mod_id: data.mod_id.into(),
author_id: data.author_id.into(),
name: data.name,
version_number: data.version_number,
changelog_url: data.changelog_url,
date_published: data.date_published,
downloads: data.downloads as u32,
version_type: match data.release_channel.as_str() {
"release" => VersionType::Release,
"beta" => VersionType::Beta,
"alpha" => VersionType::Alpha,
_ => VersionType::Alpha,
},
files: data
.files
.into_iter()
.map(|f| {
models::mods::VersionFile {
url: f.url,
filename: f.filename,
// FIXME: Hashes are currently stored as an ascii byte slice instead
// of as an actual byte array in the database
hashes: f
.hashes
.into_iter()
.map(|(k, v)| Some((k, String::from_utf8(v).ok()?)))
.collect::<Option<_>>()
.unwrap_or_else(Default::default),
}
})
.collect(),
dependencies: Vec::new(), // TODO: dependencies
game_versions: data
.game_versions
.into_iter()
.map(models::mods::GameVersion)
.collect(),
loaders: data
.loaders
.into_iter()
.map(models::mods::ModLoader)
.collect(),
}
}
#[delete("{version_id}")]
pub async fn version_delete(
req: HttpRequest,