tomasalias/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Improve error handling (#33)

Browse Source 
* refactor: improve error handling

* fix: specify bind address instead of port

* fix: remove temporary testing file

* fix(errors): change error names to snake_case

* refactor(errors): split indexing error types, remove unused errors

* feat: add env variable checking at program start

This just checks whether the enviroment variables exist and can
parse to the given type and gives a warning if they can't. This
should prevent cases where the program fails at runtime due to
checking an environment variable that doesn't exist.
This commit is contained in:
Aeledfyr
2020-07-03 12:44:39 -05:00
committed by GitHub
parent 91305262f1
commit 6ff7fa74e2
10 changed files with 171 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
src/search/indexing/curseforge_import.rs
View File

@@ -1,4 +1,5 @@
use crate::search::{SearchError, SearchMod};
use super::IndexingError;
use crate::search::SearchMod;
use log::info;
use serde::{Deserialize, Serialize};
@@ -47,7 +48,7 @@ pub struct CurseForgeMod {
pub async fn index_curseforge(
start_index: i32,
end_index: i32,
) -> Result<Vec<SearchMod>, SearchError> {
) -> Result<Vec<SearchMod>, IndexingError> {
info!("Indexing curseforge mods!");
let mut docs_to_add: Vec<SearchMod> = vec![];
@@ -60,10 +61,13 @@ pub async fn index_curseforge(
(start_index..end_index).collect::<Vec<_>>()
))
.send()
.await?;
.await
.map_err(IndexingError::CurseforgeImportError)?;
let text = &res.text().await?;
let curseforge_mods: Vec<CurseForgeMod> = serde_json::from_str(text)?;
let curseforge_mods: Vec<CurseForgeMod> = res
.json()
.await
.map_err(IndexingError::CurseforgeImportError)?;
for curseforge_mod in curseforge_mods {
if curseforge_mod.game_slug != "minecraft"
@@ -78,15 +82,14 @@ pub async fn index_curseforge(
let mut using_fabric = false;
for version in curseforge_mod.game_version_latest_files {
let version_number: String = version
if let Some(parsed) = version
.game_version
.chars()
.skip(2)
.take(version.game_version.len())
.collect();
if version_number.parse::<f32>()? < 14.0 {
using_forge = true;
.get(2..)
.and_then(|f| f.parse::<f32>().ok())
{
if parsed < 14.0 {
using_forge = true;
}
}
mod_game_versions.push(version.game_version);
@@ -188,17 +191,13 @@ pub async fn index_curseforge(
date_created: curseforge_mod.date_created.chars().take(10).collect(),
created: curseforge_mod
.date_created
.chars()
.filter(|c| c.is_ascii_digit())
.collect::<String>()
.parse()?,
.parse::<chrono::DateTime<chrono::Utc>>()?
.timestamp(),
date_modified: curseforge_mod.date_modified.chars().take(10).collect(),
updated: curseforge_mod
.date_modified
.chars()
.filter(|c| c.is_ascii_digit())
.collect::<String>()
.parse()?,
.parse::<chrono::DateTime<chrono::Utc>>()?
.timestamp(),
latest_version,
empty: String::from("{}{}{}"),
})

24
src/search/indexing/local_import.rs
View File

@@ -3,11 +3,12 @@ use futures::StreamExt;
use log::info;
use crate::database::models::Item;
use crate::database::{Mod, Version};
use crate::database::{DatabaseError, Mod, Version};
use crate::search::{SearchError, SearchMod};
use super::IndexingError;
use crate::search::SearchMod;
pub async fn index_local(client: mongodb::Client) -> Result<Vec<SearchMod>, SearchError> {
pub async fn index_local(client: mongodb::Client) -> Result<Vec<SearchMod>, IndexingError> {
info!("Indexing local mods!");
let mut docs_to_add: Vec<SearchMod> = vec![];
@@ -17,17 +18,26 @@ pub async fn index_local(client: mongodb::Client) -> Result<Vec<SearchMod>, Sear
let mods = db.collection("mods");
let versions = db.collection("versions");
let mut results = mods.find(None, None).await?;
let mut results = mods
.find(None, None)
.await
.map_err(DatabaseError::LocalDatabaseError)?;
while let Some(unparsed_result) = results.next().await {
let result: Mod = *Mod::from_doc(unparsed_result?)?;
let result: Mod =
*Mod::from_doc(unparsed_result.map_err(DatabaseError::LocalDatabaseError)?)?;
let mut mod_versions = versions.find(doc! { "mod_id": result.id}, None).await?;
let mut mod_versions = versions
.find(doc! { "mod_id": result.id }, None)
.await
.map_err(DatabaseError::LocalDatabaseError)?;
let mut mod_game_versions = vec![];
while let Some(unparsed_version) = mod_versions.next().await {
let mut version: Version = *Version::from_doc(unparsed_version?)?;
let mut version = unparsed_version
.map_err(DatabaseError::LocalDatabaseError)
.and_then(Version::from_doc)?;
mod_game_versions.append(&mut version.game_versions);
}

62
src/search/indexing/mod.rs
View File

@@ -4,12 +4,29 @@ pub mod local_import;
use crate::search::indexing::curseforge_import::index_curseforge;
use crate::search::indexing::local_import::index_local;
use crate::search::{SearchError, SearchMod};
use crate::search::SearchMod;
use meilisearch_sdk::client::Client;
use meilisearch_sdk::settings::Settings;
use std::collections::{HashMap, VecDeque};
use thiserror::Error;
pub async fn index_mods(db: mongodb::Client) -> Result<(), SearchError> {
#[derive(Error, Debug)]
pub enum IndexingError {
#[error("Error while connecting to the MeiliSearch database")]
IndexDBError(meilisearch_sdk::errors::Error),
#[error("Error while importing mods from CurseForge")]
CurseforgeImportError(reqwest::Error),
#[error("Error while serializing or deserializing JSON: {0}")]
SerDeError(#[from] serde_json::Error),
#[error("Error while parsing a timestamp: {0}")]
ParseDateError(#[from] chrono::format::ParseError),
#[error("Database Error: {0}")]
DatabaseError(#[from] crate::database::DatabaseError),
#[error("Environment Error")]
EnvError(#[from] dotenv::Error),
}
pub async fn index_mods(db: mongodb::Client) -> Result<(), IndexingError> {
// Check if the index exists
let address = &*dotenv::var("MEILISEARCH_ADDR")?;
let client = Client::new(address, "");
@@ -17,66 +34,73 @@ pub async fn index_mods(db: mongodb::Client) -> Result<(), SearchError> {
let mut docs_to_add: Vec<SearchMod> = vec![];
docs_to_add.append(&mut index_local(db.clone()).await?);
if dotenv::var("INDEX_CURSEFORGE")
.expect("`INDEX_CURSEFORGE` is missing in the .env file.")
if dotenv::var("INDEX_CURSEFORGE")?
.parse()
.unwrap()
.expect("`INDEX_CURSEFORGE` is not a boolean.")
{
docs_to_add.append(&mut index_curseforge(1, 400000).await?);
}
//Write Indexes
//Relevance Index
let mut relevance_index = client.get_or_create("relevance_mods").unwrap();
let mut relevance_index = client
.get_or_create("relevance_mods")
.map_err(IndexingError::IndexDBError)?;
let mut relevance_rules = default_rules();
relevance_rules.push_back("desc(downloads)".to_string());
relevance_index
.set_settings(&default_settings().with_ranking_rules(relevance_rules.into()))
.unwrap();
.map_err(IndexingError::IndexDBError)?;
relevance_index
.add_documents(docs_to_add.clone(), Some("mod_id"))
.unwrap();
.map_err(IndexingError::IndexDBError)?;
//Downloads Index
let mut downloads_index = client.get_or_create("downloads_mods").unwrap();
let mut downloads_index = client
.get_or_create("downloads_mods")
.map_err(IndexingError::IndexDBError)?;
let mut downloads_rules = default_rules();
downloads_rules.push_front("desc(downloads)".to_string());
downloads_index
.set_settings(&default_settings().with_ranking_rules(downloads_rules.into()))
.unwrap();
.map_err(IndexingError::IndexDBError)?;
downloads_index
.add_documents(docs_to_add.clone(), Some("mod_id"))
.unwrap();
.map_err(IndexingError::IndexDBError)?;
//Updated Index
let mut updated_index = client.get_or_create("updated_mods").unwrap();
let mut updated_index = client
.get_or_create("updated_mods")
.map_err(IndexingError::IndexDBError)?;
let mut updated_rules = default_rules();
updated_rules.push_front("desc(updated)".to_string());
updated_index
.set_settings(&default_settings().with_ranking_rules(updated_rules.into()))
.unwrap();
.map_err(IndexingError::IndexDBError)?;
updated_index
.add_documents(docs_to_add.clone(), Some("mod_id"))
.unwrap();
.map_err(IndexingError::IndexDBError)?;
//Created Index
let mut newest_index = client.get_or_create("newest_mods").unwrap();
let mut newest_index = client
.get_or_create("newest_mods")
.map_err(IndexingError::IndexDBError)?;
let mut newest_rules = default_rules();
newest_rules.push_back("desc(created)".to_string());
newest_index
.set_settings(&default_settings().with_ranking_rules(newest_rules.into()))
.unwrap();
.map_err(IndexingError::IndexDBError)?;
newest_index
.add_documents(docs_to_add.clone(), Some("mod_id"))
.unwrap();
.map_err(IndexingError::IndexDBError)?;
Ok(())
}
@@ -124,8 +148,8 @@ fn default_settings() -> Settings {
];
Settings::new()
.with_displayed_attributes(displayed_attributes.clone())
.with_searchable_attributes(searchable_attributes.clone())
.with_displayed_attributes(displayed_attributes)
.with_searchable_attributes(searchable_attributes)
.with_accept_new_fields(true)
.with_stop_words(vec![])
.with_synonyms(HashMap::new())