Rewrite the app (#23)

* chore: Removed everything not needed, and added base for rewrite
feat(error_handling): Added 404 general cache
feat(index): Added informations about the app in the / route.

* feat(indexing): Brought back the indexing, with conditions to make it easier

* fix: Fixed build error with a forgotten call

* feat: Add Docker development enviroment (#19)

* ci: add a *lot* of new actions

* fix: rename linting action

* fix: invalid yaml begone(?)

* ci: Added cache to speed up build times

* fix(ci): 🦀ed the yaml errors

* fix(ci): fixed a missing hyphen

* ci: Added matrix of rust versions, and changed way to install rust toolchain

* fix(ci): Added names to build with the matrix so it's easier to find the source of the problem

* style(ci): Added eof lines

* refactor: Finished moving the search.rs file to a separate module.

* Search Endpoint

* refactor: Moved around functions and struct for a better understanding of what it does.

* chore: Change env default settings to resolve conversation

* refactor: Removed #[use_macros]
fix: Fixed meilisearch address from env

* chore: Added email to Aeledfyr

* fix: Brought back the dotenv variables

* style: Ran `cargo fmt`

Co-authored-by: Charalampos Fanoulis <charalampos.fanoulis@gmail.com>
Co-authored-by: Jai A <jai.a@tuta.io>
This commit is contained in:
Valentin Ricard
2020-06-30 19:23:52 +02:00
committed by GitHub
parent 1ff2a08d19
commit 7a6ecd86c6
87 changed files with 881 additions and 3231 deletions

View File

@@ -1,12 +0,0 @@
use mongodb::options::ClientOptions;
use mongodb::Client;
use mongodb::error::Error;
pub async fn connect() -> Result<Client, Error> {
info!("Initializing database connection");
let mut client_options = ClientOptions::parse("mongodb://localhost:27017").await?;
client_options.app_name = Some("Actix Web Server".to_string());
Client::with_options(client_options)
}

View File

@@ -1,6 +1,19 @@
mod database;
mod models;
pub mod models;
mod mongo_database;
pub use database::connect;
pub use models::Mod;
pub use models::Version;
pub use mongo_database::connect;
use thiserror::Error;
type Result<T> = std::result::Result<T, DatabaseError>;
#[derive(Error, Debug)]
pub enum DatabaseError {
#[error("Impossible to find document")]
NotFound(),
#[error("BSON deserialization error")]
BsonError(#[from] bson::de::Error),
#[error("Local database error")]
LocalDatabaseError(#[from] mongodb::error::Error),
}

View File

@@ -1,28 +0,0 @@
use serde::{Serialize, Deserialize};
#[derive(Deserialize, Serialize)]
pub struct Mod {
pub id: i32,
pub title: String,
pub description: String,
pub published: String,
pub author: String,
pub downloads: i32,
pub categories: Vec<String>,
pub body_path: String,
pub icon_path: String,
}
#[derive(Deserialize, Serialize)]
pub struct Version {
pub version_id: i32,
pub mod_id: i32,
pub title: String,
pub changelog_path: String,
pub files_path: Vec<String>,
pub date_published: String,
pub author: String,
pub downloads: i32,
pub dependencies: Vec<String>,
pub game_versions: Vec<String>,
}

View File

@@ -0,0 +1,27 @@
mod mod_item;
mod version_item;
use crate::database::DatabaseError::NotFound;
use crate::database::Result;
use async_trait::async_trait;
use bson::doc;
use bson::Document;
pub use mod_item::Mod;
use mongodb::Database;
pub use version_item::Version;
#[async_trait]
pub trait Item {
fn get_collection() -> &'static str;
async fn get_by_id(client: Database, id: &str) -> Result<Box<Self>> {
let filter = doc! { "_id": id };
let collection = client.collection(Self::get_collection());
let doc: Document = match collection.find_one(filter, None).await? {
Some(e) => e,
None => return Err(NotFound()),
};
let elem: Box<Self> = Self::from_doc(doc)?;
Ok(elem)
}
fn from_doc(elem: Document) -> Result<Box<Self>>;
}

View File

@@ -0,0 +1,27 @@
use crate::database::models::Item;
use crate::database::Result;
use bson::{Bson, Document};
use serde::{Deserialize, Serialize};
#[derive(Deserialize, Serialize)]
pub struct Mod {
pub id: i32,
pub title: String,
pub description: String,
pub published: String,
pub author: String,
pub downloads: i32,
pub categories: Vec<String>,
pub body_path: String,
pub icon_path: String,
}
impl Item for Mod {
fn get_collection() -> &'static str {
"mods"
}
fn from_doc(elem: Document) -> Result<Box<Mod>> {
let result: Mod = bson::from_bson(Bson::from(elem))?;
Ok(Box::from(result))
}
}

View File

@@ -0,0 +1,28 @@
use crate::database::models::Item;
use crate::database::Result;
use bson::{Bson, Document};
use serde::{Deserialize, Serialize};
#[derive(Deserialize, Serialize)]
pub struct Version {
pub version_id: i32,
pub mod_id: i32,
pub title: String,
pub changelog_path: String,
pub files_path: Vec<String>,
pub date_published: String,
pub author: String,
pub downloads: i32,
pub dependencies: Vec<String>,
pub game_versions: Vec<String>,
}
impl Item for Version {
fn get_collection() -> &'static str {
"versions"
}
fn from_doc(elem: Document) -> Result<Box<Version>> {
let version: Version = bson::from_bson(Bson::from(elem))?;
Ok(Box::from(version))
}
}

View File

@@ -0,0 +1,13 @@
use log::info;
use mongodb::error::Error;
use mongodb::options::ClientOptions;
use mongodb::Client;
pub async fn connect() -> Result<Client, Error> {
info!("Initializing database connection");
let mut client_options = ClientOptions::parse(&dotenv::var("PORT").unwrap()).await?;
client_options.app_name = Some("labrinth".to_string());
Client::with_options(client_options)
}

View File

@@ -1,35 +0,0 @@
use handlebars::*;
#[derive(Clone, Copy)]
pub struct ContainsHelper;
impl HelperDef for ContainsHelper {
fn call<'reg: 'rc, 'rc>(
&self,
h: &Helper<'reg, 'rc>,
r: &'reg Handlebars<'_>,
ctx: &'rc Context,
rc: &mut RenderContext<'reg, 'rc>,
out: &mut dyn Output,
) -> HelperResult {
let array = h
.param(0)
.map(|v| serde_json::from_value::<Vec<String>>(v.value().clone()).unwrap())
.ok_or_else(|| RenderError::new("Parameter not found!"))?;
let value = h
.param(1)
.map(|v| v.value().as_str().unwrap())
.ok_or_else(|| RenderError::new("Parameter not found!"))?;
let tmpl = if array.contains(&String::from(value)) {
h.template()
} else {
h.inverse()
};
match tmpl {
Some(ref t) => t.render(r, ctx, rc, out),
None => Ok(()),
}
}
}

View File

@@ -1,36 +0,0 @@
use handlebars::*;
#[derive(Clone, Copy)]
pub struct EqualsHelper;
impl HelperDef for EqualsHelper {
fn call<'reg: 'rc, 'rc>(
&self,
h: &Helper<'reg, 'rc>,
r: &'reg Handlebars<'_>,
ctx: &'rc Context,
rc: &mut RenderContext<'reg, 'rc>,
out: &mut dyn Output,
) -> HelperResult {
let a = h
.param(0)
.map(|v| v.value().as_object().unwrap())
.ok_or_else(|| RenderError::new("Parameter not found!"))?;
let b = h
.param(1)
.map(|v| v.value().as_object().unwrap())
.ok_or_else(|| RenderError::new("Parameter not found!"))?;
let tmpl = if a == b {
h.template()
} else {
h.inverse()
};
match tmpl {
Some(ref t) => t.render(r, ctx, rc, out),
None => Ok(()),
}
}
}

View File

@@ -1,31 +0,0 @@
extern crate human_format;
use handlebars::*;
#[derive(Clone, Copy)]
pub struct HumanFormatHelper;
impl HelperDef for HumanFormatHelper {
fn call<'reg: 'rc, 'rc>(
&self,
h: &Helper<'reg, 'rc>,
_r: &'reg Handlebars<'_>,
_ctx: &'rc Context,
_rc: &mut RenderContext<'reg, 'rc>,
out: &mut dyn Output,
) -> HelperResult {
let param = h.param(0).and_then(|v| v.value().as_f64()).unwrap_or(0.0);
let string = &param.to_string();
if string.len() > 3 {
let mut formatted = human_format::Formatter::new().format(param);
formatted.retain(|c| !c.is_whitespace());
out.write(formatted.to_uppercase().as_ref())?;
} else {
out.write(string)?;
}
Ok(())
}
}

View File

@@ -1,13 +0,0 @@
mod contains;
mod format_human;
mod equals;
use handlebars::*;
pub fn register_helpers(handlebars: &mut Handlebars) {
handlebars.register_helper("contains", Box::new(contains::ContainsHelper));
handlebars.register_helper("format", Box::new(format_human::HumanFormatHelper));
//This helper is not used yet, but could be useful in many circumstances
handlebars.register_helper("equals", Box::new(equals::EqualsHelper));
}

View File

@@ -1,40 +1,40 @@
#[macro_use]
extern crate serde_json;
#[macro_use]
extern crate bson;
#[macro_use]
extern crate log;
use actix_files as fs;
use actix_web::{web, App, HttpServer};
use handlebars::*;
use crate::search::indexing::index_mods;
use actix_web::middleware::Logger;
use actix_web::{web, App, HttpServer};
use env_logger::Env;
use log::info;
use std::env;
use std::fs::File;
mod database;
mod helpers;
mod models;
mod routes;
mod search;
#[actix_rt::main]
async fn main() -> std::io::Result<()> {
env_logger::from_env(Env::default().default_filter_or("info")).init();
dotenv::dotenv().ok();
//Handlebars
let mut handlebars = Handlebars::new();
helpers::register_helpers(&mut handlebars);
handlebars
.register_templates_directory(".hbs", "./templates")
.unwrap();
let handlebars_ref = web::Data::new(handlebars);
let client = database::connect().await.unwrap();
routes::index_mods(client).await.unwrap();
// Get executable path
let mut exe_path = env::current_exe()?.parent().unwrap().to_path_buf();
// Create the path to the index lock file
exe_path.push("index.v1.lock");
//Indexing mods if not already done
if env::args().any(|x| x == "regen") {
// User forced regen of indexing
info!("Forced regeneration of indexes!");
index_mods(client).await.unwrap();
} else if exe_path.exists() {
// The indexes were not created, or the version was upgraded
info!("Indexing of mods for first time...");
index_mods(client).await.unwrap();
// Create the lock file
File::create(exe_path)?;
}
info!("Starting Actix HTTP server!");
@@ -43,15 +43,11 @@ async fn main() -> std::io::Result<()> {
App::new()
.wrap(Logger::default())
.wrap(Logger::new("%a %{User-Agent}i"))
.app_data(handlebars_ref.clone())
.service(fs::Files::new("/static", "./static").show_files_listing())
.service(routes::index_get)
.service(routes::search_post)
.service(routes::search_get)
.service(routes::mod_page_get)
.service(routes::mod_create_get)
.service(routes::mod_search)
.default_service(web::get().to(routes::not_found))
})
.bind("127.0.0.1:8000")?
.bind("127.0.0.1:".to_string() + &dotenv::var("PORT").unwrap())?
.run()
.await
}

6
src/models/error.rs Normal file
View File

@@ -0,0 +1,6 @@
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize)]
pub struct ApiError<'a> {
pub error: &'a str,
pub description: &'a str,
}

2
src/models/mod.rs Normal file
View File

@@ -0,0 +1,2 @@
pub mod error;
pub mod mods;

10
src/models/mods.rs Normal file
View File

@@ -0,0 +1,10 @@
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize)]
pub struct SearchRequest {
pub query: Option<String>,
pub filters: Option<String>,
pub version: Option<String>,
pub offset: Option<String>,
pub index: Option<String>,
}

View File

@@ -1,12 +1,15 @@
use actix_web::{get, web, HttpResponse};
use handlebars::*;
use serde_json::json;
#[get("/")]
pub async fn index_get(hb: web::Data<Handlebars<'_>>) -> HttpResponse {
pub async fn index_get() -> HttpResponse {
let data = json!({
"name": "Handlebars"
"name": "modrinth-labrinth",
"version": env!("CARGO_PKG_VERSION"),
//TODO: Add the documentation link
"documentation": "Nowhere yet",
"about": "Welcome traveler !"
});
let body = hb.render("index", &data).unwrap();
HttpResponse::Ok().body(body)
HttpResponse::Ok().json(data)
}

View File

@@ -1,14 +1,7 @@
mod index;
mod mod_page;
mod search;
mod mod_create;
pub use self::mod_page::mod_page_get;
pub use self::mod_create::mod_create_get;
pub use self::search::index_mods;
pub use self::search::search_get;
pub use self::search::search_post;
mod mods;
mod not_found;
pub use self::index::index_get;
pub use self::mods::mod_search;
pub use self::not_found::not_found;

View File

@@ -1,30 +0,0 @@
use actix_web::{get, post, web, HttpResponse};
use handlebars::*;
use serde::Deserialize;
#[derive(Deserialize)]
struct CreatedMod {
name: String,
description: String,
body: String,
}
#[get("createmod")]
pub async fn mod_create_get(hb: web::Data<Handlebars<'_>>) -> HttpResponse {
let data = json!({
"name": "Handlebars"
});
let body = hb.render("mod-create", &data).unwrap();
HttpResponse::Ok().body(body)
}
#[post("createmod")]
pub async fn mod_create_post(hb: web::Data<Handlebars<'_>>) -> HttpResponse {
let data = json!({
"name": "Handlebars"
});
let body = hb.render("mod-create", &data).unwrap();
HttpResponse::Ok().body(body)
}

View File

@@ -1,12 +0,0 @@
use actix_web::{get, web, HttpResponse};
use handlebars::*;
#[get("mod/testmod")]
pub async fn mod_page_get(hb: web::Data<Handlebars<'_>>) -> HttpResponse {
let data = json!({
"name": "Handlebars"
});
let body = hb.render("mod-page", &data).unwrap();
HttpResponse::Ok().body(body)
}

13
src/routes/mods.rs Normal file
View File

@@ -0,0 +1,13 @@
use crate::models::mods::SearchRequest;
use crate::search::search_for_mod;
use actix_web::{get, web, HttpResponse};
#[get("api/v1/mods")]
pub fn mod_search(web::Query(info): web::Query<SearchRequest>) -> HttpResponse {
//TODO: Fix this line with anyhow
let body = serde_json::to_string(&search_for_mod(&info).unwrap()).unwrap();
HttpResponse::Ok()
.content_type("application/json")
.body(body)
}

11
src/routes/not_found.rs Normal file
View File

@@ -0,0 +1,11 @@
use crate::models::error::ApiError;
use actix_web::{HttpResponse, Responder};
pub async fn not_found() -> impl Responder {
let data = ApiError {
error: "not_found",
description: "the route you called is not (yet) implemented",
};
HttpResponse::NotFound().json(data)
}

View File

@@ -1,464 +0,0 @@
use actix_web::{get, post, web, web::Data, HttpResponse};
use handlebars::*;
use meilisearch_sdk::{client::*, document::*, search::*};
use serde::{Deserialize, Serialize};
use meilisearch_sdk::settings::Settings;
use futures::stream::StreamExt;
use bson::Bson;
use std::collections::{HashMap, VecDeque};
use std::error::Error;
use crate::database::*;
use futures_timer::Delay;
use futures::TryFutureExt;
use std::time::Duration;
#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct Attachment {
url: String,
thumbnail_url: String,
is_default: bool,
}
#[derive(Serialize, Deserialize, Debug)]
struct Category {
name: String,
}
#[derive(Serialize, Deserialize, Debug)]
struct Author {
name: String,
url: String,
}
#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct CurseVersion {
game_version: String,
}
#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
struct CurseForgeMod {
id: i32,
name: String,
authors: Vec<Author>,
attachments: Vec<Attachment>,
website_url: String,
summary: String,
download_count: f32,
categories: Vec<Category>,
game_version_latest_files: Vec<CurseVersion>,
date_created: String,
date_modified: String,
game_slug: String,
}
#[derive(Serialize, Deserialize, Debug, Clone)]
struct SearchMod {
mod_id: i32,
author: String,
title: String,
description: String,
keywords: Vec<String>,
versions: Vec<String>,
downloads: i32,
page_url: String,
icon_url: String,
author_url: String,
date_created: String,
created: i64,
date_modified: String,
updated: i64,
latest_version: String,
empty: String,
}
impl Document for SearchMod {
type UIDType = i32;
fn get_uid(&self) -> &Self::UIDType {
&self.mod_id
}
}
#[derive(Serialize, Deserialize)]
pub struct SearchRequest {
#[serde(rename = "q")]
query: Option<String>,
#[serde(rename = "f")]
filters: Option<String>,
#[serde(rename = "v")]
version: Option<String>,
#[serde(rename = "o")]
offset: Option<String>,
#[serde(rename = "s")]
index: Option<String>,
}
#[post("search")]
pub async fn search_post(
web::Query(info): web::Query<SearchRequest>,
hb: Data<Handlebars<'_>>,
) -> HttpResponse {
let results = search(&info);
let data = json!({
"query": info,
"results": results,
});
let body = hb.render("search-results", &data).unwrap();
HttpResponse::Ok().body(body)
}
#[get("search")]
pub async fn search_get(
web::Query(info): web::Query<SearchRequest>,
hb: Data<Handlebars<'_>>,
) -> HttpResponse {
let results = search(&info);
let data = json!({
"query": info,
"results": results,
});
let body = hb.render("search", &data).unwrap();
HttpResponse::Ok().body(body)
}
fn search(info: &SearchRequest) -> Vec<SearchMod> {
let client = Client::new("http://localhost:7700", "");
let search_query: &str;
let mut filters = String::new();
let mut offset = 0;
let mut index = "relevance";
match info.query.as_ref() {
Some(q) => search_query = q,
None => search_query = "{}{}{}",
}
if let Some(f) = info.filters.as_ref() {
filters = f.clone();
}
if let Some(v) = info.version.as_ref() {
if filters.is_empty() {
filters = v.clone();
} else {
filters = format!("({}) AND ({})", filters, v);
}
}
if let Some(o) = info.offset.as_ref() {
offset = o.parse().unwrap();
}
if let Some(s) = info.index.as_ref() {
index = s;
}
let mut query = Query::new(search_query).with_limit(10).with_offset(offset);
if !filters.is_empty() {
query = query.with_filters(&filters);
}
client.get_index(format!("{}_mods", index).as_ref()).unwrap()
.search::<SearchMod>(&query).unwrap().hits
}
/*
TODO This method needs a lot of refactoring. Here's a list of changes that need to be made:
- Move Curseforge Indexing to another method/module
- Get rid of the 4 indexes (when MeiliSearch updates) and replace it with different rules
- Remove code fragment duplicates
*/
pub async fn index_mods(db: mongodb::Client) -> Result<(), Box<dyn Error>>{
let mut docs_to_add: Vec<SearchMod> = vec![];
docs_to_add.append(&mut index_database(db.clone()).await?);
//docs_to_add.append(&mut index_curseforge(1, 400000).await?);
//Write Indexes
//Relevance Index
let client = Client::new("http://localhost:7700", "");
let mut relevance_index = client.get_or_create("relevance_mods").unwrap();
let mut relevance_rules = default_rules();
relevance_rules.push_back("desc(downloads)".to_string());
relevance_index.set_settings(&default_settings().with_ranking_rules(relevance_rules.into())).unwrap();
relevance_index.add_documents(docs_to_add.clone(), Some("mod_id")).unwrap();
//Downloads Index
let mut downloads_index = client.get_or_create("downloads_mods").unwrap();
let mut downloads_rules = default_rules();
downloads_rules.push_front("desc(downloads)".to_string());
downloads_index.set_settings(&default_settings().with_ranking_rules(downloads_rules.into())).unwrap();
downloads_index.add_documents(docs_to_add.clone(), Some("mod_id")).unwrap();
//Updated Index
let mut updated_index = client.get_or_create("updated_mods").unwrap();
let mut updated_rules = default_rules();
updated_rules.push_front("desc(updated)".to_string());
updated_index.set_settings(&default_settings().with_ranking_rules(updated_rules.into())).unwrap();
updated_index.add_documents(docs_to_add.clone(), Some("mod_id")).unwrap();
//Created Index
let mut newest_index = client.get_or_create("newest_mods").unwrap();
let mut newest_rules = default_rules();
newest_rules.push_back("desc(created)".to_string());
newest_index.set_settings(&default_settings().with_ranking_rules(newest_rules.into())).unwrap();
newest_index.add_documents(docs_to_add.clone(), Some("mod_id")).unwrap();
Ok(())
}
async fn index_database(client: mongodb::Client) -> Result<Vec<SearchMod>, Box<dyn Error>> {
info!("Indexing database mods!");
let mut docs_to_add: Vec<SearchMod> = vec![];
let db = client.database("fabricate");
let mods = db.collection("mods");
let versions = db.collection("versions");
let mut results = mods.find(None, None).await?;
while let Some(unparsed_result) = results.next().await {
let result : Mod = bson::from_bson(Bson::from(unparsed_result?))?;
let mut mod_versions = versions.find(doc!{ "mod_id": result.id}, None).await?;
let mut mod_game_versions = vec![];
while let Some(unparsed_version) = mod_versions.next().await {
let mut version : Version = bson::from_bson(Bson::from(unparsed_version?))?;
mod_game_versions.append(&mut version.game_versions);
}
docs_to_add.push(SearchMod {
mod_id: result.id,
author: result.author,
title: result.title,
description: result.description,
keywords: result.categories,
versions: mod_game_versions,
downloads: result.downloads,
page_url: "".to_string(),
icon_url: result.icon_path,
author_url: "".to_string(),
date_created: "".to_string(),
created: 0,
date_modified: "".to_string(),
updated: 0,
latest_version: "".to_string(),
empty: String::from("{}{}{}"),
});
}
Ok(docs_to_add)
}
async fn index_curseforge(start_index: i32, end_index: i32) -> Result<Vec<SearchMod>, Box<dyn Error>>{
info!("Indexing curseforge mods!");
let mut docs_to_add: Vec<SearchMod> = vec![];
let res = reqwest::Client::new().post("https://addons-ecs.forgesvc.net/api/v2/addon")
.header(reqwest::header::CONTENT_TYPE, "application/json")
.body(format!("{:?}", (start_index..end_index).collect::<Vec<_>>()))
.send().await?;
let text = &res.text().await?;
let curseforge_mods : Vec<CurseForgeMod> = serde_json::from_str(text)?;
let mut max_index = 0;
for curseforge_mod in curseforge_mods {
max_index = curseforge_mod.id;
if curseforge_mod.game_slug != "minecraft" || !curseforge_mod.website_url.contains("/mc-mods/") { continue; }
let mut mod_game_versions = vec![];
let mut using_forge = false;
let mut using_fabric = false;
for version in curseforge_mod.game_version_latest_files {
let version_number: String = version
.game_version
.chars()
.skip(2)
.take(version.game_version.len())
.collect();
if version_number.parse::<f32>()? < 14.0 {
using_forge = true;
}
mod_game_versions.push(version.game_version);
}
let mut mod_categories = vec![];
for category in curseforge_mod.categories {
match &category.name[..] {
"World Gen" => mod_categories.push(String::from("worldgen")),
"Biomes" => mod_categories.push(String::from("worldgen")),
"Ores and Resources" => mod_categories.push(String::from("worldgen")),
"Structures" => mod_categories.push(String::from("worldgen")),
"Dimensions" => mod_categories.push(String::from("worldgen")),
"Mobs" => mod_categories.push(String::from("worldgen")),
"Technology" => mod_categories.push(String::from("technology")),
"Processing" => mod_categories.push(String::from("technology")),
"Player Transport" => mod_categories.push(String::from("technology")),
"Energy, Fluid, and Item Transport" => { mod_categories.push(String::from("technology")) }
"Food" => mod_categories.push(String::from("food")),
"Farming" => mod_categories.push(String::from("food")),
"Energy" => mod_categories.push(String::from("technology")),
"Redstone" => mod_categories.push(String::from("technology")),
"Genetics" => mod_categories.push(String::from("technology")),
"Magic" => mod_categories.push(String::from("magic")),
"Storage" => mod_categories.push(String::from("storage")),
"API and Library" => mod_categories.push(String::from("library")),
"Adventure and RPG" => mod_categories.push(String::from("adventure")),
"Map and Information" => mod_categories.push(String::from("utility")),
"Cosmetic" => mod_categories.push(String::from("decoration")),
"Addons" => mod_categories.push(String::from("misc")),
"Thermal Expansion" => mod_categories.push(String::from("misc")),
"Tinker's Construct" => mod_categories.push(String::from("misc")),
"Industrial Craft" => mod_categories.push(String::from("misc")),
"Thaumcraft" => mod_categories.push(String::from("misc")),
"Buildcraft" => mod_categories.push(String::from("misc")),
"Forestry" => mod_categories.push(String::from("misc")),
"Blood Magic" => mod_categories.push(String::from("misc")),
"Lucky Blocks" => mod_categories.push(String::from("misc")),
"Applied Energistics 2" => mod_categories.push(String::from("misc")),
"CraftTweaker" => mod_categories.push(String::from("misc")),
"Miscellaneous" => mod_categories.push(String::from("misc")),
"Armor, Tools, and Weapons" => mod_categories.push(String::from("equipment")),
"Server Utility" => mod_categories.push(String::from("utility")),
"Fabric" => mod_categories.push(String::from("fabric")),
_ => {}
}
}
if mod_categories.contains(&"fabric".to_owned()) {
using_fabric = true;
}
mod_categories.sort();
mod_categories.dedup();
mod_categories.truncate(3);
if using_forge {
mod_categories.push(String::from("forge"));
}
if using_fabric {
mod_categories.push(String::from("fabric"));
}
let mut mod_attachments = curseforge_mod.attachments;
mod_attachments.retain(|x| x.is_default);
if mod_attachments.is_empty() {
mod_attachments.push(Attachment {
url: String::new(),
thumbnail_url: String::new(),
is_default: true,
})
}
let latest_version = if !mod_game_versions.is_empty() {
mod_game_versions[0].to_string()
} else {
"None".to_string()
};
let icon_url = mod_attachments[0].thumbnail_url.replace("/256/256/", "/64/64/");
docs_to_add.push(SearchMod {
mod_id: -curseforge_mod.id,
author: (&curseforge_mod.authors[0].name).to_string(),
title: curseforge_mod.name,
description: curseforge_mod.summary.chars().take(150).collect(),
keywords: mod_categories,
versions: mod_game_versions.clone(),
downloads: curseforge_mod.download_count as i32,
page_url: curseforge_mod.website_url,
icon_url,
author_url: (&curseforge_mod.authors[0].url).to_string(),
date_created: curseforge_mod.date_created.chars().take(10).collect(),
created: curseforge_mod.date_created.chars().filter(|c| c.is_ascii_digit()).collect::<String>().parse()?,
date_modified: curseforge_mod.date_modified.chars().take(10).collect(),
updated: curseforge_mod.date_modified.chars().filter(|c| c.is_ascii_digit()).collect::<String>().parse()?,
latest_version,
empty: String::from("{}{}{}"),
})
}
//TODO Reindex every hour for new mods.
Ok(docs_to_add)
}
fn default_rules() -> VecDeque<String> {
vec![
"typo".to_string(),
"words".to_string(),
"proximity".to_string(),
"attribute".to_string(),
"wordsPosition".to_string(),
"exactness".to_string(),
].into()
}
fn default_settings() -> Settings {
let displayed_attributes = vec![
"mod_id".to_string(),
"author".to_string(),
"title".to_string(),
"description".to_string(),
"keywords".to_string(),
"versions".to_string(),
"downloads".to_string(),
"page_url".to_string(),
"icon_url".to_string(),
"author_url".to_string(),
"date_created".to_string(),
"created".to_string(),
"date_modified".to_string(),
"updated".to_string(),
"latest_version".to_string(),
"empty".to_string(),
];
let searchable_attributes = vec![
"title".to_string(),
"description".to_string(),
"keywords".to_string(),
"versions".to_string(),
"author".to_string(),
"empty".to_string(),
];
Settings::new()
.with_displayed_attributes(displayed_attributes.clone())
.with_searchable_attributes(searchable_attributes.clone())
.with_accept_new_fields(true)
.with_stop_words(vec![])
.with_synonyms(HashMap::new())
}

View File

@@ -0,0 +1,212 @@
use crate::search::{SearchError, SearchMod};
use log::info;
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct Attachment {
pub url: String,
pub thumbnail_url: String,
pub is_default: bool,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct Category {
pub name: String,
}
#[derive(Serialize, Deserialize, Debug)]
pub struct Author {
pub name: String,
pub url: String,
}
#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct CurseVersion {
pub game_version: String,
}
#[derive(Serialize, Deserialize, Debug)]
#[serde(rename_all = "camelCase")]
pub struct CurseForgeMod {
pub id: i32,
pub name: String,
pub authors: Vec<Author>,
pub attachments: Vec<Attachment>,
pub website_url: String,
pub summary: String,
pub download_count: f32,
pub categories: Vec<Category>,
pub game_version_latest_files: Vec<CurseVersion>,
pub date_created: String,
pub date_modified: String,
pub game_slug: String,
}
pub async fn index_curseforge(
start_index: i32,
end_index: i32,
) -> Result<Vec<SearchMod>, SearchError> {
info!("Indexing curseforge mods!");
let mut docs_to_add: Vec<SearchMod> = vec![];
let res = reqwest::Client::new()
.post("https://addons-ecs.forgesvc.net/api/v2/addon")
.header(reqwest::header::CONTENT_TYPE, "application/json")
.body(format!(
"{:?}",
(start_index..end_index).collect::<Vec<_>>()
))
.send()
.await?;
let text = &res.text().await?;
let curseforge_mods: Vec<CurseForgeMod> = serde_json::from_str(text)?;
let mut max_index = 0;
for curseforge_mod in curseforge_mods {
max_index = curseforge_mod.id;
if curseforge_mod.game_slug != "minecraft"
|| !curseforge_mod.website_url.contains("/mc-mods/")
{
continue;
}
let mut mod_game_versions = vec![];
let mut using_forge = false;
let mut using_fabric = false;
for version in curseforge_mod.game_version_latest_files {
let version_number: String = version
.game_version
.chars()
.skip(2)
.take(version.game_version.len())
.collect();
if version_number.parse::<f32>()? < 14.0 {
using_forge = true;
}
mod_game_versions.push(version.game_version);
}
let mut mod_categories = vec![];
for category in curseforge_mod.categories {
match &category.name[..] {
"World Gen" => mod_categories.push(String::from("worldgen")),
"Biomes" => mod_categories.push(String::from("worldgen")),
"Ores and Resources" => mod_categories.push(String::from("worldgen")),
"Structures" => mod_categories.push(String::from("worldgen")),
"Dimensions" => mod_categories.push(String::from("worldgen")),
"Mobs" => mod_categories.push(String::from("worldgen")),
"Technology" => mod_categories.push(String::from("technology")),
"Processing" => mod_categories.push(String::from("technology")),
"Player Transport" => mod_categories.push(String::from("technology")),
"Energy, Fluid, and Item Transport" => {
mod_categories.push(String::from("technology"))
}
"Food" => mod_categories.push(String::from("food")),
"Farming" => mod_categories.push(String::from("food")),
"Energy" => mod_categories.push(String::from("technology")),
"Redstone" => mod_categories.push(String::from("technology")),
"Genetics" => mod_categories.push(String::from("technology")),
"Magic" => mod_categories.push(String::from("magic")),
"Storage" => mod_categories.push(String::from("storage")),
"API and Library" => mod_categories.push(String::from("library")),
"Adventure and RPG" => mod_categories.push(String::from("adventure")),
"Map and Information" => mod_categories.push(String::from("utility")),
"Cosmetic" => mod_categories.push(String::from("decoration")),
"Addons" => mod_categories.push(String::from("misc")),
"Thermal Expansion" => mod_categories.push(String::from("misc")),
"Tinker's Construct" => mod_categories.push(String::from("misc")),
"Industrial Craft" => mod_categories.push(String::from("misc")),
"Thaumcraft" => mod_categories.push(String::from("misc")),
"Buildcraft" => mod_categories.push(String::from("misc")),
"Forestry" => mod_categories.push(String::from("misc")),
"Blood Magic" => mod_categories.push(String::from("misc")),
"Lucky Blocks" => mod_categories.push(String::from("misc")),
"Applied Energistics 2" => mod_categories.push(String::from("misc")),
"CraftTweaker" => mod_categories.push(String::from("misc")),
"Miscellaneous" => mod_categories.push(String::from("misc")),
"Armor, Tools, and Weapons" => mod_categories.push(String::from("equipment")),
"Server Utility" => mod_categories.push(String::from("utility")),
"Fabric" => mod_categories.push(String::from("fabric")),
_ => {}
}
}
if mod_categories.contains(&"fabric".to_owned()) {
using_fabric = true;
}
mod_categories.sort();
mod_categories.dedup();
mod_categories.truncate(3);
if using_forge {
mod_categories.push(String::from("forge"));
}
if using_fabric {
mod_categories.push(String::from("fabric"));
}
let mut mod_attachments = curseforge_mod.attachments;
mod_attachments.retain(|x| x.is_default);
if mod_attachments.is_empty() {
mod_attachments.push(Attachment {
url: String::new(),
thumbnail_url: String::new(),
is_default: true,
})
}
let latest_version = if !mod_game_versions.is_empty() {
mod_game_versions[0].to_string()
} else {
"None".to_string()
};
let icon_url = mod_attachments[0]
.thumbnail_url
.replace("/256/256/", "/64/64/");
docs_to_add.push(SearchMod {
mod_id: -curseforge_mod.id,
author: (&curseforge_mod.authors[0].name).to_string(),
title: curseforge_mod.name,
description: curseforge_mod.summary.chars().take(150).collect(),
keywords: mod_categories,
versions: mod_game_versions.clone(),
downloads: curseforge_mod.download_count as i32,
page_url: curseforge_mod.website_url,
icon_url,
author_url: (&curseforge_mod.authors[0].url).to_string(),
date_created: curseforge_mod.date_created.chars().take(10).collect(),
created: curseforge_mod
.date_created
.chars()
.filter(|c| c.is_ascii_digit())
.collect::<String>()
.parse()?,
date_modified: curseforge_mod.date_modified.chars().take(10).collect(),
updated: curseforge_mod
.date_modified
.chars()
.filter(|c| c.is_ascii_digit())
.collect::<String>()
.parse()?,
latest_version,
empty: String::from("{}{}{}"),
})
}
//TODO Reindex every hour for new mods.
Ok(docs_to_add)
}

View File

@@ -0,0 +1,57 @@
use bson::doc;
use bson::Bson;
use futures::StreamExt;
use log::info;
use meilisearch_sdk::client::Client;
use crate::database::models::Item;
use crate::database::{Mod, Version};
use crate::search::{SearchError, SearchMod, SearchRequest};
pub async fn index_local(client: mongodb::Client) -> Result<Vec<SearchMod>, SearchError> {
info!("Indexing local mods!");
let mut docs_to_add: Vec<SearchMod> = vec![];
let db = client.database("modrinth");
let mods = db.collection("mods");
let versions = db.collection("versions");
let mut results = mods.find(None, None).await?;
while let Some(unparsed_result) = results.next().await {
let result: Mod = *Mod::from_doc(unparsed_result?)?;
let mut mod_versions = versions.find(doc! { "mod_id": result.id}, None).await?;
let mut mod_game_versions = vec![];
while let Some(unparsed_version) = mod_versions.next().await {
let mut version: Version = *Version::from_doc(unparsed_version?)?;
mod_game_versions.append(&mut version.game_versions);
}
docs_to_add.push(SearchMod {
mod_id: result.id,
author: result.author,
title: result.title,
description: result.description,
keywords: result.categories,
versions: mod_game_versions,
downloads: result.downloads,
page_url: "".to_string(),
icon_url: result.icon_path,
author_url: "".to_string(),
date_created: "".to_string(),
created: 0,
date_modified: "".to_string(),
updated: 0,
latest_version: "".to_string(),
empty: String::from("{}{}{}"),
});
}
Ok(docs_to_add)
}

135
src/search/indexing/mod.rs Normal file
View File

@@ -0,0 +1,135 @@
/// This module is used for the indexing from any source.
pub mod curseforge_import;
pub mod local_import;
use crate::database::DatabaseError;
use crate::search::indexing::curseforge_import::index_curseforge;
use crate::search::indexing::local_import::index_local;
use crate::search::{SearchError, SearchMod};
use meilisearch_sdk::client::Client;
use meilisearch_sdk::settings::Settings;
use std::collections::{HashMap, VecDeque};
pub async fn index_mods(db: mongodb::Client) -> Result<(), SearchError> {
// Check if the index exists
let address = &*dotenv::var("MEILISEARCH_ADDR")?;
let client = Client::new(address, "");
let mut docs_to_add: Vec<SearchMod> = vec![];
docs_to_add.append(&mut index_local(db.clone()).await?);
if dotenv::var("INDEX_CURSEFORGE")
.expect("`INDEX_CURSEFORGE` is missing in the .env file.")
.parse()
.unwrap()
{
docs_to_add.append(&mut index_curseforge(1, 400000).await?);
}
//Write Indexes
//Relevance Index
let mut relevance_index = client.get_or_create("relevance_mods").unwrap();
let mut relevance_rules = default_rules();
relevance_rules.push_back("desc(downloads)".to_string());
relevance_index
.set_settings(&default_settings().with_ranking_rules(relevance_rules.into()))
.unwrap();
relevance_index
.add_documents(docs_to_add.clone(), Some("mod_id"))
.unwrap();
//Downloads Index
let mut downloads_index = client.get_or_create("downloads_mods").unwrap();
let mut downloads_rules = default_rules();
downloads_rules.push_front("desc(downloads)".to_string());
downloads_index
.set_settings(&default_settings().with_ranking_rules(downloads_rules.into()))
.unwrap();
downloads_index
.add_documents(docs_to_add.clone(), Some("mod_id"))
.unwrap();
//Updated Index
let mut updated_index = client.get_or_create("updated_mods").unwrap();
let mut updated_rules = default_rules();
updated_rules.push_front("desc(updated)".to_string());
updated_index
.set_settings(&default_settings().with_ranking_rules(updated_rules.into()))
.unwrap();
updated_index
.add_documents(docs_to_add.clone(), Some("mod_id"))
.unwrap();
//Created Index
let mut newest_index = client.get_or_create("newest_mods").unwrap();
let mut newest_rules = default_rules();
newest_rules.push_back("desc(created)".to_string());
newest_index
.set_settings(&default_settings().with_ranking_rules(newest_rules.into()))
.unwrap();
newest_index
.add_documents(docs_to_add.clone(), Some("mod_id"))
.unwrap();
Ok(())
}
//region Utils
fn default_rules() -> VecDeque<String> {
vec![
"typo".to_string(),
"words".to_string(),
"proximity".to_string(),
"attribute".to_string(),
"wordsPosition".to_string(),
"exactness".to_string(),
]
.into()
}
fn default_settings() -> Settings {
let displayed_attributes = vec![
"mod_id".to_string(),
"author".to_string(),
"title".to_string(),
"description".to_string(),
"keywords".to_string(),
"versions".to_string(),
"downloads".to_string(),
"page_url".to_string(),
"icon_url".to_string(),
"author_url".to_string(),
"date_created".to_string(),
"created".to_string(),
"date_modified".to_string(),
"updated".to_string(),
"latest_version".to_string(),
"empty".to_string(),
];
let searchable_attributes = vec![
"title".to_string(),
"description".to_string(),
"keywords".to_string(),
"versions".to_string(),
"author".to_string(),
"empty".to_string(),
];
Settings::new()
.with_displayed_attributes(displayed_attributes.clone())
.with_searchable_attributes(searchable_attributes.clone())
.with_accept_new_fields(true)
.with_stop_words(vec![])
.with_synonyms(HashMap::new())
}
//endregion

105
src/search/mod.rs Normal file
View File

@@ -0,0 +1,105 @@
use crate::database::DatabaseError;
use crate::models::mods::SearchRequest;
use meilisearch_sdk::client::Client;
use meilisearch_sdk::document::Document;
use meilisearch_sdk::search::Query;
use serde::{Deserialize, Serialize};
use thiserror::Error;
pub mod indexing;
#[derive(Error, Debug)]
pub enum SearchError {
#[error("Error while connection to the MeiliSearch database")]
IndexDBError(),
#[error("Error while connecting to the local server")]
LocalDatabaseError(#[from] mongodb::error::Error),
#[error("Error while accessing the data from remote")]
RemoteWebsiteError(#[from] reqwest::Error),
#[error("Error while serializing or deserializing JSON")]
SerDeError(#[from] serde_json::Error),
#[error("Error while parsing float")]
FloatParsingError(#[from] std::num::ParseFloatError),
#[error("Error while parsing float")]
IntParsingError(#[from] std::num::ParseIntError),
#[error("Error while parsing BSON")]
DatabaseError(#[from] DatabaseError),
#[error("Environment Error")]
EnvError(#[from] dotenv::Error),
}
#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct SearchMod {
pub mod_id: i32,
pub author: String,
pub title: String,
pub description: String,
pub keywords: Vec<String>,
pub versions: Vec<String>,
pub downloads: i32,
pub page_url: String,
pub icon_url: String,
pub author_url: String,
pub date_created: String,
pub created: i64,
pub date_modified: String,
pub updated: i64,
pub latest_version: String,
pub empty: String,
}
impl Document for SearchMod {
type UIDType = i32;
fn get_uid(&self) -> &Self::UIDType {
&self.mod_id
}
}
pub fn search_for_mod(info: &SearchRequest) -> Result<Vec<SearchMod>, SearchError> {
let address = &*dotenv::var("MEILISEARCH_ADDR")?;
let client = Client::new(address, "");
let search_query: &str;
let mut filters = String::new();
let mut offset = 0;
let mut index = "relevance";
match info.query.as_ref() {
Some(q) => search_query = q,
None => search_query = "{}{}{}",
}
if let Some(f) = info.filters.as_ref() {
filters = f.clone();
}
if let Some(v) = info.version.as_ref() {
if filters.is_empty() {
filters = v.clone();
} else {
filters = format!("({}) AND ({})", filters, v);
}
}
if let Some(o) = info.offset.as_ref() {
offset = o.parse().unwrap();
}
if let Some(s) = info.index.as_ref() {
index = s;
}
let mut query = Query::new(search_query).with_limit(10).with_offset(offset);
if !filters.is_empty() {
query = query.with_filters(&filters);
}
Ok(client
.get_index(format!("{}_mods", index).as_ref())
.unwrap()
.search::<SearchMod>(&query)
.unwrap()
.hits)
}