You've already forked AstralRinth
forked from didirus/AstralRinth
Rewrite the app (#23)
* chore: Removed everything not needed, and added base for rewrite feat(error_handling): Added 404 general cache feat(index): Added informations about the app in the / route. * feat(indexing): Brought back the indexing, with conditions to make it easier * fix: Fixed build error with a forgotten call * feat: Add Docker development enviroment (#19) * ci: add a *lot* of new actions * fix: rename linting action * fix: invalid yaml begone(?) * ci: Added cache to speed up build times * fix(ci): 🦀ed the yaml errors * fix(ci): fixed a missing hyphen * ci: Added matrix of rust versions, and changed way to install rust toolchain * fix(ci): Added names to build with the matrix so it's easier to find the source of the problem * style(ci): Added eof lines * refactor: Finished moving the search.rs file to a separate module. * Search Endpoint * refactor: Moved around functions and struct for a better understanding of what it does. * chore: Change env default settings to resolve conversation * refactor: Removed #[use_macros] fix: Fixed meilisearch address from env * chore: Added email to Aeledfyr * fix: Brought back the dotenv variables * style: Ran `cargo fmt` Co-authored-by: Charalampos Fanoulis <charalampos.fanoulis@gmail.com> Co-authored-by: Jai A <jai.a@tuta.io>
This commit is contained in:
@@ -1,12 +0,0 @@
|
||||
use mongodb::options::ClientOptions;
|
||||
use mongodb::Client;
|
||||
use mongodb::error::Error;
|
||||
|
||||
pub async fn connect() -> Result<Client, Error> {
|
||||
info!("Initializing database connection");
|
||||
|
||||
let mut client_options = ClientOptions::parse("mongodb://localhost:27017").await?;
|
||||
client_options.app_name = Some("Actix Web Server".to_string());
|
||||
|
||||
Client::with_options(client_options)
|
||||
}
|
||||
@@ -1,6 +1,19 @@
|
||||
mod database;
|
||||
mod models;
|
||||
pub mod models;
|
||||
mod mongo_database;
|
||||
|
||||
pub use database::connect;
|
||||
pub use models::Mod;
|
||||
pub use models::Version;
|
||||
pub use mongo_database::connect;
|
||||
use thiserror::Error;
|
||||
|
||||
type Result<T> = std::result::Result<T, DatabaseError>;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum DatabaseError {
|
||||
#[error("Impossible to find document")]
|
||||
NotFound(),
|
||||
#[error("BSON deserialization error")]
|
||||
BsonError(#[from] bson::de::Error),
|
||||
#[error("Local database error")]
|
||||
LocalDatabaseError(#[from] mongodb::error::Error),
|
||||
}
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
use serde::{Serialize, Deserialize};
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
pub struct Mod {
|
||||
pub id: i32,
|
||||
pub title: String,
|
||||
pub description: String,
|
||||
pub published: String,
|
||||
pub author: String,
|
||||
pub downloads: i32,
|
||||
pub categories: Vec<String>,
|
||||
pub body_path: String,
|
||||
pub icon_path: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
pub struct Version {
|
||||
pub version_id: i32,
|
||||
pub mod_id: i32,
|
||||
pub title: String,
|
||||
pub changelog_path: String,
|
||||
pub files_path: Vec<String>,
|
||||
pub date_published: String,
|
||||
pub author: String,
|
||||
pub downloads: i32,
|
||||
pub dependencies: Vec<String>,
|
||||
pub game_versions: Vec<String>,
|
||||
}
|
||||
27
src/database/models/mod.rs
Normal file
27
src/database/models/mod.rs
Normal file
@@ -0,0 +1,27 @@
|
||||
mod mod_item;
|
||||
mod version_item;
|
||||
|
||||
use crate::database::DatabaseError::NotFound;
|
||||
use crate::database::Result;
|
||||
use async_trait::async_trait;
|
||||
use bson::doc;
|
||||
use bson::Document;
|
||||
pub use mod_item::Mod;
|
||||
use mongodb::Database;
|
||||
pub use version_item::Version;
|
||||
|
||||
#[async_trait]
|
||||
pub trait Item {
|
||||
fn get_collection() -> &'static str;
|
||||
async fn get_by_id(client: Database, id: &str) -> Result<Box<Self>> {
|
||||
let filter = doc! { "_id": id };
|
||||
let collection = client.collection(Self::get_collection());
|
||||
let doc: Document = match collection.find_one(filter, None).await? {
|
||||
Some(e) => e,
|
||||
None => return Err(NotFound()),
|
||||
};
|
||||
let elem: Box<Self> = Self::from_doc(doc)?;
|
||||
Ok(elem)
|
||||
}
|
||||
fn from_doc(elem: Document) -> Result<Box<Self>>;
|
||||
}
|
||||
27
src/database/models/mod_item.rs
Normal file
27
src/database/models/mod_item.rs
Normal file
@@ -0,0 +1,27 @@
|
||||
use crate::database::models::Item;
|
||||
use crate::database::Result;
|
||||
use bson::{Bson, Document};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
pub struct Mod {
|
||||
pub id: i32,
|
||||
pub title: String,
|
||||
pub description: String,
|
||||
pub published: String,
|
||||
pub author: String,
|
||||
pub downloads: i32,
|
||||
pub categories: Vec<String>,
|
||||
pub body_path: String,
|
||||
pub icon_path: String,
|
||||
}
|
||||
impl Item for Mod {
|
||||
fn get_collection() -> &'static str {
|
||||
"mods"
|
||||
}
|
||||
|
||||
fn from_doc(elem: Document) -> Result<Box<Mod>> {
|
||||
let result: Mod = bson::from_bson(Bson::from(elem))?;
|
||||
Ok(Box::from(result))
|
||||
}
|
||||
}
|
||||
28
src/database/models/version_item.rs
Normal file
28
src/database/models/version_item.rs
Normal file
@@ -0,0 +1,28 @@
|
||||
use crate::database::models::Item;
|
||||
use crate::database::Result;
|
||||
use bson::{Bson, Document};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Deserialize, Serialize)]
|
||||
pub struct Version {
|
||||
pub version_id: i32,
|
||||
pub mod_id: i32,
|
||||
pub title: String,
|
||||
pub changelog_path: String,
|
||||
pub files_path: Vec<String>,
|
||||
pub date_published: String,
|
||||
pub author: String,
|
||||
pub downloads: i32,
|
||||
pub dependencies: Vec<String>,
|
||||
pub game_versions: Vec<String>,
|
||||
}
|
||||
impl Item for Version {
|
||||
fn get_collection() -> &'static str {
|
||||
"versions"
|
||||
}
|
||||
|
||||
fn from_doc(elem: Document) -> Result<Box<Version>> {
|
||||
let version: Version = bson::from_bson(Bson::from(elem))?;
|
||||
Ok(Box::from(version))
|
||||
}
|
||||
}
|
||||
13
src/database/mongo_database.rs
Normal file
13
src/database/mongo_database.rs
Normal file
@@ -0,0 +1,13 @@
|
||||
use log::info;
|
||||
use mongodb::error::Error;
|
||||
use mongodb::options::ClientOptions;
|
||||
use mongodb::Client;
|
||||
|
||||
pub async fn connect() -> Result<Client, Error> {
|
||||
info!("Initializing database connection");
|
||||
|
||||
let mut client_options = ClientOptions::parse(&dotenv::var("PORT").unwrap()).await?;
|
||||
client_options.app_name = Some("labrinth".to_string());
|
||||
|
||||
Client::with_options(client_options)
|
||||
}
|
||||
@@ -1,35 +0,0 @@
|
||||
use handlebars::*;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct ContainsHelper;
|
||||
|
||||
impl HelperDef for ContainsHelper {
|
||||
fn call<'reg: 'rc, 'rc>(
|
||||
&self,
|
||||
h: &Helper<'reg, 'rc>,
|
||||
r: &'reg Handlebars<'_>,
|
||||
ctx: &'rc Context,
|
||||
rc: &mut RenderContext<'reg, 'rc>,
|
||||
out: &mut dyn Output,
|
||||
) -> HelperResult {
|
||||
let array = h
|
||||
.param(0)
|
||||
.map(|v| serde_json::from_value::<Vec<String>>(v.value().clone()).unwrap())
|
||||
.ok_or_else(|| RenderError::new("Parameter not found!"))?;
|
||||
let value = h
|
||||
.param(1)
|
||||
.map(|v| v.value().as_str().unwrap())
|
||||
.ok_or_else(|| RenderError::new("Parameter not found!"))?;
|
||||
|
||||
let tmpl = if array.contains(&String::from(value)) {
|
||||
h.template()
|
||||
} else {
|
||||
h.inverse()
|
||||
};
|
||||
|
||||
match tmpl {
|
||||
Some(ref t) => t.render(r, ctx, rc, out),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,36 +0,0 @@
|
||||
use handlebars::*;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct EqualsHelper;
|
||||
|
||||
impl HelperDef for EqualsHelper {
|
||||
fn call<'reg: 'rc, 'rc>(
|
||||
&self,
|
||||
h: &Helper<'reg, 'rc>,
|
||||
r: &'reg Handlebars<'_>,
|
||||
ctx: &'rc Context,
|
||||
rc: &mut RenderContext<'reg, 'rc>,
|
||||
out: &mut dyn Output,
|
||||
) -> HelperResult {
|
||||
let a = h
|
||||
.param(0)
|
||||
.map(|v| v.value().as_object().unwrap())
|
||||
.ok_or_else(|| RenderError::new("Parameter not found!"))?;
|
||||
|
||||
let b = h
|
||||
.param(1)
|
||||
.map(|v| v.value().as_object().unwrap())
|
||||
.ok_or_else(|| RenderError::new("Parameter not found!"))?;
|
||||
|
||||
let tmpl = if a == b {
|
||||
h.template()
|
||||
} else {
|
||||
h.inverse()
|
||||
};
|
||||
|
||||
match tmpl {
|
||||
Some(ref t) => t.render(r, ctx, rc, out),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,31 +0,0 @@
|
||||
extern crate human_format;
|
||||
use handlebars::*;
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
pub struct HumanFormatHelper;
|
||||
|
||||
impl HelperDef for HumanFormatHelper {
|
||||
fn call<'reg: 'rc, 'rc>(
|
||||
&self,
|
||||
h: &Helper<'reg, 'rc>,
|
||||
_r: &'reg Handlebars<'_>,
|
||||
_ctx: &'rc Context,
|
||||
_rc: &mut RenderContext<'reg, 'rc>,
|
||||
out: &mut dyn Output,
|
||||
) -> HelperResult {
|
||||
let param = h.param(0).and_then(|v| v.value().as_f64()).unwrap_or(0.0);
|
||||
|
||||
let string = ¶m.to_string();
|
||||
|
||||
if string.len() > 3 {
|
||||
let mut formatted = human_format::Formatter::new().format(param);
|
||||
formatted.retain(|c| !c.is_whitespace());
|
||||
|
||||
out.write(formatted.to_uppercase().as_ref())?;
|
||||
} else {
|
||||
out.write(string)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
mod contains;
|
||||
mod format_human;
|
||||
mod equals;
|
||||
|
||||
use handlebars::*;
|
||||
|
||||
pub fn register_helpers(handlebars: &mut Handlebars) {
|
||||
handlebars.register_helper("contains", Box::new(contains::ContainsHelper));
|
||||
handlebars.register_helper("format", Box::new(format_human::HumanFormatHelper));
|
||||
|
||||
//This helper is not used yet, but could be useful in many circumstances
|
||||
handlebars.register_helper("equals", Box::new(equals::EqualsHelper));
|
||||
}
|
||||
58
src/main.rs
58
src/main.rs
@@ -1,40 +1,40 @@
|
||||
#[macro_use]
|
||||
extern crate serde_json;
|
||||
|
||||
#[macro_use]
|
||||
extern crate bson;
|
||||
|
||||
#[macro_use]
|
||||
extern crate log;
|
||||
|
||||
use actix_files as fs;
|
||||
use actix_web::{web, App, HttpServer};
|
||||
use handlebars::*;
|
||||
use crate::search::indexing::index_mods;
|
||||
use actix_web::middleware::Logger;
|
||||
use actix_web::{web, App, HttpServer};
|
||||
use env_logger::Env;
|
||||
use log::info;
|
||||
use std::env;
|
||||
use std::fs::File;
|
||||
|
||||
mod database;
|
||||
mod helpers;
|
||||
mod models;
|
||||
mod routes;
|
||||
mod search;
|
||||
|
||||
#[actix_rt::main]
|
||||
async fn main() -> std::io::Result<()> {
|
||||
env_logger::from_env(Env::default().default_filter_or("info")).init();
|
||||
dotenv::dotenv().ok();
|
||||
|
||||
//Handlebars
|
||||
let mut handlebars = Handlebars::new();
|
||||
|
||||
helpers::register_helpers(&mut handlebars);
|
||||
handlebars
|
||||
.register_templates_directory(".hbs", "./templates")
|
||||
.unwrap();
|
||||
|
||||
let handlebars_ref = web::Data::new(handlebars);
|
||||
|
||||
let client = database::connect().await.unwrap();
|
||||
routes::index_mods(client).await.unwrap();
|
||||
|
||||
// Get executable path
|
||||
let mut exe_path = env::current_exe()?.parent().unwrap().to_path_buf();
|
||||
// Create the path to the index lock file
|
||||
exe_path.push("index.v1.lock");
|
||||
|
||||
//Indexing mods if not already done
|
||||
if env::args().any(|x| x == "regen") {
|
||||
// User forced regen of indexing
|
||||
info!("Forced regeneration of indexes!");
|
||||
index_mods(client).await.unwrap();
|
||||
} else if exe_path.exists() {
|
||||
// The indexes were not created, or the version was upgraded
|
||||
info!("Indexing of mods for first time...");
|
||||
index_mods(client).await.unwrap();
|
||||
// Create the lock file
|
||||
File::create(exe_path)?;
|
||||
}
|
||||
|
||||
info!("Starting Actix HTTP server!");
|
||||
|
||||
@@ -43,15 +43,11 @@ async fn main() -> std::io::Result<()> {
|
||||
App::new()
|
||||
.wrap(Logger::default())
|
||||
.wrap(Logger::new("%a %{User-Agent}i"))
|
||||
.app_data(handlebars_ref.clone())
|
||||
.service(fs::Files::new("/static", "./static").show_files_listing())
|
||||
.service(routes::index_get)
|
||||
.service(routes::search_post)
|
||||
.service(routes::search_get)
|
||||
.service(routes::mod_page_get)
|
||||
.service(routes::mod_create_get)
|
||||
.service(routes::mod_search)
|
||||
.default_service(web::get().to(routes::not_found))
|
||||
})
|
||||
.bind("127.0.0.1:8000")?
|
||||
.bind("127.0.0.1:".to_string() + &dotenv::var("PORT").unwrap())?
|
||||
.run()
|
||||
.await
|
||||
}
|
||||
|
||||
6
src/models/error.rs
Normal file
6
src/models/error.rs
Normal file
@@ -0,0 +1,6 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct ApiError<'a> {
|
||||
pub error: &'a str,
|
||||
pub description: &'a str,
|
||||
}
|
||||
2
src/models/mod.rs
Normal file
2
src/models/mod.rs
Normal file
@@ -0,0 +1,2 @@
|
||||
pub mod error;
|
||||
pub mod mods;
|
||||
10
src/models/mods.rs
Normal file
10
src/models/mods.rs
Normal file
@@ -0,0 +1,10 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct SearchRequest {
|
||||
pub query: Option<String>,
|
||||
pub filters: Option<String>,
|
||||
pub version: Option<String>,
|
||||
pub offset: Option<String>,
|
||||
pub index: Option<String>,
|
||||
}
|
||||
@@ -1,12 +1,15 @@
|
||||
use actix_web::{get, web, HttpResponse};
|
||||
use handlebars::*;
|
||||
use serde_json::json;
|
||||
|
||||
#[get("/")]
|
||||
pub async fn index_get(hb: web::Data<Handlebars<'_>>) -> HttpResponse {
|
||||
pub async fn index_get() -> HttpResponse {
|
||||
let data = json!({
|
||||
"name": "Handlebars"
|
||||
"name": "modrinth-labrinth",
|
||||
"version": env!("CARGO_PKG_VERSION"),
|
||||
//TODO: Add the documentation link
|
||||
"documentation": "Nowhere yet",
|
||||
"about": "Welcome traveler !"
|
||||
});
|
||||
let body = hb.render("index", &data).unwrap();
|
||||
|
||||
HttpResponse::Ok().body(body)
|
||||
HttpResponse::Ok().json(data)
|
||||
}
|
||||
|
||||
@@ -1,14 +1,7 @@
|
||||
mod index;
|
||||
mod mod_page;
|
||||
mod search;
|
||||
mod mod_create;
|
||||
|
||||
pub use self::mod_page::mod_page_get;
|
||||
|
||||
pub use self::mod_create::mod_create_get;
|
||||
|
||||
pub use self::search::index_mods;
|
||||
pub use self::search::search_get;
|
||||
pub use self::search::search_post;
|
||||
mod mods;
|
||||
mod not_found;
|
||||
|
||||
pub use self::index::index_get;
|
||||
pub use self::mods::mod_search;
|
||||
pub use self::not_found::not_found;
|
||||
|
||||
@@ -1,30 +0,0 @@
|
||||
use actix_web::{get, post, web, HttpResponse};
|
||||
use handlebars::*;
|
||||
use serde::Deserialize;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct CreatedMod {
|
||||
name: String,
|
||||
description: String,
|
||||
body: String,
|
||||
}
|
||||
|
||||
#[get("createmod")]
|
||||
pub async fn mod_create_get(hb: web::Data<Handlebars<'_>>) -> HttpResponse {
|
||||
let data = json!({
|
||||
"name": "Handlebars"
|
||||
});
|
||||
let body = hb.render("mod-create", &data).unwrap();
|
||||
|
||||
HttpResponse::Ok().body(body)
|
||||
}
|
||||
|
||||
#[post("createmod")]
|
||||
pub async fn mod_create_post(hb: web::Data<Handlebars<'_>>) -> HttpResponse {
|
||||
let data = json!({
|
||||
"name": "Handlebars"
|
||||
});
|
||||
let body = hb.render("mod-create", &data).unwrap();
|
||||
|
||||
HttpResponse::Ok().body(body)
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
use actix_web::{get, web, HttpResponse};
|
||||
use handlebars::*;
|
||||
|
||||
#[get("mod/testmod")]
|
||||
pub async fn mod_page_get(hb: web::Data<Handlebars<'_>>) -> HttpResponse {
|
||||
let data = json!({
|
||||
"name": "Handlebars"
|
||||
});
|
||||
let body = hb.render("mod-page", &data).unwrap();
|
||||
|
||||
HttpResponse::Ok().body(body)
|
||||
}
|
||||
13
src/routes/mods.rs
Normal file
13
src/routes/mods.rs
Normal file
@@ -0,0 +1,13 @@
|
||||
use crate::models::mods::SearchRequest;
|
||||
use crate::search::search_for_mod;
|
||||
use actix_web::{get, web, HttpResponse};
|
||||
|
||||
#[get("api/v1/mods")]
|
||||
pub fn mod_search(web::Query(info): web::Query<SearchRequest>) -> HttpResponse {
|
||||
//TODO: Fix this line with anyhow
|
||||
let body = serde_json::to_string(&search_for_mod(&info).unwrap()).unwrap();
|
||||
|
||||
HttpResponse::Ok()
|
||||
.content_type("application/json")
|
||||
.body(body)
|
||||
}
|
||||
11
src/routes/not_found.rs
Normal file
11
src/routes/not_found.rs
Normal file
@@ -0,0 +1,11 @@
|
||||
use crate::models::error::ApiError;
|
||||
use actix_web::{HttpResponse, Responder};
|
||||
|
||||
pub async fn not_found() -> impl Responder {
|
||||
let data = ApiError {
|
||||
error: "not_found",
|
||||
description: "the route you called is not (yet) implemented",
|
||||
};
|
||||
|
||||
HttpResponse::NotFound().json(data)
|
||||
}
|
||||
@@ -1,464 +0,0 @@
|
||||
use actix_web::{get, post, web, web::Data, HttpResponse};
|
||||
use handlebars::*;
|
||||
use meilisearch_sdk::{client::*, document::*, search::*};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use meilisearch_sdk::settings::Settings;
|
||||
use futures::stream::StreamExt;
|
||||
use bson::Bson;
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::error::Error;
|
||||
|
||||
use crate::database::*;
|
||||
use futures_timer::Delay;
|
||||
use futures::TryFutureExt;
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct Attachment {
|
||||
url: String,
|
||||
thumbnail_url: String,
|
||||
is_default: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
struct Category {
|
||||
name: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
struct Author {
|
||||
name: String,
|
||||
url: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct CurseVersion {
|
||||
game_version: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct CurseForgeMod {
|
||||
id: i32,
|
||||
name: String,
|
||||
authors: Vec<Author>,
|
||||
attachments: Vec<Attachment>,
|
||||
website_url: String,
|
||||
summary: String,
|
||||
download_count: f32,
|
||||
categories: Vec<Category>,
|
||||
game_version_latest_files: Vec<CurseVersion>,
|
||||
date_created: String,
|
||||
date_modified: String,
|
||||
game_slug: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
struct SearchMod {
|
||||
mod_id: i32,
|
||||
author: String,
|
||||
title: String,
|
||||
description: String,
|
||||
keywords: Vec<String>,
|
||||
versions: Vec<String>,
|
||||
downloads: i32,
|
||||
page_url: String,
|
||||
icon_url: String,
|
||||
author_url: String,
|
||||
date_created: String,
|
||||
created: i64,
|
||||
date_modified: String,
|
||||
updated: i64,
|
||||
latest_version: String,
|
||||
empty: String,
|
||||
}
|
||||
|
||||
impl Document for SearchMod {
|
||||
type UIDType = i32;
|
||||
|
||||
fn get_uid(&self) -> &Self::UIDType {
|
||||
&self.mod_id
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
pub struct SearchRequest {
|
||||
#[serde(rename = "q")]
|
||||
query: Option<String>,
|
||||
#[serde(rename = "f")]
|
||||
filters: Option<String>,
|
||||
#[serde(rename = "v")]
|
||||
version: Option<String>,
|
||||
#[serde(rename = "o")]
|
||||
offset: Option<String>,
|
||||
#[serde(rename = "s")]
|
||||
index: Option<String>,
|
||||
}
|
||||
|
||||
#[post("search")]
|
||||
pub async fn search_post(
|
||||
web::Query(info): web::Query<SearchRequest>,
|
||||
hb: Data<Handlebars<'_>>,
|
||||
) -> HttpResponse {
|
||||
let results = search(&info);
|
||||
let data = json!({
|
||||
"query": info,
|
||||
"results": results,
|
||||
});
|
||||
let body = hb.render("search-results", &data).unwrap();
|
||||
|
||||
HttpResponse::Ok().body(body)
|
||||
}
|
||||
|
||||
#[get("search")]
|
||||
pub async fn search_get(
|
||||
web::Query(info): web::Query<SearchRequest>,
|
||||
hb: Data<Handlebars<'_>>,
|
||||
) -> HttpResponse {
|
||||
let results = search(&info);
|
||||
|
||||
let data = json!({
|
||||
"query": info,
|
||||
"results": results,
|
||||
});
|
||||
|
||||
let body = hb.render("search", &data).unwrap();
|
||||
|
||||
HttpResponse::Ok().body(body)
|
||||
}
|
||||
|
||||
fn search(info: &SearchRequest) -> Vec<SearchMod> {
|
||||
let client = Client::new("http://localhost:7700", "");
|
||||
|
||||
let search_query: &str;
|
||||
let mut filters = String::new();
|
||||
let mut offset = 0;
|
||||
let mut index = "relevance";
|
||||
|
||||
match info.query.as_ref() {
|
||||
Some(q) => search_query = q,
|
||||
None => search_query = "{}{}{}",
|
||||
}
|
||||
|
||||
if let Some(f) = info.filters.as_ref() {
|
||||
filters = f.clone();
|
||||
}
|
||||
|
||||
if let Some(v) = info.version.as_ref() {
|
||||
if filters.is_empty() {
|
||||
filters = v.clone();
|
||||
} else {
|
||||
filters = format!("({}) AND ({})", filters, v);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(o) = info.offset.as_ref() {
|
||||
offset = o.parse().unwrap();
|
||||
}
|
||||
|
||||
if let Some(s) = info.index.as_ref() {
|
||||
index = s;
|
||||
}
|
||||
|
||||
let mut query = Query::new(search_query).with_limit(10).with_offset(offset);
|
||||
|
||||
if !filters.is_empty() {
|
||||
query = query.with_filters(&filters);
|
||||
}
|
||||
|
||||
client.get_index(format!("{}_mods", index).as_ref()).unwrap()
|
||||
.search::<SearchMod>(&query).unwrap().hits
|
||||
}
|
||||
|
||||
/*
|
||||
TODO This method needs a lot of refactoring. Here's a list of changes that need to be made:
|
||||
- Move Curseforge Indexing to another method/module
|
||||
- Get rid of the 4 indexes (when MeiliSearch updates) and replace it with different rules
|
||||
- Remove code fragment duplicates
|
||||
*/
|
||||
|
||||
pub async fn index_mods(db: mongodb::Client) -> Result<(), Box<dyn Error>>{
|
||||
let mut docs_to_add: Vec<SearchMod> = vec![];
|
||||
|
||||
docs_to_add.append(&mut index_database(db.clone()).await?);
|
||||
//docs_to_add.append(&mut index_curseforge(1, 400000).await?);
|
||||
|
||||
//Write Indexes
|
||||
//Relevance Index
|
||||
let client = Client::new("http://localhost:7700", "");
|
||||
|
||||
let mut relevance_index = client.get_or_create("relevance_mods").unwrap();
|
||||
|
||||
let mut relevance_rules = default_rules();
|
||||
relevance_rules.push_back("desc(downloads)".to_string());
|
||||
|
||||
relevance_index.set_settings(&default_settings().with_ranking_rules(relevance_rules.into())).unwrap();
|
||||
relevance_index.add_documents(docs_to_add.clone(), Some("mod_id")).unwrap();
|
||||
|
||||
//Downloads Index
|
||||
let mut downloads_index = client.get_or_create("downloads_mods").unwrap();
|
||||
|
||||
let mut downloads_rules = default_rules();
|
||||
downloads_rules.push_front("desc(downloads)".to_string());
|
||||
|
||||
downloads_index.set_settings(&default_settings().with_ranking_rules(downloads_rules.into())).unwrap();
|
||||
downloads_index.add_documents(docs_to_add.clone(), Some("mod_id")).unwrap();
|
||||
|
||||
//Updated Index
|
||||
let mut updated_index = client.get_or_create("updated_mods").unwrap();
|
||||
|
||||
let mut updated_rules = default_rules();
|
||||
updated_rules.push_front("desc(updated)".to_string());
|
||||
|
||||
updated_index.set_settings(&default_settings().with_ranking_rules(updated_rules.into())).unwrap();
|
||||
updated_index.add_documents(docs_to_add.clone(), Some("mod_id")).unwrap();
|
||||
|
||||
//Created Index
|
||||
let mut newest_index = client.get_or_create("newest_mods").unwrap();
|
||||
|
||||
let mut newest_rules = default_rules();
|
||||
newest_rules.push_back("desc(created)".to_string());
|
||||
|
||||
newest_index.set_settings(&default_settings().with_ranking_rules(newest_rules.into())).unwrap();
|
||||
newest_index.add_documents(docs_to_add.clone(), Some("mod_id")).unwrap();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn index_database(client: mongodb::Client) -> Result<Vec<SearchMod>, Box<dyn Error>> {
|
||||
info!("Indexing database mods!");
|
||||
|
||||
let mut docs_to_add: Vec<SearchMod> = vec![];
|
||||
|
||||
let db = client.database("fabricate");
|
||||
|
||||
let mods = db.collection("mods");
|
||||
let versions = db.collection("versions");
|
||||
|
||||
let mut results = mods.find(None, None).await?;
|
||||
|
||||
while let Some(unparsed_result) = results.next().await {
|
||||
let result : Mod = bson::from_bson(Bson::from(unparsed_result?))?;
|
||||
|
||||
let mut mod_versions = versions.find(doc!{ "mod_id": result.id}, None).await?;
|
||||
|
||||
let mut mod_game_versions = vec![];
|
||||
|
||||
while let Some(unparsed_version) = mod_versions.next().await {
|
||||
let mut version : Version = bson::from_bson(Bson::from(unparsed_version?))?;
|
||||
mod_game_versions.append(&mut version.game_versions);
|
||||
}
|
||||
|
||||
docs_to_add.push(SearchMod {
|
||||
mod_id: result.id,
|
||||
author: result.author,
|
||||
title: result.title,
|
||||
description: result.description,
|
||||
keywords: result.categories,
|
||||
versions: mod_game_versions,
|
||||
downloads: result.downloads,
|
||||
page_url: "".to_string(),
|
||||
icon_url: result.icon_path,
|
||||
author_url: "".to_string(),
|
||||
date_created: "".to_string(),
|
||||
created: 0,
|
||||
date_modified: "".to_string(),
|
||||
updated: 0,
|
||||
latest_version: "".to_string(),
|
||||
empty: String::from("{}{}{}"),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(docs_to_add)
|
||||
}
|
||||
|
||||
async fn index_curseforge(start_index: i32, end_index: i32) -> Result<Vec<SearchMod>, Box<dyn Error>>{
|
||||
info!("Indexing curseforge mods!");
|
||||
|
||||
let mut docs_to_add: Vec<SearchMod> = vec![];
|
||||
|
||||
let res = reqwest::Client::new().post("https://addons-ecs.forgesvc.net/api/v2/addon")
|
||||
.header(reqwest::header::CONTENT_TYPE, "application/json")
|
||||
.body(format!("{:?}", (start_index..end_index).collect::<Vec<_>>()))
|
||||
.send().await?;
|
||||
|
||||
let text = &res.text().await?;
|
||||
let curseforge_mods : Vec<CurseForgeMod> = serde_json::from_str(text)?;
|
||||
|
||||
let mut max_index = 0;
|
||||
|
||||
for curseforge_mod in curseforge_mods {
|
||||
max_index = curseforge_mod.id;
|
||||
if curseforge_mod.game_slug != "minecraft" || !curseforge_mod.website_url.contains("/mc-mods/") { continue; }
|
||||
|
||||
let mut mod_game_versions = vec![];
|
||||
|
||||
let mut using_forge = false;
|
||||
let mut using_fabric = false;
|
||||
|
||||
for version in curseforge_mod.game_version_latest_files {
|
||||
let version_number: String = version
|
||||
.game_version
|
||||
.chars()
|
||||
.skip(2)
|
||||
.take(version.game_version.len())
|
||||
.collect();
|
||||
|
||||
if version_number.parse::<f32>()? < 14.0 {
|
||||
using_forge = true;
|
||||
}
|
||||
|
||||
mod_game_versions.push(version.game_version);
|
||||
}
|
||||
|
||||
let mut mod_categories = vec![];
|
||||
|
||||
for category in curseforge_mod.categories {
|
||||
match &category.name[..] {
|
||||
"World Gen" => mod_categories.push(String::from("worldgen")),
|
||||
"Biomes" => mod_categories.push(String::from("worldgen")),
|
||||
"Ores and Resources" => mod_categories.push(String::from("worldgen")),
|
||||
"Structures" => mod_categories.push(String::from("worldgen")),
|
||||
"Dimensions" => mod_categories.push(String::from("worldgen")),
|
||||
"Mobs" => mod_categories.push(String::from("worldgen")),
|
||||
"Technology" => mod_categories.push(String::from("technology")),
|
||||
"Processing" => mod_categories.push(String::from("technology")),
|
||||
"Player Transport" => mod_categories.push(String::from("technology")),
|
||||
"Energy, Fluid, and Item Transport" => { mod_categories.push(String::from("technology")) }
|
||||
"Food" => mod_categories.push(String::from("food")),
|
||||
"Farming" => mod_categories.push(String::from("food")),
|
||||
"Energy" => mod_categories.push(String::from("technology")),
|
||||
"Redstone" => mod_categories.push(String::from("technology")),
|
||||
"Genetics" => mod_categories.push(String::from("technology")),
|
||||
"Magic" => mod_categories.push(String::from("magic")),
|
||||
"Storage" => mod_categories.push(String::from("storage")),
|
||||
"API and Library" => mod_categories.push(String::from("library")),
|
||||
"Adventure and RPG" => mod_categories.push(String::from("adventure")),
|
||||
"Map and Information" => mod_categories.push(String::from("utility")),
|
||||
"Cosmetic" => mod_categories.push(String::from("decoration")),
|
||||
"Addons" => mod_categories.push(String::from("misc")),
|
||||
"Thermal Expansion" => mod_categories.push(String::from("misc")),
|
||||
"Tinker's Construct" => mod_categories.push(String::from("misc")),
|
||||
"Industrial Craft" => mod_categories.push(String::from("misc")),
|
||||
"Thaumcraft" => mod_categories.push(String::from("misc")),
|
||||
"Buildcraft" => mod_categories.push(String::from("misc")),
|
||||
"Forestry" => mod_categories.push(String::from("misc")),
|
||||
"Blood Magic" => mod_categories.push(String::from("misc")),
|
||||
"Lucky Blocks" => mod_categories.push(String::from("misc")),
|
||||
"Applied Energistics 2" => mod_categories.push(String::from("misc")),
|
||||
"CraftTweaker" => mod_categories.push(String::from("misc")),
|
||||
"Miscellaneous" => mod_categories.push(String::from("misc")),
|
||||
"Armor, Tools, and Weapons" => mod_categories.push(String::from("equipment")),
|
||||
"Server Utility" => mod_categories.push(String::from("utility")),
|
||||
"Fabric" => mod_categories.push(String::from("fabric")),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if mod_categories.contains(&"fabric".to_owned()) {
|
||||
using_fabric = true;
|
||||
}
|
||||
|
||||
mod_categories.sort();
|
||||
mod_categories.dedup();
|
||||
mod_categories.truncate(3);
|
||||
|
||||
if using_forge {
|
||||
mod_categories.push(String::from("forge"));
|
||||
}
|
||||
if using_fabric {
|
||||
mod_categories.push(String::from("fabric"));
|
||||
}
|
||||
|
||||
let mut mod_attachments = curseforge_mod.attachments;
|
||||
mod_attachments.retain(|x| x.is_default);
|
||||
|
||||
if mod_attachments.is_empty() {
|
||||
mod_attachments.push(Attachment {
|
||||
url: String::new(),
|
||||
thumbnail_url: String::new(),
|
||||
is_default: true,
|
||||
})
|
||||
}
|
||||
|
||||
let latest_version = if !mod_game_versions.is_empty() {
|
||||
mod_game_versions[0].to_string()
|
||||
} else {
|
||||
"None".to_string()
|
||||
};
|
||||
|
||||
let icon_url = mod_attachments[0].thumbnail_url.replace("/256/256/", "/64/64/");
|
||||
|
||||
docs_to_add.push(SearchMod {
|
||||
mod_id: -curseforge_mod.id,
|
||||
author: (&curseforge_mod.authors[0].name).to_string(),
|
||||
title: curseforge_mod.name,
|
||||
description: curseforge_mod.summary.chars().take(150).collect(),
|
||||
keywords: mod_categories,
|
||||
versions: mod_game_versions.clone(),
|
||||
downloads: curseforge_mod.download_count as i32,
|
||||
page_url: curseforge_mod.website_url,
|
||||
icon_url,
|
||||
author_url: (&curseforge_mod.authors[0].url).to_string(),
|
||||
date_created: curseforge_mod.date_created.chars().take(10).collect(),
|
||||
created: curseforge_mod.date_created.chars().filter(|c| c.is_ascii_digit()).collect::<String>().parse()?,
|
||||
date_modified: curseforge_mod.date_modified.chars().take(10).collect(),
|
||||
updated: curseforge_mod.date_modified.chars().filter(|c| c.is_ascii_digit()).collect::<String>().parse()?,
|
||||
latest_version,
|
||||
empty: String::from("{}{}{}"),
|
||||
})
|
||||
}
|
||||
|
||||
//TODO Reindex every hour for new mods.
|
||||
Ok(docs_to_add)
|
||||
}
|
||||
|
||||
fn default_rules() -> VecDeque<String> {
|
||||
vec![
|
||||
"typo".to_string(),
|
||||
"words".to_string(),
|
||||
"proximity".to_string(),
|
||||
"attribute".to_string(),
|
||||
"wordsPosition".to_string(),
|
||||
"exactness".to_string(),
|
||||
].into()
|
||||
}
|
||||
|
||||
fn default_settings() -> Settings {
|
||||
let displayed_attributes = vec![
|
||||
"mod_id".to_string(),
|
||||
"author".to_string(),
|
||||
"title".to_string(),
|
||||
"description".to_string(),
|
||||
"keywords".to_string(),
|
||||
"versions".to_string(),
|
||||
"downloads".to_string(),
|
||||
"page_url".to_string(),
|
||||
"icon_url".to_string(),
|
||||
"author_url".to_string(),
|
||||
"date_created".to_string(),
|
||||
"created".to_string(),
|
||||
"date_modified".to_string(),
|
||||
"updated".to_string(),
|
||||
"latest_version".to_string(),
|
||||
"empty".to_string(),
|
||||
];
|
||||
|
||||
let searchable_attributes = vec![
|
||||
"title".to_string(),
|
||||
"description".to_string(),
|
||||
"keywords".to_string(),
|
||||
"versions".to_string(),
|
||||
"author".to_string(),
|
||||
"empty".to_string(),
|
||||
];
|
||||
|
||||
Settings::new()
|
||||
.with_displayed_attributes(displayed_attributes.clone())
|
||||
.with_searchable_attributes(searchable_attributes.clone())
|
||||
.with_accept_new_fields(true)
|
||||
.with_stop_words(vec![])
|
||||
.with_synonyms(HashMap::new())
|
||||
}
|
||||
212
src/search/indexing/curseforge_import.rs
Normal file
212
src/search/indexing/curseforge_import.rs
Normal file
@@ -0,0 +1,212 @@
|
||||
use crate::search::{SearchError, SearchMod};
|
||||
use log::info;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Attachment {
|
||||
pub url: String,
|
||||
pub thumbnail_url: String,
|
||||
pub is_default: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
pub struct Category {
|
||||
pub name: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
pub struct Author {
|
||||
pub name: String,
|
||||
pub url: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct CurseVersion {
|
||||
pub game_version: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct CurseForgeMod {
|
||||
pub id: i32,
|
||||
pub name: String,
|
||||
pub authors: Vec<Author>,
|
||||
pub attachments: Vec<Attachment>,
|
||||
pub website_url: String,
|
||||
pub summary: String,
|
||||
pub download_count: f32,
|
||||
pub categories: Vec<Category>,
|
||||
pub game_version_latest_files: Vec<CurseVersion>,
|
||||
pub date_created: String,
|
||||
pub date_modified: String,
|
||||
pub game_slug: String,
|
||||
}
|
||||
|
||||
pub async fn index_curseforge(
|
||||
start_index: i32,
|
||||
end_index: i32,
|
||||
) -> Result<Vec<SearchMod>, SearchError> {
|
||||
info!("Indexing curseforge mods!");
|
||||
|
||||
let mut docs_to_add: Vec<SearchMod> = vec![];
|
||||
|
||||
let res = reqwest::Client::new()
|
||||
.post("https://addons-ecs.forgesvc.net/api/v2/addon")
|
||||
.header(reqwest::header::CONTENT_TYPE, "application/json")
|
||||
.body(format!(
|
||||
"{:?}",
|
||||
(start_index..end_index).collect::<Vec<_>>()
|
||||
))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let text = &res.text().await?;
|
||||
let curseforge_mods: Vec<CurseForgeMod> = serde_json::from_str(text)?;
|
||||
|
||||
let mut max_index = 0;
|
||||
|
||||
for curseforge_mod in curseforge_mods {
|
||||
max_index = curseforge_mod.id;
|
||||
if curseforge_mod.game_slug != "minecraft"
|
||||
|| !curseforge_mod.website_url.contains("/mc-mods/")
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut mod_game_versions = vec![];
|
||||
|
||||
let mut using_forge = false;
|
||||
let mut using_fabric = false;
|
||||
|
||||
for version in curseforge_mod.game_version_latest_files {
|
||||
let version_number: String = version
|
||||
.game_version
|
||||
.chars()
|
||||
.skip(2)
|
||||
.take(version.game_version.len())
|
||||
.collect();
|
||||
|
||||
if version_number.parse::<f32>()? < 14.0 {
|
||||
using_forge = true;
|
||||
}
|
||||
|
||||
mod_game_versions.push(version.game_version);
|
||||
}
|
||||
|
||||
let mut mod_categories = vec![];
|
||||
|
||||
for category in curseforge_mod.categories {
|
||||
match &category.name[..] {
|
||||
"World Gen" => mod_categories.push(String::from("worldgen")),
|
||||
"Biomes" => mod_categories.push(String::from("worldgen")),
|
||||
"Ores and Resources" => mod_categories.push(String::from("worldgen")),
|
||||
"Structures" => mod_categories.push(String::from("worldgen")),
|
||||
"Dimensions" => mod_categories.push(String::from("worldgen")),
|
||||
"Mobs" => mod_categories.push(String::from("worldgen")),
|
||||
"Technology" => mod_categories.push(String::from("technology")),
|
||||
"Processing" => mod_categories.push(String::from("technology")),
|
||||
"Player Transport" => mod_categories.push(String::from("technology")),
|
||||
"Energy, Fluid, and Item Transport" => {
|
||||
mod_categories.push(String::from("technology"))
|
||||
}
|
||||
"Food" => mod_categories.push(String::from("food")),
|
||||
"Farming" => mod_categories.push(String::from("food")),
|
||||
"Energy" => mod_categories.push(String::from("technology")),
|
||||
"Redstone" => mod_categories.push(String::from("technology")),
|
||||
"Genetics" => mod_categories.push(String::from("technology")),
|
||||
"Magic" => mod_categories.push(String::from("magic")),
|
||||
"Storage" => mod_categories.push(String::from("storage")),
|
||||
"API and Library" => mod_categories.push(String::from("library")),
|
||||
"Adventure and RPG" => mod_categories.push(String::from("adventure")),
|
||||
"Map and Information" => mod_categories.push(String::from("utility")),
|
||||
"Cosmetic" => mod_categories.push(String::from("decoration")),
|
||||
"Addons" => mod_categories.push(String::from("misc")),
|
||||
"Thermal Expansion" => mod_categories.push(String::from("misc")),
|
||||
"Tinker's Construct" => mod_categories.push(String::from("misc")),
|
||||
"Industrial Craft" => mod_categories.push(String::from("misc")),
|
||||
"Thaumcraft" => mod_categories.push(String::from("misc")),
|
||||
"Buildcraft" => mod_categories.push(String::from("misc")),
|
||||
"Forestry" => mod_categories.push(String::from("misc")),
|
||||
"Blood Magic" => mod_categories.push(String::from("misc")),
|
||||
"Lucky Blocks" => mod_categories.push(String::from("misc")),
|
||||
"Applied Energistics 2" => mod_categories.push(String::from("misc")),
|
||||
"CraftTweaker" => mod_categories.push(String::from("misc")),
|
||||
"Miscellaneous" => mod_categories.push(String::from("misc")),
|
||||
"Armor, Tools, and Weapons" => mod_categories.push(String::from("equipment")),
|
||||
"Server Utility" => mod_categories.push(String::from("utility")),
|
||||
"Fabric" => mod_categories.push(String::from("fabric")),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
if mod_categories.contains(&"fabric".to_owned()) {
|
||||
using_fabric = true;
|
||||
}
|
||||
|
||||
mod_categories.sort();
|
||||
mod_categories.dedup();
|
||||
mod_categories.truncate(3);
|
||||
|
||||
if using_forge {
|
||||
mod_categories.push(String::from("forge"));
|
||||
}
|
||||
if using_fabric {
|
||||
mod_categories.push(String::from("fabric"));
|
||||
}
|
||||
|
||||
let mut mod_attachments = curseforge_mod.attachments;
|
||||
mod_attachments.retain(|x| x.is_default);
|
||||
|
||||
if mod_attachments.is_empty() {
|
||||
mod_attachments.push(Attachment {
|
||||
url: String::new(),
|
||||
thumbnail_url: String::new(),
|
||||
is_default: true,
|
||||
})
|
||||
}
|
||||
|
||||
let latest_version = if !mod_game_versions.is_empty() {
|
||||
mod_game_versions[0].to_string()
|
||||
} else {
|
||||
"None".to_string()
|
||||
};
|
||||
|
||||
let icon_url = mod_attachments[0]
|
||||
.thumbnail_url
|
||||
.replace("/256/256/", "/64/64/");
|
||||
|
||||
docs_to_add.push(SearchMod {
|
||||
mod_id: -curseforge_mod.id,
|
||||
author: (&curseforge_mod.authors[0].name).to_string(),
|
||||
title: curseforge_mod.name,
|
||||
description: curseforge_mod.summary.chars().take(150).collect(),
|
||||
keywords: mod_categories,
|
||||
versions: mod_game_versions.clone(),
|
||||
downloads: curseforge_mod.download_count as i32,
|
||||
page_url: curseforge_mod.website_url,
|
||||
icon_url,
|
||||
author_url: (&curseforge_mod.authors[0].url).to_string(),
|
||||
date_created: curseforge_mod.date_created.chars().take(10).collect(),
|
||||
created: curseforge_mod
|
||||
.date_created
|
||||
.chars()
|
||||
.filter(|c| c.is_ascii_digit())
|
||||
.collect::<String>()
|
||||
.parse()?,
|
||||
date_modified: curseforge_mod.date_modified.chars().take(10).collect(),
|
||||
updated: curseforge_mod
|
||||
.date_modified
|
||||
.chars()
|
||||
.filter(|c| c.is_ascii_digit())
|
||||
.collect::<String>()
|
||||
.parse()?,
|
||||
latest_version,
|
||||
empty: String::from("{}{}{}"),
|
||||
})
|
||||
}
|
||||
|
||||
//TODO Reindex every hour for new mods.
|
||||
Ok(docs_to_add)
|
||||
}
|
||||
57
src/search/indexing/local_import.rs
Normal file
57
src/search/indexing/local_import.rs
Normal file
@@ -0,0 +1,57 @@
|
||||
use bson::doc;
|
||||
use bson::Bson;
|
||||
use futures::StreamExt;
|
||||
use log::info;
|
||||
use meilisearch_sdk::client::Client;
|
||||
|
||||
use crate::database::models::Item;
|
||||
use crate::database::{Mod, Version};
|
||||
|
||||
use crate::search::{SearchError, SearchMod, SearchRequest};
|
||||
|
||||
pub async fn index_local(client: mongodb::Client) -> Result<Vec<SearchMod>, SearchError> {
|
||||
info!("Indexing local mods!");
|
||||
|
||||
let mut docs_to_add: Vec<SearchMod> = vec![];
|
||||
|
||||
let db = client.database("modrinth");
|
||||
|
||||
let mods = db.collection("mods");
|
||||
let versions = db.collection("versions");
|
||||
|
||||
let mut results = mods.find(None, None).await?;
|
||||
|
||||
while let Some(unparsed_result) = results.next().await {
|
||||
let result: Mod = *Mod::from_doc(unparsed_result?)?;
|
||||
|
||||
let mut mod_versions = versions.find(doc! { "mod_id": result.id}, None).await?;
|
||||
|
||||
let mut mod_game_versions = vec![];
|
||||
|
||||
while let Some(unparsed_version) = mod_versions.next().await {
|
||||
let mut version: Version = *Version::from_doc(unparsed_version?)?;
|
||||
mod_game_versions.append(&mut version.game_versions);
|
||||
}
|
||||
|
||||
docs_to_add.push(SearchMod {
|
||||
mod_id: result.id,
|
||||
author: result.author,
|
||||
title: result.title,
|
||||
description: result.description,
|
||||
keywords: result.categories,
|
||||
versions: mod_game_versions,
|
||||
downloads: result.downloads,
|
||||
page_url: "".to_string(),
|
||||
icon_url: result.icon_path,
|
||||
author_url: "".to_string(),
|
||||
date_created: "".to_string(),
|
||||
created: 0,
|
||||
date_modified: "".to_string(),
|
||||
updated: 0,
|
||||
latest_version: "".to_string(),
|
||||
empty: String::from("{}{}{}"),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(docs_to_add)
|
||||
}
|
||||
135
src/search/indexing/mod.rs
Normal file
135
src/search/indexing/mod.rs
Normal file
@@ -0,0 +1,135 @@
|
||||
/// This module is used for the indexing from any source.
|
||||
pub mod curseforge_import;
|
||||
pub mod local_import;
|
||||
|
||||
use crate::database::DatabaseError;
|
||||
use crate::search::indexing::curseforge_import::index_curseforge;
|
||||
use crate::search::indexing::local_import::index_local;
|
||||
use crate::search::{SearchError, SearchMod};
|
||||
use meilisearch_sdk::client::Client;
|
||||
use meilisearch_sdk::settings::Settings;
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
|
||||
pub async fn index_mods(db: mongodb::Client) -> Result<(), SearchError> {
|
||||
// Check if the index exists
|
||||
let address = &*dotenv::var("MEILISEARCH_ADDR")?;
|
||||
let client = Client::new(address, "");
|
||||
|
||||
let mut docs_to_add: Vec<SearchMod> = vec![];
|
||||
|
||||
docs_to_add.append(&mut index_local(db.clone()).await?);
|
||||
if dotenv::var("INDEX_CURSEFORGE")
|
||||
.expect("`INDEX_CURSEFORGE` is missing in the .env file.")
|
||||
.parse()
|
||||
.unwrap()
|
||||
{
|
||||
docs_to_add.append(&mut index_curseforge(1, 400000).await?);
|
||||
}
|
||||
//Write Indexes
|
||||
//Relevance Index
|
||||
|
||||
let mut relevance_index = client.get_or_create("relevance_mods").unwrap();
|
||||
|
||||
let mut relevance_rules = default_rules();
|
||||
relevance_rules.push_back("desc(downloads)".to_string());
|
||||
|
||||
relevance_index
|
||||
.set_settings(&default_settings().with_ranking_rules(relevance_rules.into()))
|
||||
.unwrap();
|
||||
relevance_index
|
||||
.add_documents(docs_to_add.clone(), Some("mod_id"))
|
||||
.unwrap();
|
||||
|
||||
//Downloads Index
|
||||
let mut downloads_index = client.get_or_create("downloads_mods").unwrap();
|
||||
|
||||
let mut downloads_rules = default_rules();
|
||||
downloads_rules.push_front("desc(downloads)".to_string());
|
||||
|
||||
downloads_index
|
||||
.set_settings(&default_settings().with_ranking_rules(downloads_rules.into()))
|
||||
.unwrap();
|
||||
downloads_index
|
||||
.add_documents(docs_to_add.clone(), Some("mod_id"))
|
||||
.unwrap();
|
||||
|
||||
//Updated Index
|
||||
let mut updated_index = client.get_or_create("updated_mods").unwrap();
|
||||
|
||||
let mut updated_rules = default_rules();
|
||||
updated_rules.push_front("desc(updated)".to_string());
|
||||
|
||||
updated_index
|
||||
.set_settings(&default_settings().with_ranking_rules(updated_rules.into()))
|
||||
.unwrap();
|
||||
updated_index
|
||||
.add_documents(docs_to_add.clone(), Some("mod_id"))
|
||||
.unwrap();
|
||||
|
||||
//Created Index
|
||||
let mut newest_index = client.get_or_create("newest_mods").unwrap();
|
||||
|
||||
let mut newest_rules = default_rules();
|
||||
newest_rules.push_back("desc(created)".to_string());
|
||||
|
||||
newest_index
|
||||
.set_settings(&default_settings().with_ranking_rules(newest_rules.into()))
|
||||
.unwrap();
|
||||
newest_index
|
||||
.add_documents(docs_to_add.clone(), Some("mod_id"))
|
||||
.unwrap();
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
//region Utils
|
||||
fn default_rules() -> VecDeque<String> {
|
||||
vec![
|
||||
"typo".to_string(),
|
||||
"words".to_string(),
|
||||
"proximity".to_string(),
|
||||
"attribute".to_string(),
|
||||
"wordsPosition".to_string(),
|
||||
"exactness".to_string(),
|
||||
]
|
||||
.into()
|
||||
}
|
||||
|
||||
fn default_settings() -> Settings {
|
||||
let displayed_attributes = vec![
|
||||
"mod_id".to_string(),
|
||||
"author".to_string(),
|
||||
"title".to_string(),
|
||||
"description".to_string(),
|
||||
"keywords".to_string(),
|
||||
"versions".to_string(),
|
||||
"downloads".to_string(),
|
||||
"page_url".to_string(),
|
||||
"icon_url".to_string(),
|
||||
"author_url".to_string(),
|
||||
"date_created".to_string(),
|
||||
"created".to_string(),
|
||||
"date_modified".to_string(),
|
||||
"updated".to_string(),
|
||||
"latest_version".to_string(),
|
||||
"empty".to_string(),
|
||||
];
|
||||
|
||||
let searchable_attributes = vec![
|
||||
"title".to_string(),
|
||||
"description".to_string(),
|
||||
"keywords".to_string(),
|
||||
"versions".to_string(),
|
||||
"author".to_string(),
|
||||
"empty".to_string(),
|
||||
];
|
||||
|
||||
Settings::new()
|
||||
.with_displayed_attributes(displayed_attributes.clone())
|
||||
.with_searchable_attributes(searchable_attributes.clone())
|
||||
.with_accept_new_fields(true)
|
||||
.with_stop_words(vec![])
|
||||
.with_synonyms(HashMap::new())
|
||||
}
|
||||
|
||||
//endregion
|
||||
105
src/search/mod.rs
Normal file
105
src/search/mod.rs
Normal file
@@ -0,0 +1,105 @@
|
||||
use crate::database::DatabaseError;
|
||||
use crate::models::mods::SearchRequest;
|
||||
use meilisearch_sdk::client::Client;
|
||||
use meilisearch_sdk::document::Document;
|
||||
use meilisearch_sdk::search::Query;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use thiserror::Error;
|
||||
|
||||
pub mod indexing;
|
||||
|
||||
#[derive(Error, Debug)]
|
||||
pub enum SearchError {
|
||||
#[error("Error while connection to the MeiliSearch database")]
|
||||
IndexDBError(),
|
||||
#[error("Error while connecting to the local server")]
|
||||
LocalDatabaseError(#[from] mongodb::error::Error),
|
||||
#[error("Error while accessing the data from remote")]
|
||||
RemoteWebsiteError(#[from] reqwest::Error),
|
||||
#[error("Error while serializing or deserializing JSON")]
|
||||
SerDeError(#[from] serde_json::Error),
|
||||
#[error("Error while parsing float")]
|
||||
FloatParsingError(#[from] std::num::ParseFloatError),
|
||||
#[error("Error while parsing float")]
|
||||
IntParsingError(#[from] std::num::ParseIntError),
|
||||
#[error("Error while parsing BSON")]
|
||||
DatabaseError(#[from] DatabaseError),
|
||||
#[error("Environment Error")]
|
||||
EnvError(#[from] dotenv::Error),
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone)]
|
||||
pub struct SearchMod {
|
||||
pub mod_id: i32,
|
||||
pub author: String,
|
||||
pub title: String,
|
||||
pub description: String,
|
||||
pub keywords: Vec<String>,
|
||||
pub versions: Vec<String>,
|
||||
pub downloads: i32,
|
||||
pub page_url: String,
|
||||
pub icon_url: String,
|
||||
pub author_url: String,
|
||||
pub date_created: String,
|
||||
pub created: i64,
|
||||
pub date_modified: String,
|
||||
pub updated: i64,
|
||||
pub latest_version: String,
|
||||
pub empty: String,
|
||||
}
|
||||
|
||||
impl Document for SearchMod {
|
||||
type UIDType = i32;
|
||||
|
||||
fn get_uid(&self) -> &Self::UIDType {
|
||||
&self.mod_id
|
||||
}
|
||||
}
|
||||
|
||||
pub fn search_for_mod(info: &SearchRequest) -> Result<Vec<SearchMod>, SearchError> {
|
||||
let address = &*dotenv::var("MEILISEARCH_ADDR")?;
|
||||
let client = Client::new(address, "");
|
||||
|
||||
let search_query: &str;
|
||||
let mut filters = String::new();
|
||||
let mut offset = 0;
|
||||
let mut index = "relevance";
|
||||
|
||||
match info.query.as_ref() {
|
||||
Some(q) => search_query = q,
|
||||
None => search_query = "{}{}{}",
|
||||
}
|
||||
|
||||
if let Some(f) = info.filters.as_ref() {
|
||||
filters = f.clone();
|
||||
}
|
||||
|
||||
if let Some(v) = info.version.as_ref() {
|
||||
if filters.is_empty() {
|
||||
filters = v.clone();
|
||||
} else {
|
||||
filters = format!("({}) AND ({})", filters, v);
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(o) = info.offset.as_ref() {
|
||||
offset = o.parse().unwrap();
|
||||
}
|
||||
|
||||
if let Some(s) = info.index.as_ref() {
|
||||
index = s;
|
||||
}
|
||||
|
||||
let mut query = Query::new(search_query).with_limit(10).with_offset(offset);
|
||||
|
||||
if !filters.is_empty() {
|
||||
query = query.with_filters(&filters);
|
||||
}
|
||||
|
||||
Ok(client
|
||||
.get_index(format!("{}_mods", index).as_ref())
|
||||
.unwrap()
|
||||
.search::<SearchMod>(&query)
|
||||
.unwrap()
|
||||
.hits)
|
||||
}
|
||||
Reference in New Issue
Block a user