Migrate to MongoDB

.idea/dictionaries/geometrically.xml

@@ -1,9 +0,0 @@
-<component name="ProjectDictionaryState">
-  <dictionary name="geometrically">
-    <words>
-      <w>curseforge</w>
-      <w>serde</w>
-      <w>tantivy</w>
-    </words>
-  </dictionary>
-</component>

Cargo.toml

@@ -14,8 +14,11 @@ serde_json = "1.0"
 serde = {version="1.0", features=["derive"]}
 meilisearch-sdk = "0.1.3"
 actix-files = "0.2.1"
-diesel = { version = "1.4.4", features = ["postgres", "serde_json", "chrono"] }
-dotenv = "0.9.0"
 chrono = "0.4.11"
 human_format = "1.0.3"
 reqwest = "0.10.4"
+log = "0.4.8"
+env_logger = "0.7.1"
+mongodb = "0.10.0"
+bson = "0.14.1"
+futures = "0.3.5"

diesel.toml

@@ -1,5 +0,0 @@
-# For documentation on how to configure this file,
-# see diesel.rs/guides/configuring-diesel-cli
-
-[print_schema]
-file = "src/schema.rs"

migrations/00000000000000_diesel_initial_setup/down.sql

@@ -1,6 +0,0 @@
--- This file was automatically created by Diesel to setup helper functions
--- and other internal bookkeeping. This file is safe to edit, any future
--- changes will be added to existing projects as new migrations.
-
-DROP FUNCTION IF EXISTS diesel_manage_updated_at(_tbl regclass);
-DROP FUNCTION IF EXISTS diesel_set_updated_at();

migrations/00000000000000_diesel_initial_setup/up.sql

@@ -1,36 +0,0 @@
--- This file was automatically created by Diesel to setup helper functions
--- and other internal bookkeeping. This file is safe to edit, any future
--- changes will be added to existing projects as new migrations.
-
-
-
-
--- Sets up a trigger for the given table to automatically set a column called
--- `updated_at` whenever the row is modified (unless `updated_at` was included
--- in the modified columns)
---
--- # Example
---
--- ```sql
--- CREATE TABLE users (id SERIAL PRIMARY KEY, updated_at TIMESTAMP NOT NULL DEFAULT NOW());
---
--- SELECT diesel_manage_updated_at('users');
--- ```
-CREATE OR REPLACE FUNCTION diesel_manage_updated_at(_tbl regclass) RETURNS VOID AS $$
-BEGIN
-    EXECUTE format('CREATE TRIGGER set_updated_at BEFORE UPDATE ON %s
-                    FOR EACH ROW EXECUTE PROCEDURE diesel_set_updated_at()', _tbl);
-END;
-$$ LANGUAGE plpgsql;
-
-CREATE OR REPLACE FUNCTION diesel_set_updated_at() RETURNS trigger AS $$
-BEGIN
-    IF (
-        NEW IS DISTINCT FROM OLD AND
-        NEW.updated_at IS NOT DISTINCT FROM OLD.updated_at
-    ) THEN
-        NEW.updated_at := current_timestamp;
-    END IF;
-    RETURN NEW;
-END;
-$$ LANGUAGE plpgsql;

migrations/2020-05-15-055106_create_mods/down.sql

@@ -1 +0,0 @@
-DROP TABLE mods

migrations/2020-05-15-055106_create_mods/up.sql

@@ -1,11 +0,0 @@
-CREATE TABLE mods (
-    id SERIAL PRIMARY KEY,
-    title VARCHAR NOT NULL,
-    description VARCHAR NOT NULL,
-    datePublished DATE NOT NULL,
-    author VARCHAR NOT NULL,
-    downloads INTEGER NOT NULL DEFAULT 0,
-    categories TEXT[] NOT NULL,
-    body_path VARCHAR NOT NULL,
-    icon_path VARCHAR NOT NULL
-)

migrations/2020-05-15-055207_create_versions/down.sql

@@ -1 +0,0 @@
-DROP TABLE versions

migrations/2020-05-15-055207_create_versions/up.sql

@@ -1,12 +0,0 @@
-CREATE TABLE versions (
-    version_id SERIAL PRIMARY KEY,
-    mod_id SERIAL NOT NULL,
-    title VARCHAR NOT NULL,
-    changelog_path VARCHAR NOT NULL,
-    files_path TEXT[] NOT NULL,
-    date_published DATE NOT NULL,
-    author VARCHAR NOT NULL,
-    downloads INTEGER NOT NULL DEFAULT 0,
-    dependencies TEXT[] NOT NULL,
-    game_versions TEXT[] NOT NULL
-)

src/database/database.rs

@@ -1,12 +1,12 @@
-use diesel::pg::PgConnection;
+use mongodb::options::ClientOptions;
-use diesel::prelude::*;
+use mongodb::Client;
-use dotenv::dotenv;
+use mongodb::error::Error;
-use std::env;
 
-pub fn connect() -> PgConnection {
+pub async fn connect() -> Result<Client, Error> {
-    dotenv().ok();
+    info!("Initializing database connection");
 
-    let database_url = env::var("DATABASE_URL").expect("DATABASE_URL must be set!");
+    let mut client_options = ClientOptions::parse("mongodb://localhost:27017").await?;
-    PgConnection::establish(&database_url)
+    client_options.app_name = Some("Actix Web Server".to_string());
-        .unwrap_or_else(|_| panic!("Error connecting to {}", database_url))
+
+   Client::with_options(client_options)
 }

src/database/models.rs

@@ -1,11 +1,11 @@
-use chrono::NaiveDate;
+use serde::{Serialize, Deserialize};
 
-#[derive(Queryable)]
+#[derive(Deserialize, Serialize)]
 pub struct Mod {
     pub id: i32,
     pub title: String,
     pub description: String,
-    pub published: NaiveDate,
+    pub published: String,
     pub author: String,
     pub downloads: i32,
     pub categories: Vec<String>,
@@ -13,14 +13,14 @@ pub struct Mod {
     pub icon_path: String,
 }
 
-#[derive(Queryable)]
+#[derive(Deserialize, Serialize)]
 pub struct Version {
     pub version_id: i32,
     pub mod_id: i32,
     pub title: String,
     pub changelog_path: String,
     pub files_path: Vec<String>,
-    pub date_published: NaiveDate,
+    pub date_published: String,
     pub author: String,
     pub downloads: i32,
     pub dependencies: Vec<String>,

src/main.rs

@@ -2,19 +2,25 @@
 extern crate serde_json;
 
 #[macro_use]
-extern crate diesel;
+extern crate bson;
+
+#[macro_use]
+extern crate log;
 
 use actix_files as fs;
 use actix_web::{web, App, HttpServer};
 use handlebars::*;
+use actix_web::middleware::Logger;
+use env_logger::Env;
 
 mod database;
 mod helpers;
 mod routes;
-mod schema;
 
 #[actix_rt::main]
 async fn main() -> std::io::Result<()> {
+    env_logger::from_env(Env::default().default_filter_or("info")).init();
+
     //Handlebars
     let mut handlebars = Handlebars::new();
 
@@ -26,13 +32,17 @@ async fn main() -> std::io::Result<()> {
 
     let handlebars_ref = web::Data::new(handlebars);
 
-    let database = database::connect();
+    let client = database::connect().await.unwrap();
-    routes::index_mods(database).await;
+    routes::index_mods(client.clone()).await;
 
+    info!("Starting Actix HTTP server!");
     //Init App
     HttpServer::new(move || {
         App::new()
+            .wrap(Logger::default())
+            .wrap(Logger::new("%a %{User-Agent}i"))
             .app_data(handlebars_ref.clone())
+            .app_data(client.clone())
             .service(fs::Files::new("/static", "./static").show_files_listing())
             .service(routes::index_get)
             .service(routes::search_post)

src/routes/search.rs

@@ -1,14 +1,15 @@
-extern crate diesel;
-
 use actix_web::{get, post, web, web::Data, HttpResponse};
 use handlebars::*;
 use meilisearch_sdk::{client::*, document::*, search::*};
 use serde::{Deserialize, Serialize};
 
 use crate::database::*;
-use diesel::prelude::*;
 
+use futures::stream::StreamExt;
 use meilisearch_sdk::settings::Settings;
+use mongodb::error::Error;
+use bson::Bson;
+use mongodb::Cursor;
 
 #[derive(Serialize, Deserialize, Debug)]
 #[serde(rename_all = "camelCase")]
@@ -158,45 +159,16 @@ fn search(web::Query(info): web::Query<SearchRequest>) -> Vec<SearchMod> {
         .hits
 }
 
-pub async fn index_mods(conn: PgConnection) {
+pub async fn index_mods(conn: mongodb::Client) -> Result<(), Error>{
-    use crate::schema::mods::dsl::*;
-    use crate::schema::versions::dsl::*;
-
     let client = Client::new("http://localhost:7700", "");
     let mut mods_index = client.get_or_create("mods").unwrap();
 
-    let results = mods.load::<Mod>(&conn).expect("Error loading mods!");
     let mut docs_to_add: Vec<SearchMod> = vec![];
 
-    for result in results {
+    info!("Indexing database mods!");
-        let mod_versions = versions
-            .filter(mod_id.eq(result.id))
-            .load::<Version>(&conn)
-            .expect("Error loading versions!");
 
-        let mut mod_game_versions = vec![];
 
-        for version in mod_versions {
+    info!("Indexing curseforge mods!");
-            mod_game_versions.extend(version.game_versions.clone())
-        }
-
-        docs_to_add.push(SearchMod {
-            mod_id: result.id,
-            author: result.author,
-            title: result.title,
-            description: result.description,
-            keywords: result.categories,
-            versions: mod_game_versions,
-            downloads: result.downloads,
-            page_url: "".to_string(),
-            icon_url: "".to_string(),
-            author_url: "".to_string(),
-            date_created: "".to_string(),
-            date_modified: "".to_string(),
-            latest_version: "".to_string(),
-            empty: String::from("{}{}{}"),
-        });
-    }
 
     let body = reqwest::get("https://addons-ecs.forgesvc.net/api/v2/addon/search?categoryId=0&gameId=432&index=0&pageSize=10000&sectionId=6&sort=5")
         .await.unwrap()
@@ -237,9 +209,7 @@ pub async fn index_mods(conn: PgConnection) {
                 "Technology" => mod_categories.push(String::from("technology")),
                 "Processing" => mod_categories.push(String::from("technology")),
                 "Player Transport" => mod_categories.push(String::from("technology")),
-                "Energy, Fluid, and Item Transport" => {
+                "Energy, Fluid, and Item Transport" => { mod_categories.push(String::from("technology")) }
-                    mod_categories.push(String::from("technology"))
-                }
                 "Food" => mod_categories.push(String::from("food")),
                 "Farming" => mod_categories.push(String::from("food")),
                 "Energy" => mod_categories.push(String::from("technology")),
@@ -364,4 +334,6 @@ pub async fn index_mods(conn: PgConnection) {
         .with_ranking_rules(ranking_rules);
 
     mods_index.set_settings(&write_settings).unwrap();
+
+    Ok(())
 }

src/schema.rs

@@ -1,30 +0,0 @@
-table! {
-    mods (id) {
-        id -> Int4,
-        title -> Varchar,
-        description -> Varchar,
-        datepublished -> Date,
-        author -> Varchar,
-        downloads -> Int4,
-        categories -> Array<Text>,
-        body_path -> Varchar,
-        icon_path -> Varchar,
-    }
-}
-
-table! {
-    versions (version_id) {
-        version_id -> Int4,
-        mod_id -> Int4,
-        title -> Varchar,
-        changelog_path -> Varchar,
-        files_path -> Array<Text>,
-        date_published -> Date,
-        author -> Varchar,
-        downloads -> Int4,
-        dependencies -> Array<Text>,
-        game_versions -> Array<Text>,
-    }
-}
-
-allow_tables_to_appear_in_same_query!(mods, versions,);