Switch to Postgres (#39)

* WIP Switch to Postgres

* feat(postgres): more work on porting to postgres, now compiles

* feat(docker-compose): Changed the docker-compose.yml file to use postgres.

* Update docker, documentation, gh actions...

* Remove bson dependency

* Remove bson import

* feat: move mock filehost to trait rather than cargo feature

* feat(postgres): transactions for mod creation, multipart refactor

* fix: Add Cargo.lock so that sqlx functions

* Update sqlx offline build data

* fix: Use SQLX_OFFLINE to force sqlx into offline mode for CI

* Default release channels

* feat(postgres): refactor database models to fit postgres models

* fix: Fix sqlx prepare, fix double allocation in indexing

* Add dockerfile (#40)

Co-authored-by: Charalampos Fanoulis <charalampos.fanoulis@gmail.com>

Co-authored-by: Aeledfyr <aeledfyr@gmail.com>
Co-authored-by: redblueflame <contact@redblueflame.com>
Co-authored-by: Jai A <jai.a@tuta.io>
Co-authored-by: Valentin Ricard <redblueflame1@gmail.Com>
Co-authored-by: Charalampos Fanoulis <charalampos.fanoulis@gmail.com>

src/database/mod.rs

@@ -1,19 +1,6 @@
 pub mod models;
-mod mongo_database;
+mod postgres_database;
 
 pub use models::Mod;
 pub use models::Version;
-pub use mongo_database::connect;
-use thiserror::Error;
-
-type Result<T> = std::result::Result<T, DatabaseError>;
-
-#[derive(Error, Debug)]
-pub enum DatabaseError {
-    #[error("Impossible to find document")]
-    NotFound(),
-    #[error("BSON deserialization error")]
-    BsonError(#[from] bson::de::Error),
-    #[error("Local database error")]
-    LocalDatabaseError(#[from] mongodb::error::Error),
-}
+pub use postgres_database::connect;

src/database/models/ids.rs

@@ -0,0 +1,141 @@
+use super::DatabaseError;
+use crate::models::ids::random_base62;
+use sqlx_macros::Type;
+
+const ID_RETRY_COUNT: usize = 20;
+
+macro_rules! generate_ids {
+    ($vis:vis $function_name:ident, $return_type:ty, $id_length:expr, $select_stmnt:literal, $id_function:expr) => {
+        $vis async fn $function_name(
+            con: &mut sqlx::Transaction<'_, sqlx::Postgres>,
+        ) -> Result<$return_type, DatabaseError> {
+            let length = $id_length;
+            let mut id = random_base62(length);
+            let mut retry_count = 0;
+
+            // Check if ID is unique
+            loop {
+                let results = sqlx::query!($select_stmnt, id as i64)
+                    .fetch_one(&mut *con)
+                    .await?;
+
+                if results.exists.unwrap_or(true) {
+                    id = random_base62(length);
+                } else {
+                    break;
+                }
+
+                retry_count += 1;
+                if retry_count > ID_RETRY_COUNT {
+                    return Err(DatabaseError::RandomIdError);
+                }
+            }
+
+            Ok($id_function(id as i64))
+        }
+    };
+}
+
+generate_ids!(
+    pub generate_mod_id,
+    ModId,
+    8,
+    "SELECT EXISTS(SELECT 1 FROM mods WHERE id=$1)",
+    ModId
+);
+generate_ids!(
+    pub generate_version_id,
+    VersionId,
+    8,
+    "SELECT EXISTS(SELECT 1 FROM versions WHERE id=$1)",
+    VersionId
+);
+generate_ids!(
+    pub generate_team_id,
+    TeamId,
+    8,
+    "SELECT EXISTS(SELECT 1 FROM teams WHERE id=$1)",
+    TeamId
+);
+generate_ids!(
+    pub generate_file_id,
+    FileId,
+    8,
+    "SELECT EXISTS(SELECT 1 FROM files WHERE id=$1)",
+    FileId
+);
+generate_ids!(
+    pub generate_team_member_id,
+    TeamMemberId,
+    8,
+    "SELECT EXISTS(SELECT 1 FROM team_members WHERE id=$1)",
+    TeamMemberId
+);
+
+#[derive(Copy, Clone, Debug, Type)]
+pub struct UserId(pub i64);
+
+#[derive(Copy, Clone, Debug, Type)]
+pub struct TeamId(pub i64);
+#[derive(Copy, Clone, Debug, Type)]
+pub struct TeamMemberId(pub i64);
+
+#[derive(Copy, Clone, Debug, Type)]
+pub struct ModId(pub i64);
+
+#[derive(Copy, Clone, Debug, Type)]
+pub struct VersionId(pub i64);
+#[derive(Copy, Clone, Debug, Type)]
+pub struct ChannelId(pub i64);
+#[derive(Copy, Clone, Debug, Type)]
+pub struct GameVersionId(pub i32);
+#[derive(Copy, Clone, Debug, Type)]
+pub struct LoaderId(pub i32);
+#[derive(Copy, Clone, Debug, Type)]
+pub struct CategoryId(pub i32);
+
+#[derive(Copy, Clone, Debug, Type)]
+pub struct FileId(pub i64);
+
+use crate::models::ids;
+
+impl From<ids::ModId> for ModId {
+    fn from(id: ids::ModId) -> Self {
+        ModId(id.0 as i64)
+    }
+}
+impl From<ModId> for ids::ModId {
+    fn from(id: ModId) -> Self {
+        ids::ModId(id.0 as u64)
+    }
+}
+impl From<ids::UserId> for UserId {
+    fn from(id: ids::UserId) -> Self {
+        UserId(id.0 as i64)
+    }
+}
+impl From<UserId> for ids::UserId {
+    fn from(id: UserId) -> Self {
+        ids::UserId(id.0 as u64)
+    }
+}
+impl From<ids::TeamId> for TeamId {
+    fn from(id: ids::TeamId) -> Self {
+        TeamId(id.0 as i64)
+    }
+}
+impl From<TeamId> for ids::TeamId {
+    fn from(id: TeamId) -> Self {
+        ids::TeamId(id.0 as u64)
+    }
+}
+impl From<ids::VersionId> for VersionId {
+    fn from(id: ids::VersionId) -> Self {
+        VersionId(id.0 as i64)
+    }
+}
+impl From<VersionId> for ids::VersionId {
+    fn from(id: VersionId) -> Self {
+        ids::VersionId(id.0 as u64)
+    }
+}

src/database/models/mod.rs

@@ -1,32 +1,25 @@
-mod mod_item;
-mod team_item;
-mod version_item;
+#![allow(unused)]
+// TODO: remove attr once routes are created
 
-use crate::database::DatabaseError::NotFound;
-use crate::database::Result;
-use async_trait::async_trait;
-use bson::doc;
-use bson::Document;
+use thiserror::Error;
+
+pub mod ids;
+pub mod mod_item;
+pub mod team_item;
+pub mod version_item;
+
+pub use ids::*;
 pub use mod_item::Mod;
-use mongodb::Database;
 pub use team_item::Team;
 pub use team_item::TeamMember;
 pub use version_item::FileHash;
 pub use version_item::Version;
 pub use version_item::VersionFile;
 
-#[async_trait]
-pub trait Item {
-    fn get_collection() -> &'static str;
-    async fn get_by_id(client: Database, id: &str) -> Result<Box<Self>> {
-        let filter = doc! { "_id": id };
-        let collection = client.collection(Self::get_collection());
-        let doc: Document = match collection.find_one(filter, None).await? {
-            Some(e) => e,
-            None => return Err(NotFound()),
-        };
-        let elem: Box<Self> = Self::from_doc(doc)?;
-        Ok(elem)
-    }
-    fn from_doc(elem: Document) -> Result<Box<Self>>;
+#[derive(Error, Debug)]
+pub enum DatabaseError {
+    #[error("Error while interacting with the database")]
+    DatabaseError(#[from] sqlx::error::Error),
+    #[error("Error while trying to generate random ID")]
+    RandomIdError,
 }

src/database/models/mod_item.rs

@@ -1,36 +1,108 @@
-use crate::database::models::team_item::Team;
-use crate::database::models::Item;
-use crate::database::Result;
-use bson::{Bson, Document};
-use serde::{Deserialize, Serialize};
+use super::ids::*;
 
-#[derive(Deserialize, Serialize)]
-pub struct Mod {
-    /// The ID for the mod, must be serializable to base62
-    pub id: i32,
-    //Todo: Move to own table
-    /// The team that owns the mod
-    pub team: Team,
+pub struct ModBuilder {
+    pub mod_id: ModId,
+    pub team_id: TeamId,
     pub title: String,
     pub description: String,
     pub body_url: String,
-    pub published: String,
+    pub icon_url: Option<String>,
+    pub issues_url: Option<String>,
+    pub source_url: Option<String>,
+    pub wiki_url: Option<String>,
+    pub categories: Vec<CategoryId>,
+    pub initial_versions: Vec<super::version_item::VersionBuilder>,
+}
+
+impl ModBuilder {
+    pub async fn insert(
+        self,
+        transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
+    ) -> Result<ModId, super::DatabaseError> {
+        let mod_struct = Mod {
+            id: self.mod_id,
+            team_id: self.team_id,
+            title: self.title,
+            description: self.description,
+            body_url: self.body_url,
+            published: chrono::Utc::now(),
+            downloads: 0,
+            icon_url: self.icon_url,
+            issues_url: self.issues_url,
+            source_url: self.source_url,
+            wiki_url: self.wiki_url,
+        };
+        mod_struct.insert(&mut *transaction).await?;
+
+        for mut version in self.initial_versions {
+            version.mod_id = self.mod_id;
+            version.insert(&mut *transaction).await?;
+        }
+
+        for category in self.categories {
+            sqlx::query(
+                "
+                INSERT INTO mod_categories (joining_mod_id, joining_category_id)
+                VALUES ($1, $2)
+                ",
+            )
+            .bind(self.mod_id)
+            .bind(category)
+            .execute(&mut *transaction)
+            .await?;
+        }
+
+        Ok(self.mod_id)
+    }
+}
+
+pub struct Mod {
+    pub id: ModId,
+    pub team_id: TeamId,
+    pub title: String,
+    pub description: String,
+    pub body_url: String,
+    pub published: chrono::DateTime<chrono::Utc>,
     pub downloads: i32,
-    pub categories: Vec<String>,
-    ///A vector of Version IDs specifying the mod version of a dependency
-    pub version_ids: Vec<i32>,
     pub icon_url: Option<String>,
     pub issues_url: Option<String>,
     pub source_url: Option<String>,
     pub wiki_url: Option<String>,
 }
-impl Item for Mod {
-    fn get_collection() -> &'static str {
-        "mods"
-    }
 
-    fn from_doc(elem: Document) -> Result<Box<Mod>> {
-        let result: Mod = bson::from_bson(Bson::from(elem))?;
-        Ok(Box::from(result))
+impl Mod {
+    pub async fn insert(
+        &self,
+        transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
+    ) -> Result<(), sqlx::error::Error> {
+        sqlx::query(
+            "
+            INSERT INTO mods (
+                id, team_id, title, description, body_url,
+                published, downloads, icon_url, issues_url,
+                source_url, wiki_url
+            )
+            VALUES (
+                $1, $2, $3, $4, $5,
+                $6, $7, $8, $9,
+                $10, $11
+            )
+            ",
+        )
+        .bind(self.id)
+        .bind(self.team_id)
+        .bind(&self.title)
+        .bind(&self.description)
+        .bind(&self.body_url)
+        .bind(self.published)
+        .bind(self.downloads)
+        .bind(self.icon_url.as_ref())
+        .bind(self.issues_url.as_ref())
+        .bind(self.source_url.as_ref())
+        .bind(self.wiki_url.as_ref())
+        .execute(&mut *transaction)
+        .await?;
+
+        Ok(())
     }
 }

src/database/models/team_item.rs

@@ -1,19 +1,74 @@
-use serde::{Deserialize, Serialize};
+use super::ids::*;
+
+pub struct TeamBuilder {
+    pub members: Vec<TeamMemberBuilder>,
+}
+pub struct TeamMemberBuilder {
+    pub user_id: UserId,
+    pub name: String,
+    pub role: String,
+}
+
+impl TeamBuilder {
+    pub async fn insert(
+        self,
+        transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
+    ) -> Result<TeamId, super::DatabaseError> {
+        let team_id = generate_team_id(&mut *transaction).await?;
+
+        let team = Team { id: team_id };
+
+        sqlx::query(
+            "
+            INSERT INTO teams (id)
+            VALUES ($1)
+            ",
+        )
+        .bind(team.id)
+        .execute(&mut *transaction)
+        .await?;
+
+        for member in self.members {
+            let team_member_id = generate_team_member_id(&mut *transaction).await?;
+            let team_member = TeamMember {
+                id: team_member_id,
+                team_id,
+                user_id: member.user_id,
+                name: member.name,
+                role: member.role,
+            };
+
+            sqlx::query(
+                "
+                INSERT INTO team_members (id, team_id, user_id, name, role)
+                VALUES ($1, $2)
+                ",
+            )
+            .bind(team_member.id)
+            .bind(team_member.team_id)
+            .bind(team_member.user_id)
+            .bind(team_member.name)
+            .bind(team_member.role)
+            .execute(&mut *transaction)
+            .await?;
+        }
+
+        Ok(team_id)
+    }
+}
 
 /// A team of users who control a mod
-#[derive(Serialize, Deserialize)]
 pub struct Team {
     /// The id of the team
-    pub id: i32,
-    /// A list of the members of the team
-    pub members: Vec<TeamMember>,
+    pub id: TeamId,
 }
 
 /// A member of a team
-#[derive(Serialize, Deserialize, Clone)]
 pub struct TeamMember {
+    pub id: TeamMemberId,
+    pub team_id: TeamId,
     /// The ID of the user associated with the member
-    pub user_id: i32,
+    pub user_id: UserId,
     /// The name of the user
     pub name: String,
     pub role: String,

src/database/models/version_item.rs

@@ -1,48 +1,215 @@
-use crate::database::models::Item;
-use crate::database::Result;
-use bson::{Bson, Document};
-use serde::{Deserialize, Serialize};
+use super::ids::*;
+use super::DatabaseError;
 
-//TODO: Files should probably be moved to their own table
-#[derive(Deserialize, Serialize)]
-pub struct Version {
-    ///The unqiue VersionId of this version
-    pub version_id: i32,
-    /// The ModId of the mod that this version belongs to
-    pub mod_id: i32,
+pub struct VersionBuilder {
+    pub version_id: VersionId,
+    pub mod_id: ModId,
     pub name: String,
-    pub number: String,
+    pub version_number: String,
     pub changelog_url: Option<String>,
-    pub date_published: String,
-    pub downloads: i32,
-    pub files: Vec<VersionFile>,
-    pub dependencies: Vec<i32>,
-    pub game_versions: Vec<String>,
-    pub loaders: Vec<String>,
-    pub version_type: String,
+    pub files: Vec<VersionFileBuilder>,
+    pub dependencies: Vec<VersionId>,
+    pub game_versions: Vec<GameVersionId>,
+    pub loaders: Vec<LoaderId>,
+    pub release_channel: ChannelId,
 }
 
-#[derive(Serialize, Deserialize)]
-pub struct VersionFile {
-    pub game_versions: Vec<String>,
-    pub hashes: Vec<FileHash>,
+pub struct VersionFileBuilder {
     pub url: String,
+    pub filename: String,
+    pub hashes: Vec<HashBuilder>,
 }
 
-/// A hash of a mod's file
-#[derive(Serialize, Deserialize)]
-pub struct FileHash {
+pub struct HashBuilder {
     pub algorithm: String,
-    pub hash: String,
+    pub hash: Vec<u8>,
 }
 
-impl Item for Version {
-    fn get_collection() -> &'static str {
-        "versions"
-    }
+impl VersionBuilder {
+    pub async fn insert(
+        self,
+        transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
+    ) -> Result<VersionId, DatabaseError> {
+        let version = Version {
+            id: self.version_id,
+            mod_id: self.mod_id,
+            name: self.name,
+            version_number: self.version_number,
+            changelog_url: self.changelog_url,
+            date_published: chrono::Utc::now(),
+            downloads: 0,
+            release_channel: self.release_channel,
+        };
 
-    fn from_doc(elem: Document) -> Result<Box<Version>> {
-        let version: Version = bson::from_bson(Bson::from(elem))?;
-        Ok(Box::from(version))
+        version.insert(&mut *transaction).await?;
+
+        for file in self.files {
+            let file_id = generate_file_id(&mut *transaction).await?;
+            sqlx::query(
+                "
+                INSERT INTO files (id, version_id, url, filename)
+                VALUES ($1, $2, $3, $4)
+                ",
+            )
+            .bind(file_id)
+            .bind(self.version_id)
+            .bind(file.url)
+            .bind(file.filename)
+            .execute(&mut *transaction)
+            .await?;
+
+            for hash in file.hashes {
+                sqlx::query(
+                    "
+                    INSERT INTO hashes (file_id, algorithm, hash)
+                    VALUES ($1, $2, $3)
+                    ",
+                )
+                .bind(file_id)
+                .bind(hash.algorithm)
+                .bind(hash.hash)
+                .execute(&mut *transaction)
+                .await?;
+            }
+        }
+
+        for dependency in self.dependencies {
+            sqlx::query(
+                "
+                INSERT INTO dependencies (dependent_id, dependency_id)
+                VALUES ($1, $2)
+                ",
+            )
+            .bind(self.version_id)
+            .bind(dependency)
+            .execute(&mut *transaction)
+            .await?;
+        }
+
+        for loader in self.loaders {
+            sqlx::query(
+                "
+                INSERT INTO dependencies (loader_id, version_id)
+                VALUES ($1, $2)
+                ",
+            )
+            .bind(loader)
+            .bind(self.version_id)
+            .execute(&mut *transaction)
+            .await?;
+        }
+
+        for game_version in self.game_versions {
+            sqlx::query(
+                "
+                INSERT INTO dependencies (game_version_id, joining_version_id)
+                VALUES ($1, $2)
+                ",
+            )
+            .bind(game_version)
+            .bind(self.version_id)
+            .execute(&mut *transaction)
+            .await?;
+        }
+
+        Ok(self.version_id)
     }
 }
+
+pub struct Version {
+    pub id: VersionId,
+    pub mod_id: ModId,
+    pub name: String,
+    pub version_number: String,
+    pub changelog_url: Option<String>,
+    pub date_published: chrono::DateTime<chrono::Utc>,
+    pub downloads: i32,
+    pub release_channel: ChannelId,
+}
+
+impl Version {
+    pub async fn insert(
+        &self,
+        transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
+    ) -> Result<(), sqlx::error::Error> {
+        sqlx::query(
+            "
+            INSERT INTO versions (
+                id, mod_id, name, version_number,
+                changelog_url, date_published,
+                downloads, release_channel
+            )
+            VALUES (
+                $1, $2, $3, $4,
+                $5, $6,
+                $7, $8
+            )
+            ",
+        )
+        .bind(self.id)
+        .bind(self.mod_id)
+        .bind(&self.name)
+        .bind(&self.version_number)
+        .bind(self.changelog_url.as_ref())
+        .bind(self.date_published)
+        .bind(self.downloads)
+        .bind(self.release_channel)
+        .execute(&mut *transaction)
+        .await?;
+
+        Ok(())
+    }
+
+    pub async fn get_dependencies<'a, E>(&self, exec: E) -> Result<Vec<VersionId>, sqlx::Error>
+    where
+        E: sqlx::Executor<'a, Database = sqlx::Postgres>,
+    {
+        use futures::stream::TryStreamExt;
+
+        let vec = sqlx::query_as::<_, (VersionId,)>(
+            "
+            SELECT id FROM versions v
+            INNER JOIN dependencies d ON d.dependency_id = v.id
+            WHERE d.dependent_id = $1
+            ",
+        )
+        .bind(self.id)
+        .fetch_many(exec)
+        .try_filter_map(|e| async { Ok(e.right().map(|(v,)| v)) })
+        .try_collect::<Vec<VersionId>>()
+        .await?;
+
+        Ok(vec)
+    }
+}
+
+pub struct ReleaseChannel {
+    pub id: ChannelId,
+    pub channel: String,
+}
+pub struct Loader {
+    pub id: LoaderId,
+    pub loader: String,
+}
+pub struct GameVersion {
+    pub id: GameVersionId,
+    pub version: String,
+}
+
+pub struct VersionFile {
+    pub id: FileId,
+    pub version_id: VersionId,
+    pub url: String,
+    pub filename: String,
+}
+
+pub struct FileHash {
+    pub file_id: FileId,
+    pub algorithm: String,
+    pub hash: Vec<u8>,
+}
+
+pub struct Category {
+    pub id: CategoryId,
+    pub category: String,
+}

src/database/mongo_database.rs

@@ -1,14 +0,0 @@
-use log::info;
-use mongodb::error::Error;
-use mongodb::options::ClientOptions;
-use mongodb::Client;
-
-pub async fn connect() -> Result<Client, Error> {
-    info!("Initializing database connection");
-
-    let mongodb_addr = dotenv::var("MONGODB_ADDR").expect("`MONGO_ADDR` not in .env");
-    let mut client_options = ClientOptions::parse(&mongodb_addr).await?;
-    client_options.app_name = Some("labrinth".to_string());
-
-    Client::with_options(client_options)
-}

src/database/postgres_database.rs

@@ -0,0 +1,14 @@
+use log::info;
+use sqlx::postgres::{PgPool, PgPoolOptions};
+
+pub async fn connect() -> Result<PgPool, sqlx::Error> {
+    info!("Initializing database connection");
+
+    let database_url = dotenv::var("DATABASE_URL").expect("`DATABASE_URL` not in .env");
+    let pool = PgPoolOptions::new()
+        .max_connections(20)
+        .connect(&database_url)
+        .await?;
+
+    Ok(pool)
+}