Switch to Postgres (#39)

* WIP Switch to Postgres

* feat(postgres): more work on porting to postgres, now compiles

* feat(docker-compose): Changed the docker-compose.yml file to use postgres.

* Update docker, documentation, gh actions...

* Remove bson dependency

* Remove bson import

* feat: move mock filehost to trait rather than cargo feature

* feat(postgres): transactions for mod creation, multipart refactor

* fix: Add Cargo.lock so that sqlx functions

* Update sqlx offline build data

* fix: Use SQLX_OFFLINE to force sqlx into offline mode for CI

* Default release channels

* feat(postgres): refactor database models to fit postgres models

* fix: Fix sqlx prepare, fix double allocation in indexing

* Add dockerfile (#40)

Co-authored-by: Charalampos Fanoulis <charalampos.fanoulis@gmail.com>

Co-authored-by: Aeledfyr <aeledfyr@gmail.com>
Co-authored-by: redblueflame <contact@redblueflame.com>
Co-authored-by: Jai A <jai.a@tuta.io>
Co-authored-by: Valentin Ricard <redblueflame1@gmail.Com>
Co-authored-by: Charalampos Fanoulis <charalampos.fanoulis@gmail.com>

src/file_hosting/backblaze.rs

@@ -0,0 +1,135 @@
+use super::{DeleteFileData, FileHost, FileHostingError, UploadFileData};
+use async_trait::async_trait;
+
+mod authorization;
+mod delete;
+mod upload;
+
+pub struct BackblazeHost {
+    upload_url_data: authorization::UploadUrlData,
+    authorization_data: authorization::AuthorizationData,
+}
+
+impl BackblazeHost {
+    pub async fn new(key_id: &str, key: &str, bucket_id: &str) -> Self {
+        let authorization_data = authorization::authorize_account(key_id, key).await.unwrap();
+        let upload_url_data = authorization::get_upload_url(&authorization_data, bucket_id)
+            .await
+            .unwrap();
+
+        BackblazeHost {
+            authorization_data,
+            upload_url_data,
+        }
+    }
+}
+
+#[async_trait]
+impl FileHost for BackblazeHost {
+    async fn upload_file(
+        &self,
+        content_type: &str,
+        file_name: &str,
+        file_bytes: Vec<u8>,
+    ) -> Result<UploadFileData, FileHostingError> {
+        let upload_data =
+            upload::upload_file(&self.upload_url_data, content_type, file_name, file_bytes).await?;
+        Ok(UploadFileData {
+            file_id: upload_data.file_id,
+            file_name: upload_data.file_name,
+            content_length: upload_data.content_length,
+            content_sha1: upload_data.content_sha1,
+            content_md5: upload_data.content_md5,
+            content_type: upload_data.content_type,
+            upload_timestamp: upload_data.upload_timestamp,
+        })
+    }
+
+    /*
+        async fn upload_file_streaming(
+            &self,
+            content_type: &str,
+            file_name: &str,
+            stream: reqwest::Body
+        ) -> Result<UploadFileData, FileHostingError> {
+            use futures::stream::StreamExt;
+
+            let mut data = Vec::new();
+            while let Some(chunk) = stream.next().await {
+                data.extend_from_slice(&chunk.map_err(|e| FileHostingError::Other(e))?);
+            }
+            self.upload_file(content_type, file_name, data).await
+        }
+    */
+
+    async fn delete_file_version(
+        &self,
+        file_id: &str,
+        file_name: &str,
+    ) -> Result<DeleteFileData, FileHostingError> {
+        let delete_data =
+            delete::delete_file_version(&self.authorization_data, file_id, file_name).await?;
+        Ok(DeleteFileData {
+            file_id: delete_data.file_id,
+            file_name: delete_data.file_name,
+        })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use authorization::*;
+    use delete::*;
+    use upload::*;
+
+    #[actix_rt::test]
+    async fn test_authorization() {
+        println!("{}", dotenv::var("BACKBLAZE_BUCKET_ID").unwrap());
+        let authorization_data = authorize_account(
+            &dotenv::var("BACKBLAZE_KEY_ID").unwrap(),
+            &dotenv::var("BACKBLAZE_KEY").unwrap(),
+        )
+        .await
+        .unwrap();
+
+        get_upload_url(
+            &authorization_data,
+            &dotenv::var("BACKBLAZE_BUCKET_ID").unwrap(),
+        )
+        .await
+        .unwrap();
+    }
+
+    #[actix_rt::test]
+    async fn test_file_management() {
+        let authorization_data = authorize_account(
+            &dotenv::var("BACKBLAZE_KEY_ID").unwrap(),
+            &dotenv::var("BACKBLAZE_KEY").unwrap(),
+        )
+        .await
+        .unwrap();
+        let upload_url_data = get_upload_url(
+            &authorization_data,
+            &dotenv::var("BACKBLAZE_BUCKET_ID").unwrap(),
+        )
+        .await
+        .unwrap();
+        let upload_data = upload_file(
+            &upload_url_data,
+            "text/plain",
+            "test.txt",
+            "test file".to_string().into_bytes(),
+        )
+        .await
+        .unwrap();
+
+        delete_file_version(
+            &authorization_data,
+            &upload_data.file_id,
+            &upload_data.file_name,
+        )
+        .await
+        .unwrap();
+    }
+}

src/file_hosting/backblaze/authorization.rs

@@ -30,10 +30,9 @@ pub struct UploadUrlData {
     pub authorization_token: String,
 }
 
-#[cfg(feature = "backblaze")]
 pub async fn authorize_account(
-    key_id: String,
-    application_key: String,
+    key_id: &str,
+    application_key: &str,
 ) -> Result<AuthorizationData, FileHostingError> {
     let combined_key = format!("{}:{}", key_id, application_key);
     let formatted_key = format!("Basic {}", base64::encode(combined_key));
@@ -52,17 +51,16 @@ pub async fn authorize_account(
     }
 }
 
-#[cfg(feature = "backblaze")]
 pub async fn get_upload_url(
-    authorization_data: AuthorizationData,
-    bucket_id: String,
+    authorization_data: &AuthorizationData,
+    bucket_id: &str,
 ) -> Result<UploadUrlData, FileHostingError> {
     let response = reqwest::Client::new()
         .post(&format!("{}/b2api/v2/b2_get_upload_url", authorization_data.api_url).to_string())
         .header(reqwest::header::CONTENT_TYPE, "application/json")
         .header(
             reqwest::header::AUTHORIZATION,
-            authorization_data.authorization_token,
+            &authorization_data.authorization_token,
         )
         .body(
             serde_json::json!({
@@ -79,50 +77,3 @@ pub async fn get_upload_url(
         Err(FileHostingError::BackblazeError(response.json().await?))
     }
 }
-
-#[cfg(not(feature = "backblaze"))]
-pub async fn authorize_account(
-    _key_id: String,
-    _application_key: String,
-) -> Result<AuthorizationData, FileHostingError> {
-    Ok(AuthorizationData {
-        absolute_minimum_part_size: 5000000,
-        account_id: String::from("MOCK_ACCOUNT_ID"),
-        allowed: AuthorizationPermissions {
-            bucket_id: None,
-            bucket_name: None,
-            capabilities: vec![
-                String::from("listKeys"),
-                String::from("writeKeys"),
-                String::from("deleteKeys"),
-                String::from("listAllBucketNames"),
-                String::from("listBuckets"),
-                String::from("writeBuckets"),
-                String::from("deleteBuckets"),
-                String::from("readBuckets"),
-                String::from("listFiles"),
-                String::from("readFiles"),
-                String::from("shareFiles"),
-                String::from("writeFiles"),
-                String::from("deleteFiles"),
-            ],
-            name_prefix: None,
-        },
-        api_url: String::from("https://api.example.com"),
-        authorization_token: String::from("MOCK_AUTH_TOKEN"),
-        download_url: String::from("https://download.example.com"),
-        recommended_part_size: 100000000,
-    })
-}
-
-#[cfg(not(feature = "backblaze"))]
-pub async fn get_upload_url(
-    _authorization_data: AuthorizationData,
-    _bucket_id: String,
-) -> Result<UploadUrlData, FileHostingError> {
-    Ok(UploadUrlData {
-        bucket_id: String::from("MOCK_BUCKET_ID"),
-        upload_url: String::from("https://download.example.com"),
-        authorization_token: String::from("MOCK_AUTH_TOKEN"),
-    })
-}

src/file_hosting/backblaze/delete.rs

@@ -1,4 +1,5 @@
-use crate::file_hosting::{AuthorizationData, FileHostingError};
+use super::authorization::AuthorizationData;
+use crate::file_hosting::FileHostingError;
 use serde::{Deserialize, Serialize};
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
@@ -8,7 +9,6 @@ pub struct DeleteFileData {
     pub file_name: String,
 }
 
-#[cfg(feature = "backblaze")]
 pub async fn delete_file_version(
     authorization_data: &AuthorizationData,
     file_id: &str,
@@ -40,19 +40,3 @@ pub async fn delete_file_version(
         Err(FileHostingError::BackblazeError(response.json().await?))
     }
 }
-
-#[cfg(not(feature = "backblaze"))]
-pub async fn delete_file_version(
-    _authorization_data: &AuthorizationData,
-    file_id: &str,
-    file_name: &str,
-) -> Result<DeleteFileData, FileHostingError> {
-    let path = std::path::Path::new(&dotenv::var("MOCK_FILE_PATH").unwrap())
-        .join(file_name.replace("../", ""));
-    std::fs::remove_file(path)?;
-
-    Ok(DeleteFileData {
-        file_id: file_id.to_string(),
-        file_name: file_name.to_string(),
-    })
-}

src/file_hosting/backblaze/upload.rs

@@ -1,4 +1,4 @@
-use crate::file_hosting::authorization::UploadUrlData;
+use super::authorization::UploadUrlData;
 use crate::file_hosting::FileHostingError;
 use serde::{Deserialize, Serialize};
 
@@ -16,7 +16,6 @@ pub struct UploadFileData {
     pub upload_timestamp: u64,
 }
 
-#[cfg(feature = "backblaze")]
 //Content Types found here: https://www.backblaze.com/b2/docs/content-types.html
 pub async fn upload_file(
     url_data: &UploadUrlData,
@@ -47,29 +46,3 @@ pub async fn upload_file(
         Err(FileHostingError::BackblazeError(response.json().await?))
     }
 }
-
-#[cfg(not(feature = "backblaze"))]
-pub async fn upload_file(
-    _url_data: &UploadUrlData,
-    content_type: &str,
-    file_name: &str,
-    file_bytes: Vec<u8>,
-) -> Result<UploadFileData, FileHostingError> {
-    let path = std::path::Path::new(&dotenv::var("MOCK_FILE_PATH").unwrap())
-        .join(file_name.replace("../", ""));
-    std::fs::create_dir_all(path.parent().ok_or(FileHostingError::InvalidFilename)?)?;
-    let content_sha1 = sha1::Sha1::from(&file_bytes).hexdigest();
-
-    std::fs::write(path, &file_bytes)?;
-    Ok(UploadFileData {
-        file_id: String::from("MOCK_FILE_ID"),
-        file_name: file_name.to_string(),
-        account_id: String::from("MOCK_ACCOUNT_ID"),
-        bucket_id: String::from("MOCK_BUCKET_ID"),
-        content_length: file_bytes.len() as u32,
-        content_sha1,
-        content_md5: None,
-        content_type: content_type.to_string(),
-        upload_timestamp: chrono::Utc::now().timestamp_millis() as u64,
-    })
-}

src/file_hosting/mock.rs

@@ -0,0 +1,51 @@
+use super::{DeleteFileData, FileHost, FileHostingError, UploadFileData};
+use async_trait::async_trait;
+
+pub struct MockHost(());
+
+impl MockHost {
+    pub fn new() -> Self {
+        MockHost(())
+    }
+}
+
+#[async_trait]
+impl FileHost for MockHost {
+    async fn upload_file(
+        &self,
+        content_type: &str,
+        file_name: &str,
+        file_bytes: Vec<u8>,
+    ) -> Result<UploadFileData, FileHostingError> {
+        let path = std::path::Path::new(&dotenv::var("MOCK_FILE_PATH").unwrap())
+            .join(file_name.replace("../", ""));
+        std::fs::create_dir_all(path.parent().ok_or(FileHostingError::InvalidFilename)?)?;
+        let content_sha1 = sha1::Sha1::from(&file_bytes).hexdigest();
+
+        std::fs::write(path, &file_bytes)?;
+        Ok(UploadFileData {
+            file_id: String::from("MOCK_FILE_ID"),
+            file_name: file_name.to_string(),
+            content_length: file_bytes.len() as u32,
+            content_sha1,
+            content_md5: None,
+            content_type: content_type.to_string(),
+            upload_timestamp: chrono::Utc::now().timestamp_millis() as u64,
+        })
+    }
+
+    async fn delete_file_version(
+        &self,
+        file_id: &str,
+        file_name: &str,
+    ) -> Result<DeleteFileData, FileHostingError> {
+        let path = std::path::Path::new(&dotenv::var("MOCK_FILE_PATH").unwrap())
+            .join(file_name.replace("../", ""));
+        std::fs::remove_file(path)?;
+
+        Ok(DeleteFileData {
+            file_id: file_id.to_string(),
+            file_name: file_name.to_string(),
+        })
+    }
+}

src/file_hosting/mod.rs

@@ -1,90 +1,53 @@
+use async_trait::async_trait;
 use thiserror::Error;
 
-mod authorization;
-mod delete;
-mod upload;
+mod backblaze;
+mod mock;
 
-pub use authorization::authorize_account;
-pub use authorization::get_upload_url;
-pub use authorization::AuthorizationData;
-pub use authorization::AuthorizationPermissions;
-pub use authorization::UploadUrlData;
-
-pub use upload::upload_file;
-pub use upload::UploadFileData;
-
-pub use delete::delete_file_version;
-pub use delete::DeleteFileData;
+pub use backblaze::BackblazeHost;
+pub use mock::MockHost;
 
 #[derive(Error, Debug)]
 pub enum FileHostingError {
-    #[cfg(feature = "backblaze")]
     #[error("Error while accessing the data from backblaze")]
     HttpError(#[from] reqwest::Error),
-
-    #[cfg(feature = "backblaze")]
     #[error("Backblaze error: {0}")]
     BackblazeError(serde_json::Value),
-
-    #[cfg(not(feature = "backblaze"))]
     #[error("File system error in file hosting: {0}")]
     FileSystemError(#[from] std::io::Error),
-    #[cfg(not(feature = "backblaze"))]
     #[error("Invalid Filename")]
     InvalidFilename,
 }
 
-#[cfg(test)]
-mod tests {
-    use super::*;
-
-    #[actix_rt::test]
-    async fn test_authorization() {
-        println!("{}", dotenv::var("BACKBLAZE_BUCKET_ID").unwrap());
-        let authorization_data = authorize_account(
-            dotenv::var("BACKBLAZE_KEY_ID").unwrap(),
-            dotenv::var("BACKBLAZE_KEY").unwrap(),
-        )
-        .await
-        .unwrap();
-
-        get_upload_url(
-            authorization_data,
-            dotenv::var("BACKBLAZE_BUCKET_ID").unwrap(),
-        )
-        .await
-        .unwrap();
-    }
-
-    #[actix_rt::test]
-    async fn test_file_management() {
-        let authorization_data = authorize_account(
-            dotenv::var("BACKBLAZE_KEY_ID").unwrap(),
-            dotenv::var("BACKBLAZE_KEY").unwrap(),
-        )
-        .await
-        .unwrap();
-        let upload_url_data = get_upload_url(
-            authorization_data.clone(),
-            dotenv::var("BACKBLAZE_BUCKET_ID").unwrap(),
-        )
-        .await
-        .unwrap();
-        let upload_data = upload_file(
-            &upload_url_data,
-            "text/plain",
-            "test.txt",
-            "test file".to_string().into_bytes(),
-        )
-        .await
-        .unwrap();
-
-        delete_file_version(
-            &authorization_data,
-            &upload_data.file_id,
-            &upload_data.file_name,
-        )
-        .await
-        .unwrap();
-    }
+#[derive(Debug, Clone)]
+pub struct UploadFileData {
+    pub file_id: String,
+    pub file_name: String,
+    pub content_length: u32,
+    pub content_sha1: String,
+    pub content_md5: Option<String>,
+    pub content_type: String,
+    pub upload_timestamp: u64,
+}
+
+#[derive(Debug, Clone)]
+pub struct DeleteFileData {
+    pub file_id: String,
+    pub file_name: String,
+}
+
+#[async_trait]
+pub trait FileHost {
+    async fn upload_file(
+        &self,
+        content_type: &str,
+        file_name: &str,
+        file_bytes: Vec<u8>,
+    ) -> Result<UploadFileData, FileHostingError>;
+
+    async fn delete_file_version(
+        &self,
+        file_id: &str,
+        file_name: &str,
+    ) -> Result<DeleteFileData, FileHostingError>;
 }