From 95339a833824510db45229ddf1662afad369f44b Mon Sep 17 00:00:00 2001 From: Aeledfyr <45501007+Aeledfyr@users.noreply.github.com> Date: Thu, 16 Jul 2020 23:06:58 -0500 Subject: [PATCH] Create a mock file host for dev, Fix mod creation route (#38) * fix(mod-creation): fix actix server data & mod creation route * feat(file-host): implement mock file hosting This implements a mock file hosting system backed by the system's filesystem. It mirrors the API of the backblaze integration, but puts the files directly on disk in the path specified by the MOCK_FILE_PATH environment variable (defaults to /tmp/modrinth). The mock file hosting is enabled by default using cargo features to allow people to work on modrinth without access to a valid backblaze account and setup. To enable backblaze, specify the cargo feature "backblaze" when running, ex. `cargo run --features backblaze`. * feat(file-hosting): implement basic backblaze API error handling * fix(mod-creation): fix extension parsing, use base62 ids for paths fix(file-hosting): reduce unnecessary allocations * fix: fix auth with docker mongodb * fix: fix failing checks * fix: remove testing files --- .env | 4 +- Cargo.toml | 6 ++- src/file_hosting/authorization.rs | 79 ++++++++++++++++++++++++++----- src/file_hosting/delete.rs | 51 +++++++++++++------- src/file_hosting/mod.rs | 26 +++++++--- src/file_hosting/upload.rs | 50 +++++++++++++++---- src/main.rs | 10 ++-- src/routes/mod.rs | 1 + src/routes/mod_creation.rs | 72 ++++++++++++++-------------- 9 files changed, 211 insertions(+), 88 deletions(-) diff --git a/.env b/.env index 859e88db..7c251a03 100644 --- a/.env +++ b/.env @@ -3,11 +3,13 @@ DEBUG=true CDN_URL=cdn.modrinth.com -MONGODB_ADDR=mongodb://localhost:27017 +MONGODB_ADDR=mongodb://toor:modrinthadmin@localhost:27017 MEILISEARCH_ADDR=http://localhost:7700 BIND_ADDR=127.0.0.1:8000 +MOCK_FILE_PATH=/tmp/modrinth + BACKBLAZE_KEY_ID=none BACKBLAZE_KEY=none BACKBLAZE_BUCKET_ID=none \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index b674992e..fd495f1a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,4 +36,8 @@ futures = "0.3.5" futures-timer = "3.0.2" base64 = "0.12.3" -sha1 = {version="0.6.0", features=["std"]} \ No newline at end of file +sha1 = {version="0.6.0", features=["std"]} + +[features] +default = [] +backblaze = [] diff --git a/src/file_hosting/authorization.rs b/src/file_hosting/authorization.rs index ee5e291e..eb14cad6 100644 --- a/src/file_hosting/authorization.rs +++ b/src/file_hosting/authorization.rs @@ -1,7 +1,5 @@ use crate::file_hosting::FileHostingError; -use base64::encode; use serde::{Deserialize, Serialize}; -use serde_json::json; #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] @@ -32,28 +30,34 @@ pub struct UploadUrlData { pub authorization_token: String, } +#[cfg(feature = "backblaze")] pub async fn authorize_account( key_id: String, application_key: String, ) -> Result { let combined_key = format!("{}:{}", key_id, application_key); - let formatted_key = format!("Basic {}", encode(combined_key)); + let formatted_key = format!("Basic {}", base64::encode(combined_key)); - Ok(reqwest::Client::new() + let response = reqwest::Client::new() .get("https://api.backblazeb2.com/b2api/v2/b2_authorize_account") .header(reqwest::header::CONTENT_TYPE, "application/json") .header(reqwest::header::AUTHORIZATION, formatted_key) .send() - .await? - .json() - .await?) + .await?; + + if response.status().is_success() { + Ok(response.json().await?) + } else { + Err(FileHostingError::BackblazeError(response.json().await?)) + } } +#[cfg(feature = "backblaze")] pub async fn get_upload_url( authorization_data: AuthorizationData, bucket_id: String, ) -> Result { - Ok(reqwest::Client::new() + let response = reqwest::Client::new() .post(&format!("{}/b2api/v2/b2_get_upload_url", authorization_data.api_url).to_string()) .header(reqwest::header::CONTENT_TYPE, "application/json") .header( @@ -61,13 +65,64 @@ pub async fn get_upload_url( authorization_data.authorization_token, ) .body( - json!({ + serde_json::json!({ "bucketId": bucket_id, }) .to_string(), ) .send() - .await? - .json() - .await?) + .await?; + + if response.status().is_success() { + Ok(response.json().await?) + } else { + Err(FileHostingError::BackblazeError(response.json().await?)) + } +} + +#[cfg(not(feature = "backblaze"))] +pub async fn authorize_account( + _key_id: String, + _application_key: String, +) -> Result { + Ok(AuthorizationData { + absolute_minimum_part_size: 5000000, + account_id: String::from("MOCK_ACCOUNT_ID"), + allowed: AuthorizationPermissions { + bucket_id: None, + bucket_name: None, + capabilities: vec![ + String::from("listKeys"), + String::from("writeKeys"), + String::from("deleteKeys"), + String::from("listAllBucketNames"), + String::from("listBuckets"), + String::from("writeBuckets"), + String::from("deleteBuckets"), + String::from("readBuckets"), + String::from("listFiles"), + String::from("readFiles"), + String::from("shareFiles"), + String::from("writeFiles"), + String::from("deleteFiles"), + ], + name_prefix: None, + }, + api_url: String::from("https://api.example.com"), + authorization_token: String::from("MOCK_AUTH_TOKEN"), + download_url: String::from("https://download.example.com"), + recommended_part_size: 100000000, + }) +} + +#[cfg(not(feature = "backblaze"))] +pub async fn get_upload_url( + _authorization_data: AuthorizationData, + _bucket_id: String, +) -> Result { + Ok(UploadUrlData { + bucket_id: String::from("MOCK_BUCKET_ID"), + upload_url: String::from("https://download.example.com"), + authorization_token: String::from("MOCK_AUTH_TOKEN"), + }) } diff --git a/src/file_hosting/delete.rs b/src/file_hosting/delete.rs index 52ff6e89..2ea54bd2 100644 --- a/src/file_hosting/delete.rs +++ b/src/file_hosting/delete.rs @@ -1,6 +1,5 @@ use crate::file_hosting::{AuthorizationData, FileHostingError}; use serde::{Deserialize, Serialize}; -use serde_json::json; #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(rename_all = "camelCase")] @@ -9,33 +8,51 @@ pub struct DeleteFileData { pub file_name: String, } +#[cfg(feature = "backblaze")] pub async fn delete_file_version( - authorization_data: AuthorizationData, - file_id: String, - file_name: String, + authorization_data: &AuthorizationData, + file_id: &str, + file_name: &str, ) -> Result { - Ok(reqwest::Client::new() - .post( - &format!( - "{}/b2api/v2/b2_delete_file_version", - authorization_data.api_url - ) - .to_string(), - ) + let response = reqwest::Client::new() + .post(&format!( + "{}/b2api/v2/b2_delete_file_version", + authorization_data.api_url + )) .header(reqwest::header::CONTENT_TYPE, "application/json") .header( reqwest::header::AUTHORIZATION, - authorization_data.authorization_token, + &authorization_data.authorization_token, ) .body( - json!({ + serde_json::json!({ "fileName": file_name, "fileId": file_id }) .to_string(), ) .send() - .await? - .json() - .await?) + .await?; + + if response.status().is_success() { + Ok(response.json().await?) + } else { + Err(FileHostingError::BackblazeError(response.json().await?)) + } +} + +#[cfg(not(feature = "backblaze"))] +pub async fn delete_file_version( + _authorization_data: &AuthorizationData, + file_id: &str, + file_name: &str, +) -> Result { + let path = std::path::Path::new(&dotenv::var("MOCK_FILE_PATH").unwrap()) + .join(file_name.replace("../", "")); + std::fs::remove_file(path)?; + + Ok(DeleteFileData { + file_id: file_id.to_string(), + file_name: file_name.to_string(), + }) } diff --git a/src/file_hosting/mod.rs b/src/file_hosting/mod.rs index cc29a34f..eca576f6 100644 --- a/src/file_hosting/mod.rs +++ b/src/file_hosting/mod.rs @@ -18,8 +18,20 @@ pub use delete::DeleteFileData; #[derive(Error, Debug)] pub enum FileHostingError { + #[cfg(feature = "backblaze")] #[error("Error while accessing the data from backblaze")] - BackblazeError(#[from] reqwest::Error), + HttpError(#[from] reqwest::Error), + + #[cfg(feature = "backblaze")] + #[error("Backblaze error: {0}")] + BackblazeError(serde_json::Value), + + #[cfg(not(feature = "backblaze"))] + #[error("File system error in file hosting: {0}")] + FileSystemError(#[from] std::io::Error), + #[cfg(not(feature = "backblaze"))] + #[error("Invalid Filename")] + InvalidFilename, } #[cfg(test)] @@ -59,18 +71,18 @@ mod tests { .await .unwrap(); let upload_data = upload_file( - upload_url_data, - "text/plain".to_string(), - "test.txt".to_string(), + &upload_url_data, + "text/plain", + "test.txt", "test file".to_string().into_bytes(), ) .await .unwrap(); delete_file_version( - authorization_data, - upload_data.file_id, - upload_data.file_name, + &authorization_data, + &upload_data.file_id, + &upload_data.file_name, ) .await .unwrap(); diff --git a/src/file_hosting/upload.rs b/src/file_hosting/upload.rs index 8d3fb232..355ededd 100644 --- a/src/file_hosting/upload.rs +++ b/src/file_hosting/upload.rs @@ -16,16 +16,20 @@ pub struct UploadFileData { pub upload_timestamp: u64, } +#[cfg(feature = "backblaze")] //Content Types found here: https://www.backblaze.com/b2/docs/content-types.html pub async fn upload_file( - url_data: UploadUrlData, - content_type: String, - file_name: String, + url_data: &UploadUrlData, + content_type: &str, + file_name: &str, file_bytes: Vec, ) -> Result { - Ok(reqwest::Client::new() + let response = reqwest::Client::new() .post(&url_data.upload_url) - .header(reqwest::header::AUTHORIZATION, url_data.authorization_token) + .header( + reqwest::header::AUTHORIZATION, + &url_data.authorization_token, + ) .header("X-Bz-File-Name", file_name) .header(reqwest::header::CONTENT_TYPE, content_type) .header(reqwest::header::CONTENT_LENGTH, file_bytes.len()) @@ -35,7 +39,37 @@ pub async fn upload_file( ) .body(file_bytes) .send() - .await? - .json() - .await?) + .await?; + + if response.status().is_success() { + Ok(response.json().await?) + } else { + Err(FileHostingError::BackblazeError(response.json().await?)) + } +} + +#[cfg(not(feature = "backblaze"))] +pub async fn upload_file( + _url_data: &UploadUrlData, + content_type: &str, + file_name: &str, + file_bytes: Vec, +) -> Result { + let path = std::path::Path::new(&dotenv::var("MOCK_FILE_PATH").unwrap()) + .join(file_name.replace("../", "")); + std::fs::create_dir_all(path.parent().ok_or(FileHostingError::InvalidFilename)?)?; + let content_sha1 = sha1::Sha1::from(&file_bytes).hexdigest(); + + std::fs::write(path, &file_bytes)?; + Ok(UploadFileData { + file_id: String::from("MOCK_FILE_ID"), + file_name: file_name.to_string(), + account_id: String::from("MOCK_ACCOUNT_ID"), + bucket_id: String::from("MOCK_BUCKET_ID"), + content_length: file_bytes.len() as u32, + content_sha1, + content_md5: None, + content_type: content_type.to_string(), + upload_timestamp: chrono::Utc::now().timestamp_millis() as u64, + }) } diff --git a/src/main.rs b/src/main.rs index 0330acbb..e66105e5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -23,7 +23,7 @@ async fn main() -> std::io::Result<()> { let client = database::connect() .await .expect("Database connection failed"); - let client_ref = web::Data::new(client.clone()); + let client_ref = client.clone(); //File Hosting Initializer let authorization_data = file_hosting::authorize_account( @@ -39,9 +39,6 @@ async fn main() -> std::io::Result<()> { .await .unwrap(); - let authorization_data_ref = web::Data::new(authorization_data); - let upload_url_data_ref = web::Data::new(upload_url_data); - // Get executable path let mut exe_path = env::current_exe()?.parent().unwrap().to_path_buf(); // Create the path to the index lock file @@ -68,10 +65,11 @@ async fn main() -> std::io::Result<()> { .wrap(Logger::default()) .wrap(Logger::new("%a %{User-Agent}i")) .data(client_ref.clone()) - .data(authorization_data_ref.clone()) - .data(upload_url_data_ref.clone()) + .data(authorization_data.clone()) + .data(upload_url_data.clone()) .service(routes::index_get) .service(routes::mod_search) + .service(routes::mod_create) .default_service(web::get().to(routes::not_found)) }) .bind(dotenv::var("BIND_ADDR").unwrap())? diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 1b2985b9..0bab794f 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -4,5 +4,6 @@ mod mods; mod not_found; pub use self::index::index_get; +pub use self::mod_creation::mod_create; pub use self::mods::mod_search; pub use self::not_found::not_found; diff --git a/src/routes/mod_creation.rs b/src/routes/mod_creation.rs index 816260d9..240bbd6f 100644 --- a/src/routes/mod_creation.rs +++ b/src/routes/mod_creation.rs @@ -26,14 +26,16 @@ pub enum CreateError { MultipartError(actix_multipart::MultipartError), #[error("Error while parsing JSON")] SerDeError(#[from] serde_json::Error), + #[error("Error while serializing BSON")] + BsonError(#[from] bson::ser::Error), #[error("Error while uploading file")] FileHostingError(#[from] FileHostingError), - #[error("Error while parsing string as UTF-8")] - InvalidUtf8Input(#[source] std::string::FromUtf8Error), #[error("{}", .0)] MissingValueError(String), #[error("Error while trying to generate random ID")] RandomIdError, + #[error("Invalid format for mod icon: {0}")] + InvalidIconFormat(String), } impl actix_web::ResponseError for CreateError { @@ -42,10 +44,11 @@ impl actix_web::ResponseError for CreateError { CreateError::EnvError(..) => StatusCode::INTERNAL_SERVER_ERROR, CreateError::DatabaseError(..) => StatusCode::INTERNAL_SERVER_ERROR, CreateError::FileHostingError(..) => StatusCode::INTERNAL_SERVER_ERROR, + CreateError::BsonError(..) => StatusCode::INTERNAL_SERVER_ERROR, CreateError::SerDeError(..) => StatusCode::BAD_REQUEST, CreateError::MultipartError(..) => StatusCode::BAD_REQUEST, - CreateError::InvalidUtf8Input(..) => StatusCode::BAD_REQUEST, CreateError::MissingValueError(..) => StatusCode::BAD_REQUEST, + CreateError::InvalidIconFormat(..) => StatusCode::BAD_REQUEST, CreateError::RandomIdError => StatusCode::INTERNAL_SERVER_ERROR, } } @@ -56,11 +59,12 @@ impl actix_web::ResponseError for CreateError { CreateError::EnvError(..) => "environment_error", CreateError::DatabaseError(..) => "database_error", CreateError::FileHostingError(..) => "file_hosting_error", + CreateError::BsonError(..) => "database_error", CreateError::SerDeError(..) => "invalid_input", CreateError::MultipartError(..) => "invalid_input", - CreateError::InvalidUtf8Input(..) => "invalid_input", CreateError::MissingValueError(..) => "invalid_input", CreateError::RandomIdError => "id_generation_error", + CreateError::InvalidIconFormat(..) => "invalid_input", }, description: &self.to_string(), }) @@ -158,33 +162,30 @@ pub async fn mod_create( mod_create_data = Some(serde_json::from_slice(&data)?); } else { let file_name = content_disposition.get_filename().ok_or_else(|| { - CreateError::MissingValueError("Missing content file name!".to_string()) + CreateError::MissingValueError("Missing content file name".to_string()) })?; - let file_extension = String::from_utf8( - content_disposition - .get_filename_ext() - .ok_or_else(|| { - CreateError::MissingValueError("Missing file extension!".to_string()) - })? - .clone() - .value, - ) - .map_err(CreateError::InvalidUtf8Input)?; + let file_extension = if let Some(last_period) = file_name.rfind('.') { + file_name.get(last_period + 1..).unwrap_or("") + } else { + return Err(CreateError::MissingValueError( + "Missing content file extension".to_string(), + )); + }; if let Some(create_data) = &mod_create_data { if name == "icon" { if let Some(ext) = get_image_content_type(file_extension) { let upload_data = upload_file( - upload_url.get_ref().clone(), + upload_url.get_ref(), ext, - format!("mods/icons/{}/{}", mod_id.0, file_name), + &format!("mods/icons/{}/{}", mod_id, file_name), data.to_vec(), ) .await?; icon_url = format!("{}/{}", cdn_url, upload_data.file_name); } else { - panic!("Invalid Icon Format!"); + return Err(CreateError::InvalidIconFormat(file_extension.to_string())); } } else if &*file_extension == "jar" { let initial_version_data = create_data @@ -210,9 +211,9 @@ pub async fn mod_create( match created_version_filter.next() { Some(created_version) => { let upload_data = upload_file( - upload_url.get_ref().clone(), - "application/java-archive".to_string(), - format!( + upload_url.get_ref(), + "application/java-archive", + &format!( "{}/{}/{}", create_data.mod_namespace.replace(".", "/"), version_data.version_number, @@ -258,21 +259,21 @@ pub async fn mod_create( let body_url = format!( "data/{}/changelogs/{}/body.md", - mod_id.0, version_id.0 + mod_id, version_id ); upload_file( - upload_url.get_ref().clone(), - "text/plain".to_string(), - body_url.clone(), + upload_url.get_ref(), + "text/plain", + &body_url, version_data.version_body.into_bytes(), ) .await?; let upload_data = upload_file( - upload_url.get_ref().clone(), - "application/java-archive".to_string(), - format!( + upload_url.get_ref(), + "application/java-archive", + &format!( "{}/{}/{}", create_data.mod_namespace.replace(".", "/"), version_data.version_number, @@ -340,12 +341,12 @@ pub async fn mod_create( } if let Some(create_data) = mod_create_data { - let body_url = format!("data/{}/body.md", mod_id.0); + let body_url = format!("data/{}/body.md", mod_id); upload_file( - upload_url.get_ref().clone(), - "text/plain".to_string(), - body_url.clone(), + upload_url.get_ref(), + "text/plain", + &body_url, create_data.mod_body.into_bytes(), ) .await?; @@ -380,8 +381,7 @@ pub async fn mod_create( wiki_url: create_data.wiki_url, }; - let serialized_mod = serde_json::to_string(&created_mod)?; - let document = Bson::from(serialized_mod) + let document = bson::to_bson(&created_mod)? .as_document() .ok_or_else(|| { CreateError::MissingValueError( @@ -396,7 +396,7 @@ pub async fn mod_create( Ok(HttpResponse::Ok().into()) } -fn get_image_content_type(extension: String) -> Option { +fn get_image_content_type(extension: &str) -> Option<&'static str> { let content_type = match &*extension { "bmp" => "image/bmp", "gif" => "image/gif", @@ -409,7 +409,7 @@ fn get_image_content_type(extension: String) -> Option { }; if content_type != "" { - Some(content_type.to_string()) + Some(content_type) } else { None }