aziz/AstralRinth
0
0
Fork 0
forked from xxxOFFxxx/AstralRinth
Code Pull Requests Activity

Create a mock file host for dev, Fix mod creation route (#38)

Browse Source 
* fix(mod-creation): fix actix server data & mod creation route

* feat(file-host): implement mock file hosting

This implements a mock file hosting system backed by the system's
filesystem.  It mirrors the API of the backblaze integration, but
puts the files directly on disk in the path specified by the
MOCK_FILE_PATH environment variable (defaults to /tmp/modrinth).

The mock file hosting is enabled by default using cargo features
to allow people to work on modrinth without access to a valid
backblaze account and setup.  To enable backblaze, specify the
cargo feature "backblaze" when running, ex. `cargo run --features
backblaze`.

* feat(file-hosting): implement basic backblaze API error handling

* fix(mod-creation): fix extension parsing, use base62 ids for paths
fix(file-hosting): reduce unnecessary allocations

* fix: fix auth with docker mongodb

* fix: fix failing checks

* fix: remove testing files
This commit is contained in:
Aeledfyr
2020-07-16 23:06:58 -05:00
committed by GitHub
parent 39b1435725
commit 95339a8338
9 changed files with 211 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.env
View File

@@ -3,11 +3,13 @@ DEBUG=true
CDN_URL=cdn.modrinth.com
MONGODB_ADDR=mongodb://localhost:27017
MONGODB_ADDR=mongodb://toor:modrinthadmin@localhost:27017
MEILISEARCH_ADDR=http://localhost:7700
BIND_ADDR=127.0.0.1:8000
MOCK_FILE_PATH=/tmp/modrinth
BACKBLAZE_KEY_ID=none
BACKBLAZE_KEY=none
BACKBLAZE_BUCKET_ID=none

6
Cargo.toml
View File

@@ -36,4 +36,8 @@ futures = "0.3.5"
futures-timer = "3.0.2"
base64 = "0.12.3"
sha1 = {version="0.6.0", features=["std"]}
sha1 = {version="0.6.0", features=["std"]}
[features]
default = []
backblaze = []

79
src/file_hosting/authorization.rs
View File

@@ -1,7 +1,5 @@
use crate::file_hosting::FileHostingError;
use base64::encode;
use serde::{Deserialize, Serialize};
use serde_json::json;
#[derive(Serialize, Deserialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
@@ -32,28 +30,34 @@ pub struct UploadUrlData {
pub authorization_token: String,
}
#[cfg(feature = "backblaze")]
pub async fn authorize_account(
key_id: String,
application_key: String,
) -> Result<AuthorizationData, FileHostingError> {
let combined_key = format!("{}:{}", key_id, application_key);
let formatted_key = format!("Basic {}", encode(combined_key));
let formatted_key = format!("Basic {}", base64::encode(combined_key));
Ok(reqwest::Client::new()
let response = reqwest::Client::new()
.get("https://api.backblazeb2.com/b2api/v2/b2_authorize_account")
.header(reqwest::header::CONTENT_TYPE, "application/json")
.header(reqwest::header::AUTHORIZATION, formatted_key)
.send()
.await?
.json()
.await?)
.await?;
if response.status().is_success() {
Ok(response.json().await?)
} else {
Err(FileHostingError::BackblazeError(response.json().await?))
}
}
#[cfg(feature = "backblaze")]
pub async fn get_upload_url(
authorization_data: AuthorizationData,
bucket_id: String,
) -> Result<UploadUrlData, FileHostingError> {
Ok(reqwest::Client::new()
let response = reqwest::Client::new()
.post(&format!("{}/b2api/v2/b2_get_upload_url", authorization_data.api_url).to_string())
.header(reqwest::header::CONTENT_TYPE, "application/json")
.header(
@@ -61,13 +65,64 @@ pub async fn get_upload_url(
authorization_data.authorization_token,
)
.body(
json!({
serde_json::json!({
"bucketId": bucket_id,
})
.to_string(),
)
.send()
.await?
.json()
.await?)
.await?;
if response.status().is_success() {
Ok(response.json().await?)
} else {
Err(FileHostingError::BackblazeError(response.json().await?))
}
}
#[cfg(not(feature = "backblaze"))]
pub async fn authorize_account(
_key_id: String,
_application_key: String,
) -> Result<AuthorizationData, FileHostingError> {
Ok(AuthorizationData {
absolute_minimum_part_size: 5000000,
account_id: String::from("MOCK_ACCOUNT_ID"),
allowed: AuthorizationPermissions {
bucket_id: None,
bucket_name: None,
capabilities: vec![
String::from("listKeys"),
String::from("writeKeys"),
String::from("deleteKeys"),
String::from("listAllBucketNames"),
String::from("listBuckets"),
String::from("writeBuckets"),
String::from("deleteBuckets"),
String::from("readBuckets"),
String::from("listFiles"),
String::from("readFiles"),
String::from("shareFiles"),
String::from("writeFiles"),
String::from("deleteFiles"),
],
name_prefix: None,
},
api_url: String::from("https://api.example.com"),
authorization_token: String::from("MOCK_AUTH_TOKEN"),
download_url: String::from("https://download.example.com"),
recommended_part_size: 100000000,
})
}
#[cfg(not(feature = "backblaze"))]
pub async fn get_upload_url(
_authorization_data: AuthorizationData,
_bucket_id: String,
) -> Result<UploadUrlData, FileHostingError> {
Ok(UploadUrlData {
bucket_id: String::from("MOCK_BUCKET_ID"),
upload_url: String::from("https://download.example.com"),
authorization_token: String::from("MOCK_AUTH_TOKEN"),
})
}

51
src/file_hosting/delete.rs
View File

@@ -1,6 +1,5 @@
use crate::file_hosting::{AuthorizationData, FileHostingError};
use serde::{Deserialize, Serialize};
use serde_json::json;
#[derive(Serialize, Deserialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
@@ -9,33 +8,51 @@ pub struct DeleteFileData {
pub file_name: String,
}
#[cfg(feature = "backblaze")]
pub async fn delete_file_version(
authorization_data: AuthorizationData,
file_id: String,
file_name: String,
authorization_data: &AuthorizationData,
file_id: &str,
file_name: &str,
) -> Result<DeleteFileData, FileHostingError> {
Ok(reqwest::Client::new()
.post(
&format!(
"{}/b2api/v2/b2_delete_file_version",
authorization_data.api_url
)
.to_string(),
)
let response = reqwest::Client::new()
.post(&format!(
"{}/b2api/v2/b2_delete_file_version",
authorization_data.api_url
))
.header(reqwest::header::CONTENT_TYPE, "application/json")
.header(
reqwest::header::AUTHORIZATION,
authorization_data.authorization_token,
&authorization_data.authorization_token,
)
.body(
json!({
serde_json::json!({
"fileName": file_name,
"fileId": file_id
})
.to_string(),
)
.send()
.await?
.json()
.await?)
.await?;
if response.status().is_success() {
Ok(response.json().await?)
} else {
Err(FileHostingError::BackblazeError(response.json().await?))
}
}
#[cfg(not(feature = "backblaze"))]
pub async fn delete_file_version(
_authorization_data: &AuthorizationData,
file_id: &str,
file_name: &str,
) -> Result<DeleteFileData, FileHostingError> {
let path = std::path::Path::new(&dotenv::var("MOCK_FILE_PATH").unwrap())
.join(file_name.replace("../", ""));
std::fs::remove_file(path)?;
Ok(DeleteFileData {
file_id: file_id.to_string(),
file_name: file_name.to_string(),
})
}

26
src/file_hosting/mod.rs
View File

@@ -18,8 +18,20 @@ pub use delete::DeleteFileData;
#[derive(Error, Debug)]
pub enum FileHostingError {
#[cfg(feature = "backblaze")]
#[error("Error while accessing the data from backblaze")]
BackblazeError(#[from] reqwest::Error),
HttpError(#[from] reqwest::Error),
#[cfg(feature = "backblaze")]
#[error("Backblaze error: {0}")]
BackblazeError(serde_json::Value),
#[cfg(not(feature = "backblaze"))]
#[error("File system error in file hosting: {0}")]
FileSystemError(#[from] std::io::Error),
#[cfg(not(feature = "backblaze"))]
#[error("Invalid Filename")]
InvalidFilename,
}
#[cfg(test)]
@@ -59,18 +71,18 @@ mod tests {
.await
.unwrap();
let upload_data = upload_file(
upload_url_data,
"text/plain".to_string(),
"test.txt".to_string(),
&upload_url_data,
"text/plain",
"test.txt",
"test file".to_string().into_bytes(),
)
.await
.unwrap();
delete_file_version(
authorization_data,
upload_data.file_id,
upload_data.file_name,
&authorization_data,
&upload_data.file_id,
&upload_data.file_name,
)
.await
.unwrap();

50
src/file_hosting/upload.rs
View File

@@ -16,16 +16,20 @@ pub struct UploadFileData {
pub upload_timestamp: u64,
}
#[cfg(feature = "backblaze")]
//Content Types found here: https://www.backblaze.com/b2/docs/content-types.html
pub async fn upload_file(
url_data: UploadUrlData,
content_type: String,
file_name: String,
url_data: &UploadUrlData,
content_type: &str,
file_name: &str,
file_bytes: Vec<u8>,
) -> Result<UploadFileData, FileHostingError> {
Ok(reqwest::Client::new()
let response = reqwest::Client::new()
.post(&url_data.upload_url)
.header(reqwest::header::AUTHORIZATION, url_data.authorization_token)
.header(
reqwest::header::AUTHORIZATION,
&url_data.authorization_token,
)
.header("X-Bz-File-Name", file_name)
.header(reqwest::header::CONTENT_TYPE, content_type)
.header(reqwest::header::CONTENT_LENGTH, file_bytes.len())
@@ -35,7 +39,37 @@ pub async fn upload_file(
)
.body(file_bytes)
.send()
.await?
.json()
.await?)
.await?;
if response.status().is_success() {
Ok(response.json().await?)
} else {
Err(FileHostingError::BackblazeError(response.json().await?))
}
}
#[cfg(not(feature = "backblaze"))]
pub async fn upload_file(
_url_data: &UploadUrlData,
content_type: &str,
file_name: &str,
file_bytes: Vec<u8>,
) -> Result<UploadFileData, FileHostingError> {
let path = std::path::Path::new(&dotenv::var("MOCK_FILE_PATH").unwrap())
.join(file_name.replace("../", ""));
std::fs::create_dir_all(path.parent().ok_or(FileHostingError::InvalidFilename)?)?;
let content_sha1 = sha1::Sha1::from(&file_bytes).hexdigest();
std::fs::write(path, &file_bytes)?;
Ok(UploadFileData {
file_id: String::from("MOCK_FILE_ID"),
file_name: file_name.to_string(),
account_id: String::from("MOCK_ACCOUNT_ID"),
bucket_id: String::from("MOCK_BUCKET_ID"),
content_length: file_bytes.len() as u32,
content_sha1,
content_md5: None,
content_type: content_type.to_string(),
upload_timestamp: chrono::Utc::now().timestamp_millis() as u64,
})
}

10
src/main.rs
View File

@@ -23,7 +23,7 @@ async fn main() -> std::io::Result<()> {
let client = database::connect()
.await
.expect("Database connection failed");
let client_ref = web::Data::new(client.clone());
let client_ref = client.clone();
//File Hosting Initializer
let authorization_data = file_hosting::authorize_account(
@@ -39,9 +39,6 @@ async fn main() -> std::io::Result<()> {
.await
.unwrap();
let authorization_data_ref = web::Data::new(authorization_data);
let upload_url_data_ref = web::Data::new(upload_url_data);
// Get executable path
let mut exe_path = env::current_exe()?.parent().unwrap().to_path_buf();
// Create the path to the index lock file
@@ -68,10 +65,11 @@ async fn main() -> std::io::Result<()> {
.wrap(Logger::default())
.wrap(Logger::new("%a %{User-Agent}i"))
.data(client_ref.clone())
.data(authorization_data_ref.clone())
.data(upload_url_data_ref.clone())
.data(authorization_data.clone())
.data(upload_url_data.clone())
.service(routes::index_get)
.service(routes::mod_search)
.service(routes::mod_create)
.default_service(web::get().to(routes::not_found))
})
.bind(dotenv::var("BIND_ADDR").unwrap())?

1
src/routes/mod.rs
View File

@@ -4,5 +4,6 @@ mod mods;
mod not_found;
pub use self::index::index_get;
pub use self::mod_creation::mod_create;
pub use self::mods::mod_search;
pub use self::not_found::not_found;

72
src/routes/mod_creation.rs
View File

@@ -26,14 +26,16 @@ pub enum CreateError {
MultipartError(actix_multipart::MultipartError),
#[error("Error while parsing JSON")]
SerDeError(#[from] serde_json::Error),
#[error("Error while serializing BSON")]
BsonError(#[from] bson::ser::Error),
#[error("Error while uploading file")]
FileHostingError(#[from] FileHostingError),
#[error("Error while parsing string as UTF-8")]
InvalidUtf8Input(#[source] std::string::FromUtf8Error),
#[error("{}", .0)]
MissingValueError(String),
#[error("Error while trying to generate random ID")]
RandomIdError,
#[error("Invalid format for mod icon: {0}")]
InvalidIconFormat(String),
}
impl actix_web::ResponseError for CreateError {
@@ -42,10 +44,11 @@ impl actix_web::ResponseError for CreateError {
CreateError::EnvError(..) => StatusCode::INTERNAL_SERVER_ERROR,
CreateError::DatabaseError(..) => StatusCode::INTERNAL_SERVER_ERROR,
CreateError::FileHostingError(..) => StatusCode::INTERNAL_SERVER_ERROR,
CreateError::BsonError(..) => StatusCode::INTERNAL_SERVER_ERROR,
CreateError::SerDeError(..) => StatusCode::BAD_REQUEST,
CreateError::MultipartError(..) => StatusCode::BAD_REQUEST,
CreateError::InvalidUtf8Input(..) => StatusCode::BAD_REQUEST,
CreateError::MissingValueError(..) => StatusCode::BAD_REQUEST,
CreateError::InvalidIconFormat(..) => StatusCode::BAD_REQUEST,
CreateError::RandomIdError => StatusCode::INTERNAL_SERVER_ERROR,
}
}
@@ -56,11 +59,12 @@ impl actix_web::ResponseError for CreateError {
CreateError::EnvError(..) => "environment_error",
CreateError::DatabaseError(..) => "database_error",
CreateError::FileHostingError(..) => "file_hosting_error",
CreateError::BsonError(..) => "database_error",
CreateError::SerDeError(..) => "invalid_input",
CreateError::MultipartError(..) => "invalid_input",
CreateError::InvalidUtf8Input(..) => "invalid_input",
CreateError::MissingValueError(..) => "invalid_input",
CreateError::RandomIdError => "id_generation_error",
CreateError::InvalidIconFormat(..) => "invalid_input",
},
description: &self.to_string(),
})
@@ -158,33 +162,30 @@ pub async fn mod_create(
mod_create_data = Some(serde_json::from_slice(&data)?);
} else {
let file_name = content_disposition.get_filename().ok_or_else(|| {
CreateError::MissingValueError("Missing content file name!".to_string())
CreateError::MissingValueError("Missing content file name".to_string())
})?;
let file_extension = String::from_utf8(
content_disposition
.get_filename_ext()
.ok_or_else(|| {
CreateError::MissingValueError("Missing file extension!".to_string())
})?
.clone()
.value,
)
.map_err(CreateError::InvalidUtf8Input)?;
let file_extension = if let Some(last_period) = file_name.rfind('.') {
file_name.get(last_period + 1..).unwrap_or("")
} else {
return Err(CreateError::MissingValueError(
"Missing content file extension".to_string(),
));
};
if let Some(create_data) = &mod_create_data {
if name == "icon" {
if let Some(ext) = get_image_content_type(file_extension) {
let upload_data = upload_file(
upload_url.get_ref().clone(),
upload_url.get_ref(),
ext,
format!("mods/icons/{}/{}", mod_id.0, file_name),
&format!("mods/icons/{}/{}", mod_id, file_name),
data.to_vec(),
)
.await?;
icon_url = format!("{}/{}", cdn_url, upload_data.file_name);
} else {
panic!("Invalid Icon Format!");
return Err(CreateError::InvalidIconFormat(file_extension.to_string()));
}
} else if &*file_extension == "jar" {
let initial_version_data = create_data
@@ -210,9 +211,9 @@ pub async fn mod_create(
match created_version_filter.next() {
Some(created_version) => {
let upload_data = upload_file(
upload_url.get_ref().clone(),
"application/java-archive".to_string(),
format!(
upload_url.get_ref(),
"application/java-archive",
&format!(
"{}/{}/{}",
create_data.mod_namespace.replace(".", "/"),
version_data.version_number,
@@ -258,21 +259,21 @@ pub async fn mod_create(
let body_url = format!(
"data/{}/changelogs/{}/body.md",
mod_id.0, version_id.0
mod_id, version_id
);
upload_file(
upload_url.get_ref().clone(),
"text/plain".to_string(),
body_url.clone(),
upload_url.get_ref(),
"text/plain",
&body_url,
version_data.version_body.into_bytes(),
)
.await?;
let upload_data = upload_file(
upload_url.get_ref().clone(),
"application/java-archive".to_string(),
format!(
upload_url.get_ref(),
"application/java-archive",
&format!(
"{}/{}/{}",
create_data.mod_namespace.replace(".", "/"),
version_data.version_number,
@@ -340,12 +341,12 @@ pub async fn mod_create(
}
if let Some(create_data) = mod_create_data {
let body_url = format!("data/{}/body.md", mod_id.0);
let body_url = format!("data/{}/body.md", mod_id);
upload_file(
upload_url.get_ref().clone(),
"text/plain".to_string(),
body_url.clone(),
upload_url.get_ref(),
"text/plain",
&body_url,
create_data.mod_body.into_bytes(),
)
.await?;
@@ -380,8 +381,7 @@ pub async fn mod_create(
wiki_url: create_data.wiki_url,
};
let serialized_mod = serde_json::to_string(&created_mod)?;
let document = Bson::from(serialized_mod)
let document = bson::to_bson(&created_mod)?
.as_document()
.ok_or_else(|| {
CreateError::MissingValueError(
@@ -396,7 +396,7 @@ pub async fn mod_create(
Ok(HttpResponse::Ok().into())
}
fn get_image_content_type(extension: String) -> Option<String> {
fn get_image_content_type(extension: &str) -> Option<&'static str> {
let content_type = match &*extension {
"bmp" => "image/bmp",
"gif" => "image/gif",
@@ -409,7 +409,7 @@ fn get_image_content_type(extension: String) -> Option<String> {
};
if content_type != "" {
Some(content_type.to_string())
Some(content_type)
} else {
None
}