eightequal/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Create a mock file host for dev, Fix mod creation route (#38)

Browse Source 
* fix(mod-creation): fix actix server data & mod creation route

* feat(file-host): implement mock file hosting

This implements a mock file hosting system backed by the system's
filesystem.  It mirrors the API of the backblaze integration, but
puts the files directly on disk in the path specified by the
MOCK_FILE_PATH environment variable (defaults to /tmp/modrinth).

The mock file hosting is enabled by default using cargo features
to allow people to work on modrinth without access to a valid
backblaze account and setup.  To enable backblaze, specify the
cargo feature "backblaze" when running, ex. `cargo run --features
backblaze`.

* feat(file-hosting): implement basic backblaze API error handling

* fix(mod-creation): fix extension parsing, use base62 ids for paths
fix(file-hosting): reduce unnecessary allocations

* fix: fix auth with docker mongodb

* fix: fix failing checks

* fix: remove testing files
This commit is contained in:
Aeledfyr
2020-07-16 23:06:58 -05:00
committed by GitHub
parent 39b1435725
commit 95339a8338
9 changed files with 211 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
src/file_hosting/authorization.rs
View File

@@ -1,7 +1,5 @@
use crate::file_hosting::FileHostingError;
use base64::encode;
use serde::{Deserialize, Serialize};
use serde_json::json;
#[derive(Serialize, Deserialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
@@ -32,28 +30,34 @@ pub struct UploadUrlData {
pub authorization_token: String,
}
#[cfg(feature = "backblaze")]
pub async fn authorize_account(
key_id: String,
application_key: String,
) -> Result<AuthorizationData, FileHostingError> {
let combined_key = format!("{}:{}", key_id, application_key);
let formatted_key = format!("Basic {}", encode(combined_key));
let formatted_key = format!("Basic {}", base64::encode(combined_key));
Ok(reqwest::Client::new()
let response = reqwest::Client::new()
.get("https://api.backblazeb2.com/b2api/v2/b2_authorize_account")
.header(reqwest::header::CONTENT_TYPE, "application/json")
.header(reqwest::header::AUTHORIZATION, formatted_key)
.send()
.await?
.json()
.await?)
.await?;
if response.status().is_success() {
Ok(response.json().await?)
} else {
Err(FileHostingError::BackblazeError(response.json().await?))
}
}
#[cfg(feature = "backblaze")]
pub async fn get_upload_url(
authorization_data: AuthorizationData,
bucket_id: String,
) -> Result<UploadUrlData, FileHostingError> {
Ok(reqwest::Client::new()
let response = reqwest::Client::new()
.post(&format!("{}/b2api/v2/b2_get_upload_url", authorization_data.api_url).to_string())
.header(reqwest::header::CONTENT_TYPE, "application/json")
.header(
@@ -61,13 +65,64 @@ pub async fn get_upload_url(
authorization_data.authorization_token,
)
.body(
json!({
serde_json::json!({
"bucketId": bucket_id,
})
.to_string(),
)
.send()
.await?
.json()
.await?)
.await?;
if response.status().is_success() {
Ok(response.json().await?)
} else {
Err(FileHostingError::BackblazeError(response.json().await?))
}
}
#[cfg(not(feature = "backblaze"))]
pub async fn authorize_account(
_key_id: String,
_application_key: String,
) -> Result<AuthorizationData, FileHostingError> {
Ok(AuthorizationData {
absolute_minimum_part_size: 5000000,
account_id: String::from("MOCK_ACCOUNT_ID"),
allowed: AuthorizationPermissions {
bucket_id: None,
bucket_name: None,
capabilities: vec![
String::from("listKeys"),
String::from("writeKeys"),
String::from("deleteKeys"),
String::from("listAllBucketNames"),
String::from("listBuckets"),
String::from("writeBuckets"),
String::from("deleteBuckets"),
String::from("readBuckets"),
String::from("listFiles"),
String::from("readFiles"),
String::from("shareFiles"),
String::from("writeFiles"),
String::from("deleteFiles"),
],
name_prefix: None,
},
api_url: String::from("https://api.example.com"),
authorization_token: String::from("MOCK_AUTH_TOKEN"),
download_url: String::from("https://download.example.com"),
recommended_part_size: 100000000,
})
}
#[cfg(not(feature = "backblaze"))]
pub async fn get_upload_url(
_authorization_data: AuthorizationData,
_bucket_id: String,
) -> Result<UploadUrlData, FileHostingError> {
Ok(UploadUrlData {
bucket_id: String::from("MOCK_BUCKET_ID"),
upload_url: String::from("https://download.example.com"),
authorization_token: String::from("MOCK_AUTH_TOKEN"),
})
}

51
src/file_hosting/delete.rs
View File

@@ -1,6 +1,5 @@
use crate::file_hosting::{AuthorizationData, FileHostingError};
use serde::{Deserialize, Serialize};
use serde_json::json;
#[derive(Serialize, Deserialize, Debug, Clone)]
#[serde(rename_all = "camelCase")]
@@ -9,33 +8,51 @@ pub struct DeleteFileData {
pub file_name: String,
}
#[cfg(feature = "backblaze")]
pub async fn delete_file_version(
authorization_data: AuthorizationData,
file_id: String,
file_name: String,
authorization_data: &AuthorizationData,
file_id: &str,
file_name: &str,
) -> Result<DeleteFileData, FileHostingError> {
Ok(reqwest::Client::new()
.post(
&format!(
"{}/b2api/v2/b2_delete_file_version",
authorization_data.api_url
)
.to_string(),
)
let response = reqwest::Client::new()
.post(&format!(
"{}/b2api/v2/b2_delete_file_version",
authorization_data.api_url
))
.header(reqwest::header::CONTENT_TYPE, "application/json")
.header(
reqwest::header::AUTHORIZATION,
authorization_data.authorization_token,
&authorization_data.authorization_token,
)
.body(
json!({
serde_json::json!({
"fileName": file_name,
"fileId": file_id
})
.to_string(),
)
.send()
.await?
.json()
.await?)
.await?;
if response.status().is_success() {
Ok(response.json().await?)
} else {
Err(FileHostingError::BackblazeError(response.json().await?))
}
}
#[cfg(not(feature = "backblaze"))]
pub async fn delete_file_version(
_authorization_data: &AuthorizationData,
file_id: &str,
file_name: &str,
) -> Result<DeleteFileData, FileHostingError> {
let path = std::path::Path::new(&dotenv::var("MOCK_FILE_PATH").unwrap())
.join(file_name.replace("../", ""));
std::fs::remove_file(path)?;
Ok(DeleteFileData {
file_id: file_id.to_string(),
file_name: file_name.to_string(),
})
}

26
src/file_hosting/mod.rs
View File

@@ -18,8 +18,20 @@ pub use delete::DeleteFileData;
#[derive(Error, Debug)]
pub enum FileHostingError {
#[cfg(feature = "backblaze")]
#[error("Error while accessing the data from backblaze")]
BackblazeError(#[from] reqwest::Error),
HttpError(#[from] reqwest::Error),
#[cfg(feature = "backblaze")]
#[error("Backblaze error: {0}")]
BackblazeError(serde_json::Value),
#[cfg(not(feature = "backblaze"))]
#[error("File system error in file hosting: {0}")]
FileSystemError(#[from] std::io::Error),
#[cfg(not(feature = "backblaze"))]
#[error("Invalid Filename")]
InvalidFilename,
}
#[cfg(test)]
@@ -59,18 +71,18 @@ mod tests {
.await
.unwrap();
let upload_data = upload_file(
upload_url_data,
"text/plain".to_string(),
"test.txt".to_string(),
&upload_url_data,
"text/plain",
"test.txt",
"test file".to_string().into_bytes(),
)
.await
.unwrap();
delete_file_version(
authorization_data,
upload_data.file_id,
upload_data.file_name,
&authorization_data,
&upload_data.file_id,
&upload_data.file_name,
)
.await
.unwrap();

50
src/file_hosting/upload.rs
View File

@@ -16,16 +16,20 @@ pub struct UploadFileData {
pub upload_timestamp: u64,
}
#[cfg(feature = "backblaze")]
//Content Types found here: https://www.backblaze.com/b2/docs/content-types.html
pub async fn upload_file(
url_data: UploadUrlData,
content_type: String,
file_name: String,
url_data: &UploadUrlData,
content_type: &str,
file_name: &str,
file_bytes: Vec<u8>,
) -> Result<UploadFileData, FileHostingError> {
Ok(reqwest::Client::new()
let response = reqwest::Client::new()
.post(&url_data.upload_url)
.header(reqwest::header::AUTHORIZATION, url_data.authorization_token)
.header(
reqwest::header::AUTHORIZATION,
&url_data.authorization_token,
)
.header("X-Bz-File-Name", file_name)
.header(reqwest::header::CONTENT_TYPE, content_type)
.header(reqwest::header::CONTENT_LENGTH, file_bytes.len())
@@ -35,7 +39,37 @@ pub async fn upload_file(
)
.body(file_bytes)
.send()
.await?
.json()
.await?)
.await?;
if response.status().is_success() {
Ok(response.json().await?)
} else {
Err(FileHostingError::BackblazeError(response.json().await?))
}
}
#[cfg(not(feature = "backblaze"))]
pub async fn upload_file(
_url_data: &UploadUrlData,
content_type: &str,
file_name: &str,
file_bytes: Vec<u8>,
) -> Result<UploadFileData, FileHostingError> {
let path = std::path::Path::new(&dotenv::var("MOCK_FILE_PATH").unwrap())
.join(file_name.replace("../", ""));
std::fs::create_dir_all(path.parent().ok_or(FileHostingError::InvalidFilename)?)?;
let content_sha1 = sha1::Sha1::from(&file_bytes).hexdigest();
std::fs::write(path, &file_bytes)?;
Ok(UploadFileData {
file_id: String::from("MOCK_FILE_ID"),
file_name: file_name.to_string(),
account_id: String::from("MOCK_ACCOUNT_ID"),
bucket_id: String::from("MOCK_BUCKET_ID"),
content_length: file_bytes.len() as u32,
content_sha1,
content_md5: None,
content_type: content_type.to_string(),
upload_timestamp: chrono::Utc::now().timestamp_millis() as u64,
})
}