tomasalias/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Move validators to seperate thread, other fixes (#253)

Browse Source 
* Move validators to seperate thread, other fixes

* Update rust version in Dockerfile

* Fix notifs not working

* Fix pack validator not enforcing files
This commit is contained in:
Geometrically
2021-11-13 15:46:08 -07:00
committed by GitHub
parent 13187de97d
commit 7f791d4919
19 changed files with 257 additions and 196 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/database/models/categories.rs
View File

@@ -14,6 +14,7 @@ pub struct Loader {
pub supported_project_types: Vec<String>,
}
#[derive(Clone)]
pub struct GameVersion {
pub id: GameVersionId,
pub version: String,

4
src/database/models/notification_item.rs
View File

@@ -123,10 +123,8 @@ impl Notification {
let (notifications, actions) = futures::join!(
sqlx::query!(
"
SELECT n.user_id, n.title, n.text, n.link, n.created, n.read, n.type notification_type,
ARRAY_AGG(DISTINCT na.id || ', ' || na.title || ', ' || na.action_route || ', ' || na.action_route_method) actions
SELECT n.user_id, n.title, n.text, n.link, n.created, n.read, n.type notification_type
FROM notifications n
LEFT OUTER JOIN notifications_actions na on n.id = na.notification_id
WHERE n.id = $1
GROUP BY n.id, n.user_id;
",

6
src/database/models/version_item.rs
View File

@@ -662,7 +662,7 @@ impl Version {
for hash in hashes? {
let entry = hashes_map
.entry(FileId(hash.file_id))
.or_insert(HashMap::new());
.or_insert_with(HashMap::new);
if let Some(raw_hash) = hash.hash {
entry.insert(hash.algorithm, raw_hash.into_bytes());
@@ -695,8 +695,8 @@ impl Version {
dependencies: dependencies?
.into_iter()
.map(|x| QueryDependency {
project_id: x.mod_dependency_id.map(|x| ProjectId(x)),
version_id: x.dependency_id.map(|x| VersionId(x)),
project_id: x.mod_dependency_id.map(ProjectId),
version_id: x.dependency_id.map(VersionId),
dependency_type: x.dependency_type,
})
.collect(),

4
src/health/pod.rs
View File

@@ -10,8 +10,8 @@ pub struct PodInfo {
impl PodInfo {
pub fn new() -> Self {
Self {
pod_name: dotenv::var("POD_NAME").unwrap_or("DEV".to_string()),
node_name: dotenv::var("NODE_NAME").unwrap_or("self-hosted".to_string()),
pod_name: dotenv::var("POD_NAME").unwrap_or_else(|_| "DEV".to_string()),
node_name: dotenv::var("NODE_NAME").unwrap_or_else(|_| "self-hosted".to_string()),
pod_id: Arc::new(RwLock::new(None)),
}
}

2
src/health/scheduler.rs
View File

@@ -114,7 +114,7 @@ where
fn call(&mut self, req: ServiceRequest) -> Self::Future {
// The request has started.
let pattern_or_path = req.match_pattern().unwrap_or("unknown".to_string());
let pattern_or_path = req.match_pattern().unwrap_or_else(|| "unknown".to_string());
let counter = self
.counters
.current_requests

2
src/routes/mod.rs
View File

@@ -129,7 +129,7 @@ pub fn teams_config(cfg: &mut web::ServiceConfig) {
pub fn notifications_config(cfg: &mut web::ServiceConfig) {
cfg.service(notifications::notifications_get);
cfg.service(notifications::notification_delete);
cfg.service(notifications::notifications_delete);
cfg.service(
web::scope("notification")

2
src/routes/project_creation.rs
View File

@@ -510,7 +510,7 @@ pub async fn project_create_inner(
&*project_create_data.project_type,
version_data.loaders.clone(),
version_data.game_versions.clone(),
&all_game_versions,
all_game_versions.clone(),
false,
&mut transaction,
)

22
src/routes/projects.rs
View File

@@ -373,7 +373,7 @@ pub async fn project_edit(
}
}
let status_id = database::models::StatusId::get_id(&status, &mut *transaction)
let status_id = database::models::StatusId::get_id(status, &mut *transaction)
.await?
.ok_or_else(|| {
ApiError::InvalidInputError(
@@ -423,17 +423,15 @@ pub async fn project_edit(
.await?;
for category in categories {
let category_id = database::models::categories::Category::get_id(
&category,
&mut *transaction,
)
.await?
.ok_or_else(|| {
ApiError::InvalidInputError(format!(
"Category {} does not exist.",
category.clone()
))
})?;
let category_id =
database::models::categories::Category::get_id(category, &mut *transaction)
.await?
.ok_or_else(|| {
ApiError::InvalidInputError(format!(
"Category {} does not exist.",
category.clone()
))
})?;
sqlx::query!(
"

8
src/routes/teams.rs
View File

@@ -218,7 +218,9 @@ pub async fn add_team_member(
let result = sqlx::query!(
"
SELECT m.title, m.id FROM mods m
SELECT m.title title, m.id id, pt.name project_type
FROM mods m
INNER JOIN project_types pt ON pt.id = m.project_type
WHERE m.team_id = $1
",
team_id as crate::database::models::ids::TeamId
@@ -234,7 +236,7 @@ pub async fn add_team_member(
"Team invite from {} to join the team for project {}",
current_user.username, result.title
),
link: format!("project/{}", ProjectId(result.id as u64)),
link: format!("/{}/{}", result.project_type, ProjectId(result.id as u64)),
actions: vec![
NotificationActionBuilder {
title: "Accept".to_string(),
@@ -361,7 +363,7 @@ pub async fn transfer_ownership(
TeamMember::edit_team_member(
id.into(),
current_user.id.into(),
None,
Some(Permissions::ALL),
Some(crate::models::teams::DEFAULT_ROLE.to_string()),
None,
&mut transaction,

3
src/routes/users.rs
View File

@@ -323,8 +323,7 @@ pub async fn user_icon_edit(
}
let bytes =
read_from_payload(&mut payload, 262144, "Icons must be smaller than 256KiB")
.await?;
read_from_payload(&mut payload, 2097152, "Icons must be smaller than 2MiB").await?;
let upload_data = file_host
.upload_file(

26
src/routes/version_creation.rs
View File

@@ -288,7 +288,7 @@ async fn version_create_inner(
&*project_type,
version_data.loaders,
version_data.game_versions,
&all_game_versions,
all_game_versions.clone(),
false,
&mut transaction,
)
@@ -308,8 +308,10 @@ async fn version_create_inner(
let result = sqlx::query!(
"
SELECT m.title FROM mods m
WHERE id = $1
SELECT m.title title, pt.name project_type
FROM mods m
INNER JOIN project_types pt ON pt.id = m.project_type
WHERE m.id = $1
",
builder.project_id as crate::database::models::ids::ProjectId
)
@@ -344,7 +346,10 @@ async fn version_create_inner(
result.title,
version_data.version_number.clone()
),
link: format!("project/{}/version/{}", project_id, version_id),
link: format!(
"/{}/{}/version/{}",
result.project_type, project_id, version_id
),
actions: vec![],
}
.insert_many(users, &mut *transaction)
@@ -544,7 +549,7 @@ async fn upload_file_to_version_inner(
.into_iter()
.map(GameVersion)
.collect(),
&all_game_versions,
all_game_versions.clone(),
true,
&mut transaction,
)
@@ -579,7 +584,7 @@ pub async fn upload_file(
project_type: &str,
loaders: Vec<Loader>,
game_versions: Vec<GameVersion>,
all_game_versions: &[models::categories::GameVersion],
all_game_versions: Vec<models::categories::GameVersion>,
ignore_primary: bool,
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<(), CreateError> {
@@ -614,13 +619,14 @@ pub async fn upload_file(
}
let validation_result = validate_file(
&data,
file_extension,
project_type,
data.clone().into(),
file_extension.to_string(),
project_type.to_string(),
loaders,
game_versions,
all_game_versions,
)?;
)
.await?;
let upload_data = file_host
.upload_file(

4
src/routes/versions.rs
View File

@@ -259,8 +259,8 @@ pub async fn version_edit(
let builders = dependencies
.iter()
.map(|x| database::models::version_item::DependencyBuilder {
project_id: x.project_id.clone().map(|x| x.into()),
version_id: x.version_id.clone().map(|x| x.into()),
project_id: x.project_id.map(|x| x.into()),
version_id: x.version_id.map(|x| x.into()),
dependency_type: x.dependency_type.to_string(),
})
.collect::<Vec<database::models::version_item::DependencyBuilder>>();

6
src/search/indexing/mod.rs
View File

@@ -80,7 +80,7 @@ async fn update_index_helper<'a>(
name: &'static str,
rule: &'static str,
) -> Result<Index<'a>, IndexingError> {
update_index(&client, name, {
update_index(client, name, {
let mut rules = default_rules();
rules.push_back(rule);
rules.into()
@@ -159,11 +159,11 @@ async fn add_to_index(index: Index<'_>, mods: &[UploadSearchProject]) -> Result<
async fn create_and_add_to_index<'a>(
client: &'a Client<'a>,
projects: &'a Vec<UploadSearchProject>,
projects: &'a [UploadSearchProject],
name: &'static str,
rule: &'static str,
) -> Result<(), IndexingError> {
let index = create_index(&client, name, || {
let index = create_index(client, name, || {
let mut relevance_rules = default_rules();
relevance_rules.push_back(rule);
relevance_rules.into()

4
src/validate/fabric.rs
View File

@@ -28,7 +28,7 @@ impl super::Validator for FabricValidator {
fn validate(
&self,
archive: &mut ZipArchive<Cursor<&[u8]>>,
archive: &mut ZipArchive<Cursor<bytes::Bytes>>,
) -> Result<ValidationResult, ValidationError> {
archive.by_name("fabric.mod.json").map_err(|_| {
ValidationError::InvalidInputError("No fabric.mod.json present for Fabric file.".into())
@@ -39,7 +39,7 @@ impl super::Validator for FabricValidator {
.any(|name| name.ends_with("refmap.json") || name.ends_with(".class"))
{
return Ok(ValidationResult::Warning(
"Fabric mod file is a source file!".into(),
"Fabric mod file is a source file!",
));
}

8
src/validate/forge.rs
View File

@@ -28,7 +28,7 @@ impl super::Validator for ForgeValidator {
fn validate(
&self,
archive: &mut ZipArchive<Cursor<&[u8]>>,
archive: &mut ZipArchive<Cursor<bytes::Bytes>>,
) -> Result<ValidationResult, ValidationError> {
archive.by_name("META-INF/mods.toml").map_err(|_| {
ValidationError::InvalidInputError("No mods.toml present for Forge file.".into())
@@ -36,7 +36,7 @@ impl super::Validator for ForgeValidator {
if !archive.file_names().any(|name| name.ends_with(".class")) {
return Ok(ValidationResult::Warning(
"Forge mod file is a source file!".into(),
"Forge mod file is a source file!",
));
}
@@ -71,7 +71,7 @@ impl super::Validator for LegacyForgeValidator {
fn validate(
&self,
archive: &mut ZipArchive<Cursor<&[u8]>>,
archive: &mut ZipArchive<Cursor<bytes::Bytes>>,
) -> Result<ValidationResult, ValidationError> {
archive.by_name("mcmod.info").map_err(|_| {
ValidationError::InvalidInputError("No mcmod.info present for Forge file.".into())
@@ -79,7 +79,7 @@ impl super::Validator for LegacyForgeValidator {
if !archive.file_names().any(|name| name.ends_with(".class")) {
return Ok(ValidationResult::Warning(
"Forge mod file is a source file!".into(),
"Forge mod file is a source file!",
));
}

78
src/validate/mod.rs
View File

@@ -21,6 +21,8 @@ pub enum ValidationError {
SerdeError(#[from] serde_json::Error),
#[error("Invalid Input: {0}")]
InvalidInputError(std::borrow::Cow<'static, str>),
#[error("Error while managing threads")]
BlockingError,
}
#[derive(Eq, PartialEq)]
@@ -45,7 +47,7 @@ pub trait Validator: Sync {
fn get_supported_game_versions(&self) -> SupportedGameVersions;
fn validate(
&self,
archive: &mut ZipArchive<Cursor<&[u8]>>,
archive: &mut ZipArchive<Cursor<bytes::Bytes>>,
) -> Result<ValidationResult, ValidationError>;
}
@@ -57,48 +59,52 @@ static VALIDATORS: [&dyn Validator; 4] = [
];
/// The return value is whether this file should be marked as primary or not, based on the analysis of the file
pub fn validate_file(
data: &[u8],
file_extension: &str,
project_type: &str,
pub async fn validate_file(
data: bytes::Bytes,
file_extension: String,
project_type: String,
loaders: Vec<Loader>,
game_versions: Vec<GameVersion>,
all_game_versions: &[crate::database::models::categories::GameVersion],
all_game_versions: Vec<crate::database::models::categories::GameVersion>,
) -> Result<ValidationResult, ValidationError> {
let reader = std::io::Cursor::new(data);
let mut zip = zip::ZipArchive::new(reader)?;
Ok(actix_web::web::block(move || {
let reader = std::io::Cursor::new(data);
let mut zip = zip::ZipArchive::new(reader)?;
let mut visited = false;
for validator in &VALIDATORS {
if validator.get_project_types().contains(&project_type)
&& loaders
.iter()
.any(|x| validator.get_supported_loaders().contains(&&*x.0))
&& game_version_supported(
&game_versions,
all_game_versions,
validator.get_supported_game_versions(),
)
{
if validator.get_file_extensions().contains(&file_extension) {
return validator.validate(&mut zip);
} else {
visited = true;
let mut visited = false;
for validator in &VALIDATORS {
if validator.get_project_types().contains(&&*project_type)
&& loaders
.iter()
.any(|x| validator.get_supported_loaders().contains(&&*x.0))
&& game_version_supported(
&game_versions,
&all_game_versions,
validator.get_supported_game_versions(),
)
{
if validator.get_file_extensions().contains(&&*file_extension) {
return validator.validate(&mut zip);
} else {
visited = true;
}
}
}
}
if visited {
Err(ValidationError::InvalidInputError(
format!(
"File extension {} is invalid for input file",
file_extension
)
.into(),
))
} else {
Ok(ValidationResult::Pass)
}
if visited {
Err(ValidationError::InvalidInputError(
format!(
"File extension {} is invalid for input file",
file_extension
)
.into(),
))
} else {
Ok(ValidationResult::Pass)
}
})
.await
.map_err(|_| ValidationError::BlockingError)?)
}
fn game_version_supported(

49
src/validate/pack.rs
View File

@@ -1,19 +1,60 @@
use crate::models::projects::SideType;
use crate::validate::{SupportedGameVersions, ValidationError, ValidationResult};
use serde::{Deserialize, Serialize};
use std::io::{Cursor, Read};
use zip::ZipArchive;
use validator::Validate;
use crate::util::validate::validation_errors_to_string;
#[derive(Serialize, Deserialize)]
#[derive(Serialize, Deserialize, Validate)]
#[serde(rename_all = "camelCase")]
pub struct PackFormat<'a> {
pub game: &'a str,
pub format_version: i32,
#[validate(length(min = 3, max = 512))]
pub version_id: &'a str,
#[validate(length(min = 3, max = 512))]
pub name: &'a str,
#[validate(length(max = 2048))]
pub summary: Option<&'a str>,
#[validate]
pub files: Vec<PackFile<'a>>,
pub dependencies: std::collections::HashMap<PackDependency, &'a str>,
}
#[derive(Serialize, Deserialize, Validate)]
pub struct PackFile<'a> {
pub path: &'a str,
pub hashes: std::collections::HashMap<FileHash, &'a str>,
pub env: std::collections::HashMap<EnvType, SideType>,
#[validate(custom(function = "validate_download_url"))]
pub downloads: Vec<&'a str>,
}
fn validate_download_url(values: &Vec<&str>) -> Result<(), validator::ValidationError> {
for value in values {
if !validator::validate_url(*value) {
return Err(validator::ValidationError::new("invalid URL"));
}
}
Ok(())
}
#[derive(Serialize, Deserialize, Eq, PartialEq, Hash)]
#[serde(rename_all = "camelCase")]
pub enum FileHash {
Sha1,
Sha512,
}
#[derive(Serialize, Deserialize, Eq, PartialEq, Hash)]
#[serde(rename_all = "camelCase")]
pub enum EnvType {
Client,
Server,
}
#[derive(Serialize, Deserialize, Clone, Hash, PartialEq, Eq)]
#[serde(rename_all = "kebab-case")]
pub enum PackDependency {
@@ -60,7 +101,7 @@ impl super::Validator for PackValidator {
fn validate(
&self,
archive: &mut ZipArchive<Cursor<&[u8]>>,
archive: &mut ZipArchive<Cursor<bytes::Bytes>>,
) -> Result<ValidationResult, ValidationError> {
let mut file = archive
.by_name("index.json")
@@ -71,6 +112,10 @@ impl super::Validator for PackValidator {
let pack: PackFormat = serde_json::from_str(&contents)?;
pack
.validate()
.map_err(|err| ValidationError::InvalidInputError(validation_errors_to_string(err, None).into()))?;
if pack.game != "minecraft" {
return Err(ValidationError::InvalidInputError(
format!("Game {0} does not exist!", pack.game).into(),