eightequal/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Automatic moderation (#875)

Browse Source 
* Automatic moderation

* finish

* modpack fixes

* fix unknown license msg

* fix moderation issues
This commit is contained in:
Geometrically
2024-02-21 16:24:21 -07:00
committed by GitHub
parent 33b2a94d90
commit 04d834187b
43 changed files with 1597 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
src/routes/internal/mod.rs
View File

@@ -1,5 +1,6 @@
pub(crate) mod admin;
pub mod flows;
pub mod moderation;
pub mod pats;
pub mod session;
@@ -12,10 +13,10 @@ pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
actix_web::web::scope("_internal")
.wrap(default_cors())
.configure(admin::config)
// TODO: write tests that catch these
.configure(oauth_clients::config)
.configure(session::config)
.configure(flows::config)
.configure(pats::config),
.configure(pats::config)
.configure(moderation::config),
);
}

313
src/routes/internal/moderation.rs Normal file
View File

@@ -0,0 +1,313 @@
use super::ApiError;
use crate::database;
use crate::database::redis::RedisPool;
use crate::models::ids::random_base62;
use crate::models::projects::ProjectStatus;
use crate::queue::moderation::{ApprovalType, IdentifiedFile, MissingMetadata};
use crate::queue::session::AuthQueue;
use crate::{auth::check_is_moderator_from_headers, models::pats::Scopes};
use actix_web::{web, HttpRequest, HttpResponse};
use serde::Deserialize;
use sqlx::PgPool;
use std::collections::HashMap;
pub fn config(cfg: &mut web::ServiceConfig) {
cfg.route("moderation/projects", web::get().to(get_projects));
cfg.route("moderation/project/{id}", web::get().to(get_project_meta));
cfg.route("moderation/project", web::post().to(set_project_meta));
}
#[derive(Deserialize)]
pub struct ResultCount {
#[serde(default = "default_count")]
pub count: i16,
}
fn default_count() -> i16 {
100
}
pub async fn get_projects(
req: HttpRequest,
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
count: web::Query<ResultCount>,
session_queue: web::Data<AuthQueue>,
) -> Result<HttpResponse, ApiError> {
check_is_moderator_from_headers(
&req,
&**pool,
&redis,
&session_queue,
Some(&[Scopes::PROJECT_READ]),
)
.await?;
use futures::stream::TryStreamExt;
let project_ids = sqlx::query!(
"
SELECT id FROM mods
WHERE status = $1
ORDER BY queued ASC
LIMIT $2;
",
ProjectStatus::Processing.as_str(),
count.count as i64
)
.fetch_many(&**pool)
.try_filter_map(|e| async { Ok(e.right().map(|m| database::models::ProjectId(m.id))) })
.try_collect::<Vec<database::models::ProjectId>>()
.await?;
let projects: Vec<_> = database::Project::get_many_ids(&project_ids, &**pool, &redis)
.await?
.into_iter()
.map(crate::models::projects::Project::from)
.collect();
Ok(HttpResponse::Ok().json(projects))
}
pub async fn get_project_meta(
req: HttpRequest,
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
info: web::Path<(String,)>,
) -> Result<HttpResponse, ApiError> {
check_is_moderator_from_headers(
&req,
&**pool,
&redis,
&session_queue,
Some(&[Scopes::PROJECT_READ]),
)
.await?;
let project_id = info.into_inner().0;
let project = database::models::Project::get(&project_id, &**pool, &redis).await?;
if let Some(project) = project {
let rows = sqlx::query!(
"
SELECT
f.metadata, v.id version_id
FROM versions v
INNER JOIN files f ON f.version_id = v.id
WHERE v.mod_id = $1
",
project.inner.id.0
)
.fetch_all(&**pool)
.await?;
let mut merged = MissingMetadata {
identified: HashMap::new(),
flame_files: HashMap::new(),
unknown_files: HashMap::new(),
};
let mut check_hashes = Vec::new();
let mut check_flames = Vec::new();
for row in rows {
if let Some(metadata) = row
.metadata
.and_then(|x| serde_json::from_value::<MissingMetadata>(x).ok())
{
merged.identified.extend(metadata.identified);
merged.flame_files.extend(metadata.flame_files);
merged.unknown_files.extend(metadata.unknown_files);
check_hashes.extend(merged.flame_files.keys().cloned());
check_hashes.extend(merged.unknown_files.keys().cloned());
check_flames.extend(merged.flame_files.values().map(|x| x.id as i32));
}
}
let rows = sqlx::query!(
"
SELECT encode(mef.sha1, 'escape') sha1, mel.status status
FROM moderation_external_files mef
INNER JOIN moderation_external_licenses mel ON mef.external_license_id = mel.id
WHERE mef.sha1 = ANY($1)
",
&check_hashes
.iter()
.map(|x| x.as_bytes().to_vec())
.collect::<Vec<_>>()
)
.fetch_all(&**pool)
.await?;
for row in rows {
if let Some(sha1) = row.sha1 {
if let Some(val) = merged.flame_files.remove(&sha1) {
merged.identified.insert(
sha1,
IdentifiedFile {
file_name: val.file_name,
status: ApprovalType::from_string(&row.status)
.unwrap_or(ApprovalType::Unidentified),
},
);
} else if let Some(val) = merged.unknown_files.remove(&sha1) {
merged.identified.insert(
sha1,
IdentifiedFile {
file_name: val,
status: ApprovalType::from_string(&row.status)
.unwrap_or(ApprovalType::Unidentified),
},
);
}
}
}
let rows = sqlx::query!(
"
SELECT mel.id, mel.flame_project_id, mel.status status
FROM moderation_external_licenses mel
WHERE mel.flame_project_id = ANY($1)
",
&check_flames,
)
.fetch_all(&**pool)
.await?;
for row in rows {
if let Some(sha1) = merged
.flame_files
.iter()
.find(|x| Some(x.1.id as i32) == row.flame_project_id)
.map(|x| x.0.clone())
{
if let Some(val) = merged.flame_files.remove(&sha1) {
merged.identified.insert(
sha1,
IdentifiedFile {
file_name: val.file_name.clone(),
status: ApprovalType::from_string(&row.status)
.unwrap_or(ApprovalType::Unidentified),
},
);
}
}
}
Ok(HttpResponse::Ok().json(merged))
} else {
Err(ApiError::NotFound)
}
}
#[derive(Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum Judgement {
Flame {
id: i32,
status: ApprovalType,
link: String,
title: String,
},
Unknown {
status: ApprovalType,
proof: Option<String>,
link: Option<String>,
title: Option<String>,
},
}
pub async fn set_project_meta(
req: HttpRequest,
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
judgements: web::Json<HashMap<String, Judgement>>,
) -> Result<HttpResponse, ApiError> {
check_is_moderator_from_headers(
&req,
&**pool,
&redis,
&session_queue,
Some(&[Scopes::PROJECT_READ]),
)
.await?;
let mut transaction = pool.begin().await?;
let mut ids = Vec::new();
let mut titles = Vec::new();
let mut statuses = Vec::new();
let mut links = Vec::new();
let mut proofs = Vec::new();
let mut flame_ids = Vec::new();
let mut file_hashes = Vec::new();
for (hash, judgement) in judgements.0 {
let id = random_base62(8);
let (title, status, link, proof, flame_id) = match judgement {
Judgement::Flame {
id,
status,
link,
title,
} => (
Some(title),
status,
Some(link),
Some("See Flame page/license for permission".to_string()),
Some(id),
),
Judgement::Unknown {
status,
proof,
link,
title,
} => (title, status, link, proof, None),
};
ids.push(id as i64);
titles.push(title);
statuses.push(status.as_str());
links.push(link);
proofs.push(proof);
flame_ids.push(flame_id);
file_hashes.push(hash);
}
sqlx::query(
"
INSERT INTO moderation_external_licenses (id, title, status, link, proof, flame_project_id)
SELECT * FROM UNNEST ($1::bigint[], $2::varchar[], $3::varchar[], $4::varchar[], $5::varchar[], $6::integer[])
"
)
.bind(&ids[..])
.bind(&titles[..])
.bind(&statuses[..])
.bind(&links[..])
.bind(&proofs[..])
.bind(&flame_ids[..])
.execute(&mut *transaction)
.await?;
sqlx::query(
"
INSERT INTO moderation_external_files (sha1, external_license_id)
SELECT * FROM UNNEST ($1::bytea[], $2::bigint[])
ON CONFLICT (sha1)
DO NOTHING
",
)
.bind(&file_hashes[..])
.bind(&ids[..])
.execute(&mut *transaction)
.await?;
transaction.commit().await?;
Ok(HttpResponse::NoContent().finish())
}

70
src/routes/mod.rs
View File

@@ -123,10 +123,49 @@ pub enum ApiError {
Mail(#[from] crate::auth::email::MailError),
#[error("Error while rerouting request: {0}")]
Reroute(#[from] reqwest::Error),
#[error("Unable to read Zip Archive: {0}")]
Zip(#[from] zip::result::ZipError),
#[error("IO Error: {0}")]
Io(#[from] std::io::Error),
#[error("Resource not found")]
NotFound,
}
impl ApiError {
pub fn as_api_error<'a>(&self) -> crate::models::error::ApiError<'a> {
crate::models::error::ApiError {
error: match self {
ApiError::Env(..) => "environment_error",
ApiError::SqlxDatabase(..) => "database_error",
ApiError::Database(..) => "database_error",
ApiError::Authentication(..) => "unauthorized",
ApiError::CustomAuthentication(..) => "unauthorized",
ApiError::Xml(..) => "xml_error",
ApiError::Json(..) => "json_error",
ApiError::Search(..) => "search_error",
ApiError::Indexing(..) => "indexing_error",
ApiError::FileHosting(..) => "file_hosting_error",
ApiError::InvalidInput(..) => "invalid_input",
ApiError::Validation(..) => "invalid_input",
ApiError::Payments(..) => "payments_error",
ApiError::Discord(..) => "discord_error",
ApiError::Turnstile => "turnstile_error",
ApiError::Decoding(..) => "decoding_error",
ApiError::ImageParse(..) => "invalid_image",
ApiError::PasswordHashing(..) => "password_hashing_error",
ApiError::PasswordStrengthCheck(..) => "strength_check_error",
ApiError::Mail(..) => "mail_error",
ApiError::Clickhouse(..) => "clickhouse_error",
ApiError::Reroute(..) => "reroute_error",
ApiError::NotFound => "not_found",
ApiError::Zip(..) => "zip_error",
ApiError::Io(..) => "io_error",
},
description: self.to_string(),
}
}
}
impl actix_web::ResponseError for ApiError {
fn status_code(&self) -> StatusCode {
match self {
@@ -153,37 +192,12 @@ impl actix_web::ResponseError for ApiError {
ApiError::Mail(..) => StatusCode::INTERNAL_SERVER_ERROR,
ApiError::Reroute(..) => StatusCode::INTERNAL_SERVER_ERROR,
ApiError::NotFound => StatusCode::NOT_FOUND,
ApiError::Zip(..) => StatusCode::BAD_REQUEST,
ApiError::Io(..) => StatusCode::BAD_REQUEST,
}
}
fn error_response(&self) -> HttpResponse {
HttpResponse::build(self.status_code()).json(crate::models::error::ApiError {
error: match self {
ApiError::Env(..) => "environment_error",
ApiError::SqlxDatabase(..) => "database_error",
ApiError::Database(..) => "database_error",
ApiError::Authentication(..) => "unauthorized",
ApiError::CustomAuthentication(..) => "unauthorized",
ApiError::Xml(..) => "xml_error",
ApiError::Json(..) => "json_error",
ApiError::Search(..) => "search_error",
ApiError::Indexing(..) => "indexing_error",
ApiError::FileHosting(..) => "file_hosting_error",
ApiError::InvalidInput(..) => "invalid_input",
ApiError::Validation(..) => "invalid_input",
ApiError::Payments(..) => "payments_error",
ApiError::Discord(..) => "discord_error",
ApiError::Turnstile => "turnstile_error",
ApiError::Decoding(..) => "decoding_error",
ApiError::ImageParse(..) => "invalid_image",
ApiError::PasswordHashing(..) => "password_hashing_error",
ApiError::PasswordStrengthCheck(..) => "strength_check_error",
ApiError::Mail(..) => "mail_error",
ApiError::Clickhouse(..) => "clickhouse_error",
ApiError::Reroute(..) => "reroute_error",
ApiError::NotFound => "not_found",
},
description: &self.to_string(),
})
HttpResponse::build(self.status_code()).json(self.as_api_error())
}
}

2
src/routes/not_found.rs
View File

@@ -4,7 +4,7 @@ use actix_web::{HttpResponse, Responder};
pub async fn not_found() -> impl Responder {
let data = ApiError {
error: "not_found",
description: "the requested route does not exist",
description: "the requested route does not exist".to_string(),
};
HttpResponse::NotFound().json(data)

6
src/routes/v2/moderation.rs
View File

@@ -2,7 +2,7 @@ use super::ApiError;
use crate::models::projects::Project;
use crate::models::v2::projects::LegacyProject;
use crate::queue::session::AuthQueue;
use crate::routes::v3;
use crate::routes::internal;
use crate::{database::redis::RedisPool, routes::v2_reroute};
use actix_web::{get, web, HttpRequest, HttpResponse};
use serde::Deserialize;
@@ -30,11 +30,11 @@ pub async fn get_projects(
count: web::Query<ResultCount>,
session_queue: web::Data<AuthQueue>,
) -> Result<HttpResponse, ApiError> {
let response = v3::moderation::get_projects(
let response = internal::moderation::get_projects(
req,
pool.clone(),
redis.clone(),
web::Query(v3::moderation::ResultCount { count: count.count }),
web::Query(internal::moderation::ResultCount { count: count.count }),
session_queue,
)
.await

4
src/routes/v2/projects.rs
View File

@@ -7,6 +7,7 @@ use crate::models::projects::{
};
use crate::models::v2::projects::{DonationLink, LegacyProject, LegacySideType, LegacyVersion};
use crate::models::v2::search::LegacySearchResults;
use crate::queue::moderation::AutomatedModerationQueue;
use crate::queue::session::AuthQueue;
use crate::routes::v3::projects::ProjectIds;
use crate::routes::{v2_reroute, v3, ApiError};
@@ -380,6 +381,7 @@ pub struct EditProject {
}
#[patch("{id}")]
#[allow(clippy::too_many_arguments)]
pub async fn project_edit(
req: HttpRequest,
info: web::Path<(String,)>,
@@ -388,6 +390,7 @@ pub async fn project_edit(
new_project: web::Json<EditProject>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
moderation_queue: web::Data<AutomatedModerationQueue>,
) -> Result<HttpResponse, ApiError> {
let v2_new_project = new_project.into_inner();
let client_side = v2_new_project.client_side;
@@ -494,6 +497,7 @@ pub async fn project_edit(
web::Json(new_project),
redis.clone(),
session_queue.clone(),
moderation_queue,
)
.await
.or_else(v2_reroute::flatten_404_error)?;

2
src/routes/v3/mod.rs
View File

@@ -6,7 +6,6 @@ use serde_json::json;
pub mod analytics_get;
pub mod collections;
pub mod images;
pub mod moderation;
pub mod notifications;
pub mod organizations;
pub mod payouts;
@@ -31,7 +30,6 @@ pub fn config(cfg: &mut web::ServiceConfig) {
.configure(analytics_get::config)
.configure(collections::config)
.configure(images::config)
.configure(moderation::config)
.configure(notifications::config)
.configure(organizations::config)
.configure(project_creation::config)

65
src/routes/v3/moderation.rs
View File

@@ -1,65 +0,0 @@
use super::ApiError;
use crate::database;
use crate::database::redis::RedisPool;
use crate::models::projects::ProjectStatus;
use crate::queue::session::AuthQueue;
use crate::{auth::check_is_moderator_from_headers, models::pats::Scopes};
use actix_web::{web, HttpRequest, HttpResponse};
use serde::Deserialize;
use sqlx::PgPool;
pub fn config(cfg: &mut web::ServiceConfig) {
cfg.route("moderation/projects", web::get().to(get_projects));
}
#[derive(Deserialize)]
pub struct ResultCount {
#[serde(default = "default_count")]
pub count: i16,
}
fn default_count() -> i16 {
100
}
pub async fn get_projects(
req: HttpRequest,
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
count: web::Query<ResultCount>,
session_queue: web::Data<AuthQueue>,
) -> Result<HttpResponse, ApiError> {
check_is_moderator_from_headers(
&req,
&**pool,
&redis,
&session_queue,
Some(&[Scopes::PROJECT_READ]),
)
.await?;
use futures::stream::TryStreamExt;
let project_ids = sqlx::query!(
"
SELECT id FROM mods
WHERE status = $1
ORDER BY queued ASC
LIMIT $2;
",
ProjectStatus::Processing.as_str(),
count.count as i64
)
.fetch_many(&**pool)
.try_filter_map(|e| async { Ok(e.right().map(|m| database::models::ProjectId(m.id))) })
.try_collect::<Vec<database::models::ProjectId>>()
.await?;
let projects: Vec<_> = database::Project::get_many_ids(&project_ids, &**pool, &redis)
.await?
.into_iter()
.map(crate::models::projects::Project::from)
.collect();
Ok(HttpResponse::Ok().json(projects))
}

2
src/routes/v3/project_creation.rs
View File

@@ -137,7 +137,7 @@ impl actix_web::ResponseError for CreateError {
CreateError::ImageError(..) => "invalid_image",
CreateError::RerouteError(..) => "reroute_error",
},
description: &self.to_string(),
description: self.to_string(),
})
}
}

7
src/routes/v3/projects.rs
View File

@@ -20,6 +20,7 @@ use crate::models::projects::{
};
use crate::models::teams::ProjectPermissions;
use crate::models::threads::MessageBody;
use crate::queue::moderation::AutomatedModerationQueue;
use crate::queue::session::AuthQueue;
use crate::routes::ApiError;
use crate::search::indexing::remove_documents;
@@ -229,6 +230,7 @@ pub struct EditProject {
pub monetization_status: Option<MonetizationStatus>,
}
#[allow(clippy::too_many_arguments)]
pub async fn project_edit(
req: HttpRequest,
info: web::Path<(String,)>,
@@ -237,6 +239,7 @@ pub async fn project_edit(
new_project: web::Json<EditProject>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
moderation_queue: web::Data<AutomatedModerationQueue>,
) -> Result<HttpResponse, ApiError> {
let user = get_user_from_headers(
&req,
@@ -362,6 +365,10 @@ pub async fn project_edit(
)
.execute(&mut *transaction)
.await?;
moderation_queue
.projects
.insert(project_item.inner.id.into());
}
if status.is_approved() && !project_item.inner.status.is_approved() {

7
src/routes/v3/threads.rs
View File

@@ -379,6 +379,7 @@ pub async fn thread_send_message(
body,
replying_to,
private,
hide_identity,
..
} = &new_message.body
{
@@ -394,6 +395,12 @@ pub async fn thread_send_message(
));
}
if *hide_identity && !user.role.is_mod() {
return Err(ApiError::InvalidInput(
"You are not allowed to send masked messages!".to_string(),
));
}
if let Some(replying_to) = replying_to {
let thread_message =
database::models::ThreadMessage::get((*replying_to).into(), &**pool).await?;