tomasalias/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Automatic moderation (#875)

Browse Source 
* Automatic moderation

* finish

* modpack fixes

* fix unknown license msg

* fix moderation issues
This commit is contained in:
Geometrically
2024-02-21 16:24:21 -07:00
committed by GitHub
parent 33b2a94d90
commit 04d834187b
43 changed files with 1597 additions and 158 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.env
View File

@@ -98,3 +98,5 @@ CLICKHOUSE_DATABASE=staging_ariadne
MAXMIND_LICENSE_KEY=none
PAYOUTS_BUDGET=100
FLAME_ANVIL_URL=none

34
.sqlx/query-3151ef71738a1f0d097aa14967d7b9eb1f24d4de1f81b80c4bd186427edc1399.json generated Normal file
View File

@@ -0,0 +1,34 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT mel.id, mel.flame_project_id, mel.status status\n FROM moderation_external_licenses mel\n WHERE mel.flame_project_id = ANY($1)\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "flame_project_id",
"type_info": "Int4"
},
{
"ordinal": 2,
"name": "status",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int4Array"
]
},
"nullable": [
false,
true,
false
]
},
"hash": "3151ef71738a1f0d097aa14967d7b9eb1f24d4de1f81b80c4bd186427edc1399"
}

34
.sqlx/query-3c875a8a1c03432f258040c436e19dbab6e78bd1789dc70f445578c779c7b995.json generated Normal file
View File

@@ -0,0 +1,34 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT mel.id, mel.flame_project_id, mel.status status\n FROM moderation_external_licenses mel\n WHERE mel.flame_project_id = ANY($1)\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "flame_project_id",
"type_info": "Int4"
},
{
"ordinal": 2,
"name": "status",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"Int4Array"
]
},
"nullable": [
false,
true,
false
]
},
"hash": "3c875a8a1c03432f258040c436e19dbab6e78bd1789dc70f445578c779c7b995"
}

15
.sqlx/query-3d535886d8a239967e6556fb0cd0588b79a7787b9b3cbbd4f8968cd0d99ed49d.json generated Normal file
View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "\n INSERT INTO moderation_external_files (sha1, external_license_id)\n SELECT * FROM UNNEST ($1::bytea[], $2::bigint[])\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"ByteaArray",
"Int8Array"
]
},
"nullable": []
},
"hash": "3d535886d8a239967e6556fb0cd0588b79a7787b9b3cbbd4f8968cd0d99ed49d"
}

28
.sqlx/query-4cb9fe3dbb2cbfe30a49487f896fb7890f726af2ff11da53f450a88c3dc5fc64.json generated Normal file
View File

@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT encode(mef.sha1, 'escape') sha1, mel.status status\n FROM moderation_external_files mef\n INNER JOIN moderation_external_licenses mel ON mef.external_license_id = mel.id\n WHERE mef.sha1 = ANY($1)\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "sha1",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "status",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"ByteaArray"
]
},
"nullable": [
null,
false
]
},
"hash": "4cb9fe3dbb2cbfe30a49487f896fb7890f726af2ff11da53f450a88c3dc5fc64"
}

28
.sqlx/query-6e4ff5010b19890e26867611a243a308fb32f7439a18c83d1e16d3e537a43e7d.json generated Normal file
View File

@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT encode(mef.sha1, 'escape') sha1, mel.status status\n FROM moderation_external_files mef\n INNER JOIN moderation_external_licenses mel ON mef.external_license_id = mel.id\n WHERE mef.sha1 = ANY($1)\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "sha1",
"type_info": "Text"
},
{
"ordinal": 1,
"name": "status",
"type_info": "Text"
}
],
"parameters": {
"Left": [
"ByteaArray"
]
},
"nullable": [
null,
false
]
},
"hash": "6e4ff5010b19890e26867611a243a308fb32f7439a18c83d1e16d3e537a43e7d"
}

14
.sqlx/query-95e17b2512494ffcbfe6278b87aa273edc5729633aeaa87f6239667d2f861e68.json generated Normal file
View File

@@ -0,0 +1,14 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE mods\n SET status = 'rejected'\n WHERE id = $1\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": []
},
"hash": "95e17b2512494ffcbfe6278b87aa273edc5729633aeaa87f6239667d2f861e68"
}

7
.sqlx/query-b768d9db6c785d6a701324ea746794d33e94121403163a774b6ef775640fd3d3.json → .sqlx/query-a8bfce13de871daf0bb1cf73b4c5ded611ff58d94461404182942210492e8010.json generated
View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT m.id id, tm.user_id user_id, tm.payouts_split payouts_split\n FROM mods m\n INNER JOIN team_members tm on m.team_id = tm.team_id AND tm.accepted = TRUE\n WHERE m.id = ANY($1) AND m.monetization_status = $2\n ",
"query": "\n SELECT m.id id, tm.user_id user_id, tm.payouts_split payouts_split\n FROM mods m\n INNER JOIN team_members tm on m.team_id = tm.team_id AND tm.accepted = TRUE\n WHERE m.id = ANY($1) AND m.monetization_status = $2 AND m.status = ANY($3)\n ",
"describe": {
"columns": [
{
@@ -22,7 +22,8 @@
"parameters": {
"Left": [
"Int8Array",
"Text"
"Text",
"TextArray"
]
},
"nullable": [
@@ -31,5 +32,5 @@
false
]
},
"hash": "b768d9db6c785d6a701324ea746794d33e94121403163a774b6ef775640fd3d3"
"hash": "a8bfce13de871daf0bb1cf73b4c5ded611ff58d94461404182942210492e8010"
}

7
.sqlx/query-03cd8926d18aa8c11934fdc0da32ccbbbccf2527c523336f230c0e344c471a0f.json → .sqlx/query-b82d35429e009e515ae1e0332142b3bd0bec55f38807eded9130b932929f2ebe.json generated
View File

@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT m.id id, tm.user_id user_id, tm.payouts_split payouts_split\n FROM mods m\n INNER JOIN organizations o ON m.organization_id = o.id\n INNER JOIN team_members tm on o.team_id = tm.team_id AND tm.accepted = TRUE\n WHERE m.id = ANY($1) AND m.monetization_status = $2 AND m.organization_id IS NOT NULL\n ",
"query": "\n SELECT m.id id, tm.user_id user_id, tm.payouts_split payouts_split\n FROM mods m\n INNER JOIN organizations o ON m.organization_id = o.id\n INNER JOIN team_members tm on o.team_id = tm.team_id AND tm.accepted = TRUE\n WHERE m.id = ANY($1) AND m.monetization_status = $2 AND m.status = ANY($3) AND m.organization_id IS NOT NULL\n ",
"describe": {
"columns": [
{
@@ -22,7 +22,8 @@
"parameters": {
"Left": [
"Int8Array",
"Text"
"Text",
"TextArray"
]
},
"nullable": [
@@ -31,5 +32,5 @@
false
]
},
"hash": "03cd8926d18aa8c11934fdc0da32ccbbbccf2527c523336f230c0e344c471a0f"
"hash": "b82d35429e009e515ae1e0332142b3bd0bec55f38807eded9130b932929f2ebe"
}

15
.sqlx/query-c2924fff035e92f7bd2279517310ba391ced72b38be97d462cdfe60048e947db.json generated Normal file
View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE files\n SET metadata = $1\n WHERE id = $2\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb",
"Int8"
]
},
"nullable": []
},
"hash": "c2924fff035e92f7bd2279517310ba391ced72b38be97d462cdfe60048e947db"
}

28
.sqlx/query-cc1f2f568a0ba1d285a95fd9b6e3b118a0eaa26e2851bcc3f1920ae0140b48ae.json generated Normal file
View File

@@ -0,0 +1,28 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT\n f.metadata, v.id version_id\n FROM versions v\n INNER JOIN files f ON f.version_id = v.id\n WHERE v.mod_id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "metadata",
"type_info": "Jsonb"
},
{
"ordinal": 1,
"name": "version_id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
true,
false
]
},
"hash": "cc1f2f568a0ba1d285a95fd9b6e3b118a0eaa26e2851bcc3f1920ae0140b48ae"
}

15
.sqlx/query-ccf57f9c1026927afc940a20ebad9fb58ded7171b21e91973d1f13c91eab9b37.json generated Normal file
View File

@@ -0,0 +1,15 @@
{
"db_name": "PostgreSQL",
"query": "\n UPDATE files\n SET metadata = $1\n WHERE id = $2\n ",
"describe": {
"columns": [],
"parameters": {
"Left": [
"Jsonb",
"Int8"
]
},
"nullable": []
},
"hash": "ccf57f9c1026927afc940a20ebad9fb58ded7171b21e91973d1f13c91eab9b37"
}

7
Cargo.lock generated
View File

@@ -2343,6 +2343,7 @@ dependencies = [
"log",
"maxminddb",
"meilisearch-sdk",
"murmur2",
"rand",
"rand_chacha",
"redis",
@@ -2695,6 +2696,12 @@ dependencies = [
"windows-sys 0.48.0",
]
[[package]]
name = "murmur2"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb585ade2549a017db2e35978b77c319214fa4b37cede841e27954dd6e8f3ca8"
[[package]]
name = "native-tls"
version = "0.2.11"

1
Cargo.toml
View File

@@ -51,6 +51,7 @@ sha1 = { version = "0.6.1", features = ["std"] }
sha2 = "0.9.9"
hmac = "0.11.0"
argon2 = { version = "0.5.0", features = ["std"] }
murmur2 = "0.1.0"
bitflags = "2.4.0"
hex = "0.4.3"
zxcvbn = "2.2.2"

19
migrations/20240131224610_moderation_packs.sql Normal file
View File

@@ -0,0 +1,19 @@
CREATE TABLE moderation_external_licenses (
id bigint PRIMARY KEY,
title text not null,
status text not null,
link text null,
exceptions text null,
proof text null,
flame_project_id integer null
);
CREATE TABLE moderation_external_files (
sha1 bytea PRIMARY KEY,
external_license_id bigint references moderation_external_licenses not null
);
ALTER TABLE files ADD COLUMN metadata jsonb NULL;
INSERT INTO users (id, username, name, email, avatar_url, bio, role, badges, balance)
VALUES (0, 'AutoMod', 'AutoMod', 'support@modrinth.com', 'https://cdn.modrinth.com/user/2REoufqX/6aabaf2d1fca2935662eca4ce451cd9775054c22.png', 'An automated account performing moderation utilities for Modrinth.', 'moderator', 0, 0)

2
migrations/20240221215354_moderation_pack_fixes.sql Normal file
View File

@@ -0,0 +1,2 @@
ALTER TABLE moderation_external_files ALTER COLUMN sha1 SET NOT NULL;
ALTER TABLE moderation_external_licenses ALTER COLUMN title DROP NOT NULL;

27
src/auth/checks.rs
View File

@@ -6,7 +6,6 @@ use crate::database::redis::RedisPool;
use crate::database::{models, Project, Version};
use crate::models::users::User;
use crate::routes::ApiError;
use actix_web::web;
use itertools::Itertools;
use sqlx::PgPool;
@@ -32,7 +31,7 @@ where
pub async fn is_visible_project(
project_data: &Project,
user_option: &Option<User>,
pool: &web::Data<PgPool>,
pool: &PgPool,
hide_unlisted: bool,
) -> Result<bool, ApiError> {
filter_visible_project_ids(vec![project_data], user_option, pool, hide_unlisted)
@@ -43,7 +42,7 @@ pub async fn is_visible_project(
pub async fn is_team_member_project(
project_data: &Project,
user_option: &Option<User>,
pool: &web::Data<PgPool>,
pool: &PgPool,
) -> Result<bool, ApiError> {
filter_enlisted_projects_ids(vec![project_data], user_option, pool)
.await
@@ -53,7 +52,7 @@ pub async fn is_team_member_project(
pub async fn filter_visible_projects(
mut projects: Vec<QueryProject>,
user_option: &Option<User>,
pool: &web::Data<PgPool>,
pool: &PgPool,
hide_unlisted: bool,
) -> Result<Vec<crate::models::projects::Project>, ApiError> {
let filtered_project_ids = filter_visible_project_ids(
@@ -76,7 +75,7 @@ pub async fn filter_visible_projects(
pub async fn filter_visible_project_ids(
projects: Vec<&Project>,
user_option: &Option<User>,
pool: &web::Data<PgPool>,
pool: &PgPool,
hide_unlisted: bool,
) -> Result<Vec<crate::database::models::ProjectId>, ApiError> {
let mut return_projects = Vec::new();
@@ -114,7 +113,7 @@ pub async fn filter_visible_project_ids(
pub async fn filter_enlisted_projects_ids(
projects: Vec<&Project>,
user_option: &Option<User>,
pool: &web::Data<PgPool>,
pool: &PgPool,
) -> Result<Vec<crate::database::models::ProjectId>, ApiError> {
let mut return_projects = vec![];
@@ -142,7 +141,7 @@ pub async fn filter_enlisted_projects_ids(
.collect::<Vec<_>>(),
user_id as database::models::ids::UserId,
)
.fetch_many(&***pool)
.fetch_many(pool)
.try_for_each(|e| {
if let Some(row) = e.right() {
for x in projects.iter() {
@@ -163,7 +162,7 @@ pub async fn filter_enlisted_projects_ids(
pub async fn is_visible_version(
version_data: &Version,
user_option: &Option<User>,
pool: &web::Data<PgPool>,
pool: &PgPool,
redis: &RedisPool,
) -> Result<bool, ApiError> {
filter_visible_version_ids(vec![version_data], user_option, pool, redis)
@@ -174,7 +173,7 @@ pub async fn is_visible_version(
pub async fn is_team_member_version(
version_data: &Version,
user_option: &Option<User>,
pool: &web::Data<PgPool>,
pool: &PgPool,
redis: &RedisPool,
) -> Result<bool, ApiError> {
filter_enlisted_version_ids(vec![version_data], user_option, pool, redis)
@@ -185,7 +184,7 @@ pub async fn is_team_member_version(
pub async fn filter_visible_versions(
mut versions: Vec<QueryVersion>,
user_option: &Option<User>,
pool: &web::Data<PgPool>,
pool: &PgPool,
redis: &RedisPool,
) -> Result<Vec<crate::models::projects::Version>, ApiError> {
let filtered_version_ids = filter_visible_version_ids(
@@ -220,7 +219,7 @@ impl ValidateAuthorized for models::OAuthClient {
pub async fn filter_visible_version_ids(
versions: Vec<&Version>,
user_option: &Option<User>,
pool: &web::Data<PgPool>,
pool: &PgPool,
redis: &RedisPool,
) -> Result<Vec<crate::database::models::VersionId>, ApiError> {
let mut return_versions = Vec::new();
@@ -233,7 +232,7 @@ pub async fn filter_visible_version_ids(
// Get visible projects- ones we are allowed to see public versions for.
let visible_project_ids = filter_visible_project_ids(
Project::get_many_ids(&project_ids, &***pool, redis)
Project::get_many_ids(&project_ids, pool, redis)
.await?
.iter()
.map(|x| &x.inner)
@@ -273,7 +272,7 @@ pub async fn filter_visible_version_ids(
pub async fn filter_enlisted_version_ids(
versions: Vec<&Version>,
user_option: &Option<User>,
pool: &web::Data<PgPool>,
pool: &PgPool,
redis: &RedisPool,
) -> Result<Vec<crate::database::models::VersionId>, ApiError> {
let mut return_versions = Vec::new();
@@ -283,7 +282,7 @@ pub async fn filter_enlisted_version_ids(
// Get enlisted projects- ones we are allowed to see hidden versions for.
let authorized_project_ids = filter_enlisted_projects_ids(
Project::get_many_ids(&project_ids, &***pool, redis)
Project::get_many_ids(&project_ids, pool, redis)
.await?
.iter()
.map(|x| &x.inner)

2
src/auth/mod.rs
View File

@@ -72,7 +72,7 @@ impl actix_web::ResponseError for AuthenticationError {
fn error_response(&self) -> HttpResponse {
HttpResponse::build(self.status_code()).json(ApiError {
error: self.error_name(),
description: &self.to_string(),
description: self.to_string(),
})
}
}

2
src/auth/oauth/errors.rs
View File

@@ -100,7 +100,7 @@ impl actix_web::ResponseError for OAuthError {
} else {
HttpResponse::build(self.status_code()).json(ApiError {
error: &self.error_type.error_name(),
description: &self.error_type.to_string(),
description: self.error_type.to_string(),
})
}
}

4
src/database/models/ids.rs
View File

@@ -2,6 +2,8 @@ use super::DatabaseError;
use crate::models::ids::base62_impl::to_base62;
use crate::models::ids::{random_base62_rng, random_base62_rng_range};
use censor::Censor;
use rand::SeedableRng;
use rand_chacha::ChaCha20Rng;
use serde::{Deserialize, Serialize};
use sqlx::sqlx_macros::Type;
@@ -12,7 +14,7 @@ macro_rules! generate_ids {
$vis async fn $function_name(
con: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<$return_type, DatabaseError> {
let mut rng = rand::thread_rng();
let mut rng = ChaCha20Rng::from_entropy();
let length = $id_length;
let mut id = random_base62_rng(&mut rng, length);
let mut retry_count = 0;

10
src/database/models/team_item.rs
View File

@@ -269,9 +269,13 @@ impl TeamMember {
.try_collect::<Vec<TeamMember>>()
.await?;
for (id, members) in &teams.into_iter().group_by(|x| x.team_id) {
let mut members = members.collect::<Vec<_>>();
for (id, mut members) in teams
.into_iter()
.group_by(|x| x.team_id)
.into_iter()
.map(|(key, group)| (key, group.collect::<Vec<_>>()))
.collect::<Vec<_>>()
{
redis
.set_serialized_to_json(TEAMS_NAMESPACE, id.0, &members, None)
.await?;

57
src/lib.rs
View File

@@ -14,8 +14,9 @@ extern crate clickhouse as clickhouse_crate;
use clickhouse_crate::Client;
use util::cors::default_cors;
use crate::queue::moderation::AutomatedModerationQueue;
use crate::{
queue::payouts::process_payout,
// queue::payouts::process_payout,
search::indexing::index_projects,
util::env::{parse_strings_from_var, parse_var},
};
@@ -52,6 +53,7 @@ pub struct LabrinthConfig {
pub payouts_queue: web::Data<PayoutsQueue>,
pub analytics_queue: Arc<AnalyticsQueue>,
pub active_sockets: web::Data<RwLock<ActiveSockets>>,
pub automated_moderation_queue: web::Data<AutomatedModerationQueue>,
}
pub fn app_setup(
@@ -67,6 +69,17 @@ pub fn app_setup(
dotenvy::var("BIND_ADDR").unwrap()
);
let automated_moderation_queue = web::Data::new(AutomatedModerationQueue::default());
let automated_moderation_queue_ref = automated_moderation_queue.clone();
let pool_ref = pool.clone();
let redis_pool_ref = redis_pool.clone();
actix_rt::spawn(async move {
automated_moderation_queue_ref
.task(pool_ref, redis_pool_ref)
.await;
});
let mut scheduler = scheduler::Scheduler::new();
// The interval in seconds at which the local database is indexed
@@ -201,25 +214,25 @@ pub fn app_setup(
});
}
{
let pool_ref = pool.clone();
let redis_ref = redis_pool.clone();
let client_ref = clickhouse.clone();
scheduler.run(std::time::Duration::from_secs(60 * 60 * 6), move || {
let pool_ref = pool_ref.clone();
let redis_ref = redis_ref.clone();
let client_ref = client_ref.clone();
async move {
info!("Started running payouts");
let result = process_payout(&pool_ref, &redis_ref, &client_ref).await;
if let Err(e) = result {
warn!("Payouts run failed: {:?}", e);
}
info!("Done running payouts");
}
});
}
// {
// let pool_ref = pool.clone();
// let redis_ref = redis_pool.clone();
// let client_ref = clickhouse.clone();
// scheduler.run(std::time::Duration::from_secs(60 * 60 * 6), move || {
// let pool_ref = pool_ref.clone();
// let redis_ref = redis_ref.clone();
// let client_ref = client_ref.clone();
//
// async move {
// info!("Started running payouts");
// let result = process_payout(&pool_ref, &redis_ref, &client_ref).await;
// if let Err(e) = result {
// warn!("Payouts run failed: {:?}", e);
// }
// info!("Done running payouts");
// }
// });
// }
let ip_salt = Pepper {
pepper: models::ids::Base62Id(models::ids::random_base62(11)).to_string(),
@@ -241,6 +254,7 @@ pub fn app_setup(
payouts_queue,
analytics_queue,
active_sockets,
automated_moderation_queue,
}
}
@@ -272,6 +286,7 @@ pub fn app_config(cfg: &mut web::ServiceConfig, labrinth_config: LabrinthConfig)
.app_data(web::Data::new(labrinth_config.clickhouse.clone()))
.app_data(web::Data::new(labrinth_config.maxmind.clone()))
.app_data(labrinth_config.active_sockets.clone())
.app_data(labrinth_config.automated_moderation_queue.clone())
.configure(routes::v2::config)
.configure(routes::v3::config)
.configure(routes::internal::config)
@@ -397,5 +412,7 @@ pub fn check_env_vars() -> bool {
failed |= check_var::<u64>("PAYOUTS_BUDGET");
failed |= check_var::<String>("FLAME_ANVIL_URL");
failed
}

2
src/models/error.rs
View File

@@ -4,5 +4,5 @@ use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize)]
pub struct ApiError<'a> {
pub error: &'a str,
pub description: &'a str,
pub description: String,
}

4
src/models/v2/threads.rs
View File

@@ -30,6 +30,8 @@ pub enum LegacyMessageBody {
body: String,
#[serde(default)]
private: bool,
#[serde(default)]
hide_identity: bool,
replying_to: Option<ThreadMessageId>,
#[serde(default)]
associated_images: Vec<ImageId>,
@@ -74,11 +76,13 @@ impl From<crate::models::v3::threads::MessageBody> for LegacyMessageBody {
private,
replying_to,
associated_images,
hide_identity,
} => LegacyMessageBody::Text {
body,
private,
replying_to,
associated_images,
hide_identity,
},
crate::models::v3::threads::MessageBody::StatusChange {
new_status,

6
src/models/v3/pack.rs
View File

@@ -18,7 +18,7 @@ pub struct PackFormat {
pub dependencies: std::collections::HashMap<PackDependency, String>,
}
#[derive(Serialize, Deserialize, Validate, Eq, PartialEq, Debug)]
#[derive(Serialize, Deserialize, Validate, Eq, PartialEq, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub struct PackFile {
pub path: String,
@@ -54,7 +54,7 @@ fn validate_download_url(values: &[String]) -> Result<(), validator::ValidationE
Ok(())
}
#[derive(Serialize, Deserialize, Eq, PartialEq, Hash, Debug)]
#[derive(Serialize, Deserialize, Eq, PartialEq, Hash, Debug, Clone)]
#[serde(rename_all = "camelCase", from = "String")]
pub enum PackFileHash {
Sha1,
@@ -72,7 +72,7 @@ impl From<String> for PackFileHash {
}
}
#[derive(Serialize, Deserialize, Eq, PartialEq, Hash, Debug)]
#[derive(Serialize, Deserialize, Eq, PartialEq, Hash, Debug, Clone)]
#[serde(rename_all = "camelCase")]
pub enum EnvType {
Client,

2
src/models/v3/projects.rs
View File

@@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize};
use validator::Validate;
/// The ID of a specific project, encoded as base62 for usage in the API
#[derive(Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Debug)]
#[derive(Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Debug, Hash)]
#[serde(from = "Base62Id")]
#[serde(into = "Base62Id")]
pub struct ProjectId(pub u64);

2
src/models/v3/threads.rs
View File

@@ -41,6 +41,8 @@ pub enum MessageBody {
body: String,
#[serde(default)]
private: bool,
#[serde(default)]
hide_identity: bool,
replying_to: Option<ThreadMessageId>,
#[serde(default)]
associated_images: Vec<ImageId>,

1
src/queue/mod.rs
View File

@@ -1,5 +1,6 @@
pub mod analytics;
pub mod maxmind;
pub mod moderation;
pub mod payouts;
pub mod session;
pub mod socket;

879
src/queue/moderation.rs Normal file
View File

@@ -0,0 +1,879 @@
use crate::auth::checks::filter_visible_versions;
use crate::database;
use crate::database::models::notification_item::NotificationBuilder;
use crate::database::models::thread_item::ThreadMessageBuilder;
use crate::database::redis::RedisPool;
use crate::models::ids::ProjectId;
use crate::models::notifications::NotificationBody;
use crate::models::pack::{PackFile, PackFileHash, PackFormat};
use crate::models::projects::ProjectStatus;
use crate::models::threads::MessageBody;
use crate::routes::ApiError;
use dashmap::DashSet;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use sqlx::PgPool;
use std::collections::HashMap;
use std::io::{Cursor, Read};
use std::time::Duration;
use zip::ZipArchive;
const AUTOMOD_ID: i64 = 0;
pub struct ModerationMessages {
pub messages: Vec<ModerationMessage>,
pub version_specific: HashMap<String, Vec<ModerationMessage>>,
}
impl ModerationMessages {
pub fn is_empty(&self) -> bool {
self.messages.is_empty() && self.version_specific.is_empty()
}
pub fn markdown(&self, auto_mod: bool) -> String {
let mut str = "".to_string();
for message in &self.messages {
str.push_str(&format!("## {}\n", message.header()));
str.push_str(&format!("{}\n", message.body()));
str.push('\n');
}
for (version_num, messages) in &self.version_specific {
for message in messages {
str.push_str(&format!(
"## Version {}: {}\n",
version_num,
message.header()
));
str.push_str(&format!("{}\n", message.body()));
str.push('\n');
}
}
if auto_mod {
str.push_str("<hr />\n\n");
str.push_str("🤖 This is an automated message generated by AutoMod (BETA). If you are facing issues, please [contact support](https://support.modrinth.com).");
}
str
}
pub fn should_reject(&self, first_time: bool) -> bool {
self.messages.iter().any(|x| x.rejectable(first_time))
|| self
.version_specific
.values()
.any(|x| x.iter().any(|x| x.rejectable(first_time)))
}
pub fn approvable(&self) -> bool {
self.messages.iter().all(|x| x.approvable())
&& self
.version_specific
.values()
.all(|x| x.iter().all(|x| x.approvable()))
}
}
pub enum ModerationMessage {
MissingGalleryImage,
NoPrimaryFile,
NoSideTypes,
PackFilesNotAllowed {
files: HashMap<String, IdentifiedFile>,
incomplete: bool,
},
MissingLicense,
MissingCustomLicenseUrl {
license: String,
},
}
impl ModerationMessage {
pub fn rejectable(&self, first_time: bool) -> bool {
match self {
ModerationMessage::NoPrimaryFile => true,
ModerationMessage::PackFilesNotAllowed { files, incomplete } => {
(!incomplete || first_time)
&& files.values().any(|x| match x.status {
ApprovalType::Yes => false,
ApprovalType::WithAttributionAndSource => false,
ApprovalType::WithAttribution => false,
ApprovalType::No => first_time,
ApprovalType::PermanentNo => true,
ApprovalType::Unidentified => first_time,
})
}
ModerationMessage::MissingGalleryImage => true,
ModerationMessage::MissingLicense => true,
ModerationMessage::MissingCustomLicenseUrl { .. } => true,
ModerationMessage::NoSideTypes => true,
}
}
pub fn approvable(&self) -> bool {
match self {
ModerationMessage::NoPrimaryFile => false,
ModerationMessage::PackFilesNotAllowed { files, .. } => {
files.values().all(|x| x.status.approved())
}
ModerationMessage::MissingGalleryImage => false,
ModerationMessage::MissingLicense => false,
ModerationMessage::MissingCustomLicenseUrl { .. } => false,
ModerationMessage::NoSideTypes => false,
}
}
pub fn header(&self) -> &'static str {
match self {
ModerationMessage::NoPrimaryFile => "No primary files",
ModerationMessage::PackFilesNotAllowed { .. } => "Copyrighted Content",
ModerationMessage::MissingGalleryImage => "Missing Gallery Images",
ModerationMessage::MissingLicense => "Missing License",
ModerationMessage::MissingCustomLicenseUrl { .. } => "Missing License URL",
ModerationMessage::NoSideTypes => "Missing Environment Information",
}
}
pub fn body(&self) -> String {
match self {
ModerationMessage::NoPrimaryFile => "Please attach a file to this version. All files on Modrinth must have files associated with their versions.\n".to_string(),
ModerationMessage::PackFilesNotAllowed { files, .. } => {
let mut str = "".to_string();
str.push_str("This pack redistributes copyrighted material. Please refer to [Modrinth's guide on obtaining modpack permissions](https://docs.modrinth.com/modpacks/permissions) for more information.\n\n");
let mut attribute_mods = Vec::new();
let mut no_mods = Vec::new();
let mut permanent_no_mods = Vec::new();
let mut unidentified_mods = Vec::new();
for (_, approval) in files.iter() {
match approval.status {
ApprovalType::Yes | ApprovalType::WithAttributionAndSource => {}
ApprovalType::WithAttribution => attribute_mods.push(&approval.file_name),
ApprovalType::No => no_mods.push(&approval.file_name),
ApprovalType::PermanentNo => permanent_no_mods.push(&approval.file_name),
ApprovalType::Unidentified => unidentified_mods.push(&approval.file_name),
}
}
fn print_mods(projects: Vec<&String>, headline: &str, val: &mut String) {
if projects.is_empty() { return }
val.push_str(&format!("{headline}\n\n"));
for project in &projects {
let additional_text = if project.contains("ftb-quests") {
Some("Heracles")
} else if project.contains("ftb-ranks") || project.contains("ftb-essentials") {
Some("Prometheus")
} else if project.contains("ftb-teams") {
Some("Argonauts")
} else if project.contains("ftb-chunks") {
Some("Cadmus")
} else {
None
};
val.push_str(&if let Some(additional_text) = additional_text {
format!("- {project}(consider using [{additional_text}](https://modrinth.com/mod/{}) instead)\n", additional_text.to_lowercase())
} else {
format!("- {project}\n")
})
}
if !projects.is_empty() {
val.push('\n');
}
}
print_mods(attribute_mods, "The following content has attribution requirements, meaning that you must link back to the page where you originally found this content in your modpack description or version changelog (e.g. linking a mod's CurseForge page if you got it from CurseForge):", &mut str);
print_mods(no_mods, "The following content is not allowed in Modrinth modpacks due to licensing restrictions. Please contact the author(s) directly for permission or remove the content from your modpack:", &mut str);
print_mods(permanent_no_mods, "The following content is not allowed in Modrinth modpacks, regardless of permission obtained. This may be because it breaks Modrinth's content rules or because the authors, upon being contacted for permission, have declined. Please remove the content from your modpack:", &mut str);
print_mods(unidentified_mods, "The following content could not be identified. Please provide proof of its origin along with proof that you have permission to include it:", &mut str);
str
},
ModerationMessage::MissingGalleryImage => "We ask that resource packs like yours show off their content using images in the Gallery, or optionally in the Description, in order to effectively and clearly inform users of the content in your pack per section 2.1 of [Modrinth's content rules](https://modrinth.com/legal/rules#general-expectations).\n
Keep in mind that you should:\n
- Set a featured image that best represents your pack.
- Ensure all your images have titles that accurately label the image, and optionally, details on the contents of the image in the images Description.
- Upload any relevant images in your Description to your Gallery tab for best results.".to_string(),
ModerationMessage::MissingLicense => "You must select a License before your project can be published publicly, having a License associated with your project is important to protecting your rights and allowing others to use your content as you intend. For more information, you can see our [Guide to Licensing Mods](<https://blog.modrinth.com/licensing-guide/>).".to_string(),
ModerationMessage::MissingCustomLicenseUrl { license } => format!("It looks like you've selected the License \"{license}\" without providing a valid License link. When using a custom License you must provide a link directly to the License in the License Link field."),
ModerationMessage::NoSideTypes => "Your project's side types are currently set to Unknown on both sides. Please set accurate side types!".to_string(),
}
}
}
pub struct AutomatedModerationQueue {
pub projects: DashSet<ProjectId>,
}
impl Default for AutomatedModerationQueue {
fn default() -> Self {
Self {
projects: DashSet::new(),
}
}
}
impl AutomatedModerationQueue {
pub async fn task(&self, pool: PgPool, redis: RedisPool) {
loop {
let projects = self.projects.clone();
self.projects.clear();
for project in projects {
async {
let project =
database::Project::get_id((project).into(), &pool, &redis).await?;
if let Some(project) = project {
let res = async {
let mut mod_messages = ModerationMessages {
messages: vec![],
version_specific: HashMap::new(),
};
if project.project_types.iter().any(|x| ["mod", "modpack"].contains(&&**x)) && !project.aggregate_version_fields.iter().any(|x| ["server_only", "client_only", "client_and_server", "singleplayer"].contains(&&*x.field_name)) {
mod_messages.messages.push(ModerationMessage::NoSideTypes);
}
if project.inner.license == "LicenseRef-Unknown" || project.inner.license == "LicenseRef-" {
mod_messages.messages.push(ModerationMessage::MissingLicense);
} else if project.inner.license.starts_with("LicenseRef-") && project.inner.license != "LicenseRef-All-Rights-Reserved" && project.inner.license_url.is_none() {
mod_messages.messages.push(ModerationMessage::MissingCustomLicenseUrl { license: project.inner.license.clone() });
}
if (project.project_types.contains(&"resourcepack".to_string()) || project.project_types.contains(&"shader".to_string())) && project.gallery_items.is_empty() {
mod_messages.messages.push(ModerationMessage::MissingGalleryImage);
}
let versions =
database::Version::get_many(&project.versions, &pool, &redis)
.await?
.into_iter()
// we only support modpacks at this time
.filter(|x| x.project_types.contains(&"modpack".to_string()))
.collect::<Vec<_>>();
for version in versions {
let primary_file = version.files.iter().find_or_first(|x| x.primary);
if let Some(primary_file) = primary_file {
let data = reqwest::get(&primary_file.url).await?.bytes().await?;
let reader = Cursor::new(data);
let mut zip = ZipArchive::new(reader)?;
let pack: PackFormat = {
let mut file =
if let Ok(file) = zip.by_name("modrinth.index.json") {
file
} else {
continue;
};
let mut contents = String::new();
file.read_to_string(&mut contents)?;
serde_json::from_str(&contents)?
};
// sha1, pack file, file path, murmur
let mut hashes: Vec<(
String,
Option<PackFile>,
String,
Option<u32>,
)> = pack
.files
.clone()
.into_iter()
.flat_map(|x| {
let hash = x.hashes.get(&PackFileHash::Sha1);
if let Some(hash) = hash {
let path = x.path.clone();
Some((hash.clone(), Some(x), path, None))
} else {
None
}
})
.collect();
for i in 0..zip.len() {
let mut file = zip.by_index(i)?;
if file.name().starts_with("overrides/mods")
|| file.name().starts_with("client-overrides/mods")
|| file.name().starts_with("server-overrides/mods")
|| file.name().starts_with("overrides/shaderpacks")
|| file.name().starts_with("client-overrides/shaderpacks")
|| file.name().starts_with("overrides/resourcepacks")
|| file.name().starts_with("client-overrides/resourcepacks")
{
if file.name().matches('/').count() > 2 || file.name().ends_with(".txt") {
continue;
}
let mut contents = Vec::new();
file.read_to_end(&mut contents)?;
let hash = sha1::Sha1::from(&contents).hexdigest();
let murmur = hash_flame_murmur32(contents);
hashes.push((
hash,
None,
file.name().to_string(),
Some(murmur),
));
}
}
let files = database::models::Version::get_files_from_hash(
"sha1".to_string(),
&hashes.iter().map(|x| x.0.clone()).collect::<Vec<_>>(),
&pool,
&redis,
)
.await?;
let version_ids =
files.iter().map(|x| x.version_id).collect::<Vec<_>>();
let versions_data = filter_visible_versions(
database::models::Version::get_many(
&version_ids,
&pool,
&redis,
)
.await?,
&None,
&pool,
&redis,
)
.await?;
let mut final_hashes = HashMap::new();
for version in versions_data {
for file in
files.iter().filter(|x| x.version_id == version.id.into())
{
if let Some(hash) = file.hashes.get(&"sha1".to_string()) {
if let Some((index, (sha1, _, file_name, _))) = hashes
.iter()
.enumerate()
.find(|(_, (value, _, _, _))| value == hash)
{
final_hashes
.insert(sha1.clone(), IdentifiedFile { status: ApprovalType::Yes, file_name: file_name.clone() });
hashes.remove(index);
}
}
}
}
// All files are on Modrinth, so we don't send any messages
if hashes.is_empty() {
sqlx::query!(
"
UPDATE files
SET metadata = $1
WHERE id = $2
",
serde_json::to_value(&MissingMetadata {
identified: final_hashes,
flame_files: Default::default(),
unknown_files: Default::default(),
})?,
primary_file.id.0
)
.execute(&pool)
.await?;
continue;
}
let rows = sqlx::query!(
"
SELECT encode(mef.sha1, 'escape') sha1, mel.status status
FROM moderation_external_files mef
INNER JOIN moderation_external_licenses mel ON mef.external_license_id = mel.id
WHERE mef.sha1 = ANY($1)
",
&hashes.iter().map(|x| x.0.as_bytes().to_vec()).collect::<Vec<_>>()
)
.fetch_all(&pool)
.await?;
for row in rows {
if let Some(sha1) = row.sha1 {
if let Some((index, (sha1, _, file_name, _))) = hashes.iter().enumerate().find(|(_, (value, _, _, _))| value == &sha1) {
final_hashes.insert(sha1.clone(), IdentifiedFile { file_name: file_name.clone(), status: ApprovalType::from_string(&row.status).unwrap_or(ApprovalType::Unidentified) });
hashes.remove(index);
}
}
}
if hashes.is_empty() {
let metadata = MissingMetadata {
identified: final_hashes,
flame_files: Default::default(),
unknown_files: Default::default(),
};
sqlx::query!(
"
UPDATE files
SET metadata = $1
WHERE id = $2
",
serde_json::to_value(&metadata)?,
primary_file.id.0
)
.execute(&pool)
.await?;
if metadata.identified.values().any(|x| x.status != ApprovalType::Yes && x.status != ApprovalType::WithAttributionAndSource) {
let val = mod_messages.version_specific.entry(version.inner.version_number).or_default();
val.push(ModerationMessage::PackFilesNotAllowed {files: metadata.identified, incomplete: false });
}
continue;
}
let client = reqwest::Client::new();
let res = client
.post(format!("{}/v1/fingerprints", dotenvy::var("FLAME_ANVIL_URL")?))
.json(&serde_json::json!({
"fingerprints": hashes.iter().filter_map(|x| x.3).collect::<Vec<u32>>()
}))
.send()
.await?.text()
.await?;
let flame_hashes = serde_json::from_str::<FlameResponse<FingerprintResponse>>(&res)?
.data
.exact_matches
.into_iter()
.map(|x| x.file)
.collect::<Vec<_>>();
let mut flame_files = Vec::new();
for file in flame_hashes {
let hash = file
.hashes
.iter()
.find(|x| x.algo == 1)
.map(|x| x.value.clone());
if let Some(hash) = hash {
flame_files.push((hash, file.mod_id))
}
}
let rows = sqlx::query!(
"
SELECT mel.id, mel.flame_project_id, mel.status status
FROM moderation_external_licenses mel
WHERE mel.flame_project_id = ANY($1)
",
&flame_files.iter().map(|x| x.1 as i32).collect::<Vec<_>>()
)
.fetch_all(&pool).await?;
let mut insert_hashes = Vec::new();
let mut insert_ids = Vec::new();
for row in rows {
if let Some((curse_index, (hash, _flame_id))) = flame_files.iter().enumerate().find(|(_, x)| Some(x.1 as i32) == row.flame_project_id) {
if let Some((index, (sha1, _, file_name, _))) = hashes.iter().enumerate().find(|(_, (value, _, _, _))| value == hash) {
final_hashes.insert(sha1.clone(), IdentifiedFile {
file_name: file_name.clone(),
status: ApprovalType::from_string(&row.status).unwrap_or(ApprovalType::Unidentified),
});
insert_hashes.push(hash.clone().as_bytes().to_vec());
insert_ids.push(row.id);
hashes.remove(index);
flame_files.remove(curse_index);
}
}
}
if !insert_ids.is_empty() && !insert_hashes.is_empty() {
sqlx::query!(
"
INSERT INTO moderation_external_files (sha1, external_license_id)
SELECT * FROM UNNEST ($1::bytea[], $2::bigint[])
",
&insert_hashes[..],
&insert_ids[..]
)
.execute(&pool)
.await?;
}
if hashes.is_empty() {
let metadata = MissingMetadata {
identified: final_hashes,
flame_files: Default::default(),
unknown_files: Default::default(),
};
sqlx::query!(
"
UPDATE files
SET metadata = $1
WHERE id = $2
",
serde_json::to_value(&metadata)?,
primary_file.id.0
)
.execute(&pool)
.await?;
if metadata.identified.values().any(|x| x.status != ApprovalType::Yes && x.status != ApprovalType::WithAttributionAndSource) {
let val = mod_messages.version_specific.entry(version.inner.version_number).or_default();
val.push(ModerationMessage::PackFilesNotAllowed {files: metadata.identified, incomplete: false });
}
continue;
}
let flame_projects = if flame_files.is_empty() {
Vec::new()
} else {
let res = client
.post(format!("{}v1/mods", dotenvy::var("FLAME_ANVIL_URL")?))
.json(&serde_json::json!({
"modIds": flame_files.iter().map(|x| x.1).collect::<Vec<_>>()
}))
.send()
.await?
.text()
.await?;
serde_json::from_str::<FlameResponse<Vec<FlameProject>>>(&res)?.data
};
let mut missing_metadata = MissingMetadata {
identified: final_hashes,
flame_files: HashMap::new(),
unknown_files: HashMap::new(),
};
for (sha1, _pack_file, file_name, _mumur2) in hashes {
let flame_file = flame_files.iter().find(|x| x.0 == sha1);
if let Some((_, flame_project_id)) = flame_file {
if let Some(project) = flame_projects.iter().find(|x| &x.id == flame_project_id) {
missing_metadata.flame_files.insert(sha1, MissingMetadataFlame {
title: project.name.clone(),
file_name,
url: project.links.website_url.clone(),
id: *flame_project_id,
});
continue;
}
}
missing_metadata.unknown_files.insert(sha1, file_name);
}
sqlx::query!(
"
UPDATE files
SET metadata = $1
WHERE id = $2
",
serde_json::to_value(&missing_metadata)?,
primary_file.id.0
)
.execute(&pool)
.await?;
if missing_metadata.identified.values().any(|x| x.status != ApprovalType::Yes && x.status != ApprovalType::WithAttributionAndSource) {
let val = mod_messages.version_specific.entry(version.inner.version_number).or_default();
val.push(ModerationMessage::PackFilesNotAllowed {files: missing_metadata.identified, incomplete: true });
}
} else {
let val = mod_messages.version_specific.entry(version.inner.version_number).or_default();
val.push(ModerationMessage::NoPrimaryFile);
}
}
if !mod_messages.is_empty() {
let first_time = database::models::Thread::get(project.thread_id, &pool).await?
.map(|x| x.messages.iter().all(|x| match x.body { MessageBody::Text { hide_identity, .. } => x.author_id == Some(database::models::UserId(AUTOMOD_ID)) || hide_identity, _ => true}))
.unwrap_or(true);
let mut transaction = pool.begin().await?;
let id = ThreadMessageBuilder {
author_id: Some(database::models::UserId(AUTOMOD_ID)),
body: MessageBody::Text {
body: mod_messages.markdown(true),
private: false,
hide_identity: false,
replying_to: None,
associated_images: vec![],
},
thread_id: project.thread_id,
}
.insert(&mut transaction)
.await?;
let members = database::models::TeamMember::get_from_team_full(
project.inner.team_id,
&pool,
&redis,
)
.await?;
if mod_messages.should_reject(first_time) {
ThreadMessageBuilder {
author_id: Some(database::models::UserId(AUTOMOD_ID)),
body: MessageBody::StatusChange {
new_status: ProjectStatus::Rejected,
old_status: project.inner.status,
},
thread_id: project.thread_id,
}
.insert(&mut transaction)
.await?;
NotificationBuilder {
body: NotificationBody::StatusChange {
project_id: project.inner.id.into(),
old_status: project.inner.status,
new_status: ProjectStatus::Rejected,
},
}
.insert_many(members.into_iter().map(|x| x.user_id).collect(), &mut transaction, &redis)
.await?;
if let Ok(webhook_url) = dotenvy::var("MODERATION_DISCORD_WEBHOOK") {
crate::util::webhook::send_discord_webhook(
project.inner.id.into(),
&pool,
&redis,
webhook_url,
Some(
format!(
"**[AutoMod]({}/user/AutoMod)** changed project status from **{}** to **Rejected**",
dotenvy::var("SITE_URL")?,
&project.inner.status.as_friendly_str(),
)
.to_string(),
),
)
.await
.ok();
}
sqlx::query!(
"
UPDATE mods
SET status = 'rejected'
WHERE id = $1
",
project.inner.id.0
)
.execute(&pool)
.await?;
database::models::Project::clear_cache(
project.inner.id,
project.inner.slug.clone(),
None,
&redis,
)
.await?;
} else {
NotificationBuilder {
body: NotificationBody::ModeratorMessage {
thread_id: project.thread_id.into(),
message_id: id.into(),
project_id: Some(project.inner.id.into()),
report_id: None,
},
}
.insert_many(
members.into_iter().map(|x| x.user_id).collect(),
&mut transaction,
&redis,
)
.await?;
}
transaction.commit().await?;
}
Ok::<(), ApiError>(())
}.await;
if let Err(err) = res {
let err = err.as_api_error();
let mut str = String::new();
str.push_str("## Internal AutoMod Error\n\n");
str.push_str(&format!("Error code: {}\n\n", err.error));
str.push_str(&format!("Error description: {}\n\n", err.description));
let mut transaction = pool.begin().await?;
ThreadMessageBuilder {
author_id: Some(database::models::UserId(AUTOMOD_ID)),
body: MessageBody::Text {
body: str,
private: true,
hide_identity: false,
replying_to: None,
associated_images: vec![],
},
thread_id: project.thread_id,
}
.insert(&mut transaction)
.await?;
transaction.commit().await?;
}
}
Ok::<(), ApiError>(())
}.await.ok();
}
tokio::time::sleep(Duration::from_secs(5)).await
}
}
}
#[derive(Serialize, Deserialize)]
pub struct MissingMetadata {
pub identified: HashMap<String, IdentifiedFile>,
pub flame_files: HashMap<String, MissingMetadataFlame>,
pub unknown_files: HashMap<String, String>,
}
#[derive(Serialize, Deserialize)]
pub struct IdentifiedFile {
pub file_name: String,
pub status: ApprovalType,
}
#[derive(Serialize, Deserialize)]
pub struct MissingMetadataFlame {
pub title: String,
pub file_name: String,
pub url: String,
pub id: u32,
}
#[derive(Deserialize, Serialize, Copy, Clone, PartialEq, Eq, Debug)]
#[serde(rename_all = "kebab-case")]
pub enum ApprovalType {
Yes,
WithAttributionAndSource,
WithAttribution,
No,
PermanentNo,
Unidentified,
}
impl ApprovalType {
fn approved(&self) -> bool {
match self {
ApprovalType::Yes => true,
ApprovalType::WithAttributionAndSource => true,
ApprovalType::WithAttribution => true,
ApprovalType::No => false,
ApprovalType::PermanentNo => false,
ApprovalType::Unidentified => false,
}
}
pub fn from_string(string: &str) -> Option<Self> {
match string {
"yes" => Some(ApprovalType::Yes),
"with-attribution-and-source" => Some(ApprovalType::WithAttributionAndSource),
"with-attribution" => Some(ApprovalType::WithAttribution),
"no" => Some(ApprovalType::No),
"permanent-no" => Some(ApprovalType::PermanentNo),
"unidentified" => Some(ApprovalType::Unidentified),
_ => None,
}
}
pub(crate) fn as_str(&self) -> &'static str {
match self {
ApprovalType::Yes => "yes",
ApprovalType::WithAttributionAndSource => "with-attribution-and-source",
ApprovalType::WithAttribution => "with-attribution",
ApprovalType::No => "no",
ApprovalType::PermanentNo => "permanent-no",
ApprovalType::Unidentified => "unidentified",
}
}
}
#[derive(Deserialize, Serialize)]
pub struct FlameResponse<T> {
pub data: T,
}
#[derive(Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct FingerprintResponse {
pub exact_matches: Vec<FingerprintMatch>,
}
#[derive(Deserialize, Serialize)]
pub struct FingerprintMatch {
pub id: u32,
pub file: FlameFile,
}
#[derive(Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct FlameFile {
pub id: u32,
pub mod_id: u32,
pub hashes: Vec<FlameFileHash>,
pub file_fingerprint: u32,
}
#[derive(Deserialize, Serialize, Debug)]
pub struct FlameFileHash {
pub value: String,
pub algo: u32,
}
#[derive(Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct FlameProject {
pub id: u32,
pub name: String,
pub slug: String,
pub links: FlameLinks,
}
#[derive(Deserialize, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct FlameLinks {
pub website_url: String,
}
fn hash_flame_murmur32(input: Vec<u8>) -> u32 {
murmur2::murmur2(
&input
.into_iter()
.filter(|x| *x != 9 && *x != 10 && *x != 13 && *x != 32)
.collect::<Vec<u8>>(),
1,
)
}

12
src/queue/payouts.rs
View File

@@ -627,10 +627,14 @@ pub async fn process_payout(
FROM mods m
INNER JOIN organizations o ON m.organization_id = o.id
INNER JOIN team_members tm on o.team_id = tm.team_id AND tm.accepted = TRUE
WHERE m.id = ANY($1) AND m.monetization_status = $2 AND m.organization_id IS NOT NULL
WHERE m.id = ANY($1) AND m.monetization_status = $2 AND m.status = ANY($3) AND m.organization_id IS NOT NULL
",
&project_ids,
MonetizationStatus::Monetized.as_str(),
&*crate::models::projects::ProjectStatus::iterator()
.filter(|x| !x.is_hidden())
.map(|x| x.to_string())
.collect::<Vec<String>>(),
)
.fetch_all(&mut *transaction)
.await?;
@@ -640,10 +644,14 @@ pub async fn process_payout(
SELECT m.id id, tm.user_id user_id, tm.payouts_split payouts_split
FROM mods m
INNER JOIN team_members tm on m.team_id = tm.team_id AND tm.accepted = TRUE
WHERE m.id = ANY($1) AND m.monetization_status = $2
WHERE m.id = ANY($1) AND m.monetization_status = $2 AND m.status = ANY($3)
",
&project_ids,
MonetizationStatus::Monetized.as_str(),
&*crate::models::projects::ProjectStatus::iterator()
.filter(|x| !x.is_hidden())
.map(|x| x.to_string())
.collect::<Vec<String>>(),
)
.fetch_all(&mut *transaction)
.await?;

4
src/ratelimit/errors.rs
View File

@@ -40,12 +40,12 @@ impl ResponseError for ARError {
response.insert_header(("x-ratelimit-reset", reset.to_string()));
response.json(ApiError {
error: "ratelimit_error",
description: &self.to_string(),
description: self.to_string(),
})
}
_ => actix_web::HttpResponse::build(self.status_code()).json(ApiError {
error: "ratelimit_error",
description: &self.to_string(),
description: self.to_string(),
}),
}
}

5
src/routes/internal/mod.rs
View File

@@ -1,5 +1,6 @@
pub(crate) mod admin;
pub mod flows;
pub mod moderation;
pub mod pats;
pub mod session;
@@ -12,10 +13,10 @@ pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
actix_web::web::scope("_internal")
.wrap(default_cors())
.configure(admin::config)
// TODO: write tests that catch these
.configure(oauth_clients::config)
.configure(session::config)
.configure(flows::config)
.configure(pats::config),
.configure(pats::config)
.configure(moderation::config),
);
}

313
src/routes/internal/moderation.rs Normal file
View File

@@ -0,0 +1,313 @@
use super::ApiError;
use crate::database;
use crate::database::redis::RedisPool;
use crate::models::ids::random_base62;
use crate::models::projects::ProjectStatus;
use crate::queue::moderation::{ApprovalType, IdentifiedFile, MissingMetadata};
use crate::queue::session::AuthQueue;
use crate::{auth::check_is_moderator_from_headers, models::pats::Scopes};
use actix_web::{web, HttpRequest, HttpResponse};
use serde::Deserialize;
use sqlx::PgPool;
use std::collections::HashMap;
pub fn config(cfg: &mut web::ServiceConfig) {
cfg.route("moderation/projects", web::get().to(get_projects));
cfg.route("moderation/project/{id}", web::get().to(get_project_meta));
cfg.route("moderation/project", web::post().to(set_project_meta));
}
#[derive(Deserialize)]
pub struct ResultCount {
#[serde(default = "default_count")]
pub count: i16,
}
fn default_count() -> i16 {
100
}
pub async fn get_projects(
req: HttpRequest,
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
count: web::Query<ResultCount>,
session_queue: web::Data<AuthQueue>,
) -> Result<HttpResponse, ApiError> {
check_is_moderator_from_headers(
&req,
&**pool,
&redis,
&session_queue,
Some(&[Scopes::PROJECT_READ]),
)
.await?;
use futures::stream::TryStreamExt;
let project_ids = sqlx::query!(
"
SELECT id FROM mods
WHERE status = $1
ORDER BY queued ASC
LIMIT $2;
",
ProjectStatus::Processing.as_str(),
count.count as i64
)
.fetch_many(&**pool)
.try_filter_map(|e| async { Ok(e.right().map(|m| database::models::ProjectId(m.id))) })
.try_collect::<Vec<database::models::ProjectId>>()
.await?;
let projects: Vec<_> = database::Project::get_many_ids(&project_ids, &**pool, &redis)
.await?
.into_iter()
.map(crate::models::projects::Project::from)
.collect();
Ok(HttpResponse::Ok().json(projects))
}
pub async fn get_project_meta(
req: HttpRequest,
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
info: web::Path<(String,)>,
) -> Result<HttpResponse, ApiError> {
check_is_moderator_from_headers(
&req,
&**pool,
&redis,
&session_queue,
Some(&[Scopes::PROJECT_READ]),
)
.await?;
let project_id = info.into_inner().0;
let project = database::models::Project::get(&project_id, &**pool, &redis).await?;
if let Some(project) = project {
let rows = sqlx::query!(
"
SELECT
f.metadata, v.id version_id
FROM versions v
INNER JOIN files f ON f.version_id = v.id
WHERE v.mod_id = $1
",
project.inner.id.0
)
.fetch_all(&**pool)
.await?;
let mut merged = MissingMetadata {
identified: HashMap::new(),
flame_files: HashMap::new(),
unknown_files: HashMap::new(),
};
let mut check_hashes = Vec::new();
let mut check_flames = Vec::new();
for row in rows {
if let Some(metadata) = row
.metadata
.and_then(|x| serde_json::from_value::<MissingMetadata>(x).ok())
{
merged.identified.extend(metadata.identified);
merged.flame_files.extend(metadata.flame_files);
merged.unknown_files.extend(metadata.unknown_files);
check_hashes.extend(merged.flame_files.keys().cloned());
check_hashes.extend(merged.unknown_files.keys().cloned());
check_flames.extend(merged.flame_files.values().map(|x| x.id as i32));
}
}
let rows = sqlx::query!(
"
SELECT encode(mef.sha1, 'escape') sha1, mel.status status
FROM moderation_external_files mef
INNER JOIN moderation_external_licenses mel ON mef.external_license_id = mel.id
WHERE mef.sha1 = ANY($1)
",
&check_hashes
.iter()
.map(|x| x.as_bytes().to_vec())
.collect::<Vec<_>>()
)
.fetch_all(&**pool)
.await?;
for row in rows {
if let Some(sha1) = row.sha1 {
if let Some(val) = merged.flame_files.remove(&sha1) {
merged.identified.insert(
sha1,
IdentifiedFile {
file_name: val.file_name,
status: ApprovalType::from_string(&row.status)
.unwrap_or(ApprovalType::Unidentified),
},
);
} else if let Some(val) = merged.unknown_files.remove(&sha1) {
merged.identified.insert(
sha1,
IdentifiedFile {
file_name: val,
status: ApprovalType::from_string(&row.status)
.unwrap_or(ApprovalType::Unidentified),
},
);
}
}
}
let rows = sqlx::query!(
"
SELECT mel.id, mel.flame_project_id, mel.status status
FROM moderation_external_licenses mel
WHERE mel.flame_project_id = ANY($1)
",
&check_flames,
)
.fetch_all(&**pool)
.await?;
for row in rows {
if let Some(sha1) = merged
.flame_files
.iter()
.find(|x| Some(x.1.id as i32) == row.flame_project_id)
.map(|x| x.0.clone())
{
if let Some(val) = merged.flame_files.remove(&sha1) {
merged.identified.insert(
sha1,
IdentifiedFile {
file_name: val.file_name.clone(),
status: ApprovalType::from_string(&row.status)
.unwrap_or(ApprovalType::Unidentified),
},
);
}
}
}
Ok(HttpResponse::Ok().json(merged))
} else {
Err(ApiError::NotFound)
}
}
#[derive(Deserialize)]
#[serde(tag = "type", rename_all = "snake_case")]
pub enum Judgement {
Flame {
id: i32,
status: ApprovalType,
link: String,
title: String,
},
Unknown {
status: ApprovalType,
proof: Option<String>,
link: Option<String>,
title: Option<String>,
},
}
pub async fn set_project_meta(
req: HttpRequest,
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
judgements: web::Json<HashMap<String, Judgement>>,
) -> Result<HttpResponse, ApiError> {
check_is_moderator_from_headers(
&req,
&**pool,
&redis,
&session_queue,
Some(&[Scopes::PROJECT_READ]),
)
.await?;
let mut transaction = pool.begin().await?;
let mut ids = Vec::new();
let mut titles = Vec::new();
let mut statuses = Vec::new();
let mut links = Vec::new();
let mut proofs = Vec::new();
let mut flame_ids = Vec::new();
let mut file_hashes = Vec::new();
for (hash, judgement) in judgements.0 {
let id = random_base62(8);
let (title, status, link, proof, flame_id) = match judgement {
Judgement::Flame {
id,
status,
link,
title,
} => (
Some(title),
status,
Some(link),
Some("See Flame page/license for permission".to_string()),
Some(id),
),
Judgement::Unknown {
status,
proof,
link,
title,
} => (title, status, link, proof, None),
};
ids.push(id as i64);
titles.push(title);
statuses.push(status.as_str());
links.push(link);
proofs.push(proof);
flame_ids.push(flame_id);
file_hashes.push(hash);
}
sqlx::query(
"
INSERT INTO moderation_external_licenses (id, title, status, link, proof, flame_project_id)
SELECT * FROM UNNEST ($1::bigint[], $2::varchar[], $3::varchar[], $4::varchar[], $5::varchar[], $6::integer[])
"
)
.bind(&ids[..])
.bind(&titles[..])
.bind(&statuses[..])
.bind(&links[..])
.bind(&proofs[..])
.bind(&flame_ids[..])
.execute(&mut *transaction)
.await?;
sqlx::query(
"
INSERT INTO moderation_external_files (sha1, external_license_id)
SELECT * FROM UNNEST ($1::bytea[], $2::bigint[])
ON CONFLICT (sha1)
DO NOTHING
",
)
.bind(&file_hashes[..])
.bind(&ids[..])
.execute(&mut *transaction)
.await?;
transaction.commit().await?;
Ok(HttpResponse::NoContent().finish())
}

70
src/routes/mod.rs
View File

@@ -123,10 +123,49 @@ pub enum ApiError {
Mail(#[from] crate::auth::email::MailError),
#[error("Error while rerouting request: {0}")]
Reroute(#[from] reqwest::Error),
#[error("Unable to read Zip Archive: {0}")]
Zip(#[from] zip::result::ZipError),
#[error("IO Error: {0}")]
Io(#[from] std::io::Error),
#[error("Resource not found")]
NotFound,
}
impl ApiError {
pub fn as_api_error<'a>(&self) -> crate::models::error::ApiError<'a> {
crate::models::error::ApiError {
error: match self {
ApiError::Env(..) => "environment_error",
ApiError::SqlxDatabase(..) => "database_error",
ApiError::Database(..) => "database_error",
ApiError::Authentication(..) => "unauthorized",
ApiError::CustomAuthentication(..) => "unauthorized",
ApiError::Xml(..) => "xml_error",
ApiError::Json(..) => "json_error",
ApiError::Search(..) => "search_error",
ApiError::Indexing(..) => "indexing_error",
ApiError::FileHosting(..) => "file_hosting_error",
ApiError::InvalidInput(..) => "invalid_input",
ApiError::Validation(..) => "invalid_input",
ApiError::Payments(..) => "payments_error",
ApiError::Discord(..) => "discord_error",
ApiError::Turnstile => "turnstile_error",
ApiError::Decoding(..) => "decoding_error",
ApiError::ImageParse(..) => "invalid_image",
ApiError::PasswordHashing(..) => "password_hashing_error",
ApiError::PasswordStrengthCheck(..) => "strength_check_error",
ApiError::Mail(..) => "mail_error",
ApiError::Clickhouse(..) => "clickhouse_error",
ApiError::Reroute(..) => "reroute_error",
ApiError::NotFound => "not_found",
ApiError::Zip(..) => "zip_error",
ApiError::Io(..) => "io_error",
},
description: self.to_string(),
}
}
}
impl actix_web::ResponseError for ApiError {
fn status_code(&self) -> StatusCode {
match self {
@@ -153,37 +192,12 @@ impl actix_web::ResponseError for ApiError {
ApiError::Mail(..) => StatusCode::INTERNAL_SERVER_ERROR,
ApiError::Reroute(..) => StatusCode::INTERNAL_SERVER_ERROR,
ApiError::NotFound => StatusCode::NOT_FOUND,
ApiError::Zip(..) => StatusCode::BAD_REQUEST,
ApiError::Io(..) => StatusCode::BAD_REQUEST,
}
}
fn error_response(&self) -> HttpResponse {
HttpResponse::build(self.status_code()).json(crate::models::error::ApiError {
error: match self {
ApiError::Env(..) => "environment_error",
ApiError::SqlxDatabase(..) => "database_error",
ApiError::Database(..) => "database_error",
ApiError::Authentication(..) => "unauthorized",
ApiError::CustomAuthentication(..) => "unauthorized",
ApiError::Xml(..) => "xml_error",
ApiError::Json(..) => "json_error",
ApiError::Search(..) => "search_error",
ApiError::Indexing(..) => "indexing_error",
ApiError::FileHosting(..) => "file_hosting_error",
ApiError::InvalidInput(..) => "invalid_input",
ApiError::Validation(..) => "invalid_input",
ApiError::Payments(..) => "payments_error",
ApiError::Discord(..) => "discord_error",
ApiError::Turnstile => "turnstile_error",
ApiError::Decoding(..) => "decoding_error",
ApiError::ImageParse(..) => "invalid_image",
ApiError::PasswordHashing(..) => "password_hashing_error",
ApiError::PasswordStrengthCheck(..) => "strength_check_error",
ApiError::Mail(..) => "mail_error",
ApiError::Clickhouse(..) => "clickhouse_error",
ApiError::Reroute(..) => "reroute_error",
ApiError::NotFound => "not_found",
},
description: &self.to_string(),
})
HttpResponse::build(self.status_code()).json(self.as_api_error())
}
}

2
src/routes/not_found.rs
View File

@@ -4,7 +4,7 @@ use actix_web::{HttpResponse, Responder};
pub async fn not_found() -> impl Responder {
let data = ApiError {
error: "not_found",
description: "the requested route does not exist",
description: "the requested route does not exist".to_string(),
};
HttpResponse::NotFound().json(data)

6
src/routes/v2/moderation.rs
View File

@@ -2,7 +2,7 @@ use super::ApiError;
use crate::models::projects::Project;
use crate::models::v2::projects::LegacyProject;
use crate::queue::session::AuthQueue;
use crate::routes::v3;
use crate::routes::internal;
use crate::{database::redis::RedisPool, routes::v2_reroute};
use actix_web::{get, web, HttpRequest, HttpResponse};
use serde::Deserialize;
@@ -30,11 +30,11 @@ pub async fn get_projects(
count: web::Query<ResultCount>,
session_queue: web::Data<AuthQueue>,
) -> Result<HttpResponse, ApiError> {
let response = v3::moderation::get_projects(
let response = internal::moderation::get_projects(
req,
pool.clone(),
redis.clone(),
web::Query(v3::moderation::ResultCount { count: count.count }),
web::Query(internal::moderation::ResultCount { count: count.count }),
session_queue,
)
.await

4
src/routes/v2/projects.rs
View File

@@ -7,6 +7,7 @@ use crate::models::projects::{
};
use crate::models::v2::projects::{DonationLink, LegacyProject, LegacySideType, LegacyVersion};
use crate::models::v2::search::LegacySearchResults;
use crate::queue::moderation::AutomatedModerationQueue;
use crate::queue::session::AuthQueue;
use crate::routes::v3::projects::ProjectIds;
use crate::routes::{v2_reroute, v3, ApiError};
@@ -380,6 +381,7 @@ pub struct EditProject {
}
#[patch("{id}")]
#[allow(clippy::too_many_arguments)]
pub async fn project_edit(
req: HttpRequest,
info: web::Path<(String,)>,
@@ -388,6 +390,7 @@ pub async fn project_edit(
new_project: web::Json<EditProject>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
moderation_queue: web::Data<AutomatedModerationQueue>,
) -> Result<HttpResponse, ApiError> {
let v2_new_project = new_project.into_inner();
let client_side = v2_new_project.client_side;
@@ -494,6 +497,7 @@ pub async fn project_edit(
web::Json(new_project),
redis.clone(),
session_queue.clone(),
moderation_queue,
)
.await
.or_else(v2_reroute::flatten_404_error)?;

2
src/routes/v3/mod.rs
View File

@@ -6,7 +6,6 @@ use serde_json::json;
pub mod analytics_get;
pub mod collections;
pub mod images;
pub mod moderation;
pub mod notifications;
pub mod organizations;
pub mod payouts;
@@ -31,7 +30,6 @@ pub fn config(cfg: &mut web::ServiceConfig) {
.configure(analytics_get::config)
.configure(collections::config)
.configure(images::config)
.configure(moderation::config)
.configure(notifications::config)
.configure(organizations::config)
.configure(project_creation::config)

65
src/routes/v3/moderation.rs
View File

@@ -1,65 +0,0 @@
use super::ApiError;
use crate::database;
use crate::database::redis::RedisPool;
use crate::models::projects::ProjectStatus;
use crate::queue::session::AuthQueue;
use crate::{auth::check_is_moderator_from_headers, models::pats::Scopes};
use actix_web::{web, HttpRequest, HttpResponse};
use serde::Deserialize;
use sqlx::PgPool;
pub fn config(cfg: &mut web::ServiceConfig) {
cfg.route("moderation/projects", web::get().to(get_projects));
}
#[derive(Deserialize)]
pub struct ResultCount {
#[serde(default = "default_count")]
pub count: i16,
}
fn default_count() -> i16 {
100
}
pub async fn get_projects(
req: HttpRequest,
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
count: web::Query<ResultCount>,
session_queue: web::Data<AuthQueue>,
) -> Result<HttpResponse, ApiError> {
check_is_moderator_from_headers(
&req,
&**pool,
&redis,
&session_queue,
Some(&[Scopes::PROJECT_READ]),
)
.await?;
use futures::stream::TryStreamExt;
let project_ids = sqlx::query!(
"
SELECT id FROM mods
WHERE status = $1
ORDER BY queued ASC
LIMIT $2;
",
ProjectStatus::Processing.as_str(),
count.count as i64
)
.fetch_many(&**pool)
.try_filter_map(|e| async { Ok(e.right().map(|m| database::models::ProjectId(m.id))) })
.try_collect::<Vec<database::models::ProjectId>>()
.await?;
let projects: Vec<_> = database::Project::get_many_ids(&project_ids, &**pool, &redis)
.await?
.into_iter()
.map(crate::models::projects::Project::from)
.collect();
Ok(HttpResponse::Ok().json(projects))
}

2
src/routes/v3/project_creation.rs
View File

@@ -137,7 +137,7 @@ impl actix_web::ResponseError for CreateError {
CreateError::ImageError(..) => "invalid_image",
CreateError::RerouteError(..) => "reroute_error",
},
description: &self.to_string(),
description: self.to_string(),
})
}
}

7
src/routes/v3/projects.rs
View File

@@ -20,6 +20,7 @@ use crate::models::projects::{
};
use crate::models::teams::ProjectPermissions;
use crate::models::threads::MessageBody;
use crate::queue::moderation::AutomatedModerationQueue;
use crate::queue::session::AuthQueue;
use crate::routes::ApiError;
use crate::search::indexing::remove_documents;
@@ -229,6 +230,7 @@ pub struct EditProject {
pub monetization_status: Option<MonetizationStatus>,
}
#[allow(clippy::too_many_arguments)]
pub async fn project_edit(
req: HttpRequest,
info: web::Path<(String,)>,
@@ -237,6 +239,7 @@ pub async fn project_edit(
new_project: web::Json<EditProject>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
moderation_queue: web::Data<AutomatedModerationQueue>,
) -> Result<HttpResponse, ApiError> {
let user = get_user_from_headers(
&req,
@@ -362,6 +365,10 @@ pub async fn project_edit(
)
.execute(&mut *transaction)
.await?;
moderation_queue
.projects
.insert(project_item.inner.id.into());
}
if status.is_approved() && !project_item.inner.status.is_approved() {

7
src/routes/v3/threads.rs
View File

@@ -379,6 +379,7 @@ pub async fn thread_send_message(
body,
replying_to,
private,
hide_identity,
..
} = &new_message.body
{
@@ -394,6 +395,12 @@ pub async fn thread_send_message(
));
}
if *hide_identity && !user.role.is_mod() {
return Err(ApiError::InvalidInput(
"You are not allowed to send masked messages!".to_string(),
));
}
if let Some(replying_to) = replying_to {
let thread_message =
database::models::ThreadMessage::get((*replying_to).into(), &**pool).await?;

2
src/search/mod.rs
View File

@@ -52,7 +52,7 @@ impl actix_web::ResponseError for SearchError {
SearchError::InvalidIndex(..) => "invalid_input",
SearchError::FormatError(..) => "invalid_input",
},
description: &self.to_string(),
description: self.to_string(),
})
}
}