From 04d834187b8b4ef1e888bc4f13c6bf61166f8877 Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Wed, 21 Feb 2024 16:24:21 -0700 Subject: [PATCH] Automatic moderation (#875) * Automatic moderation * finish * modpack fixes * fix unknown license msg * fix moderation issues --- .env | 2 + ...7b9eb1f24d4de1f81b80c4bd186427edc1399.json | 34 + ...19dbab6e78bd1789dc70f445578c779c7b995.json | 34 + ...0588b79a7787b9b3cbbd4f8968cd0d99ed49d.json | 15 + ...fb7890f726af2ff11da53f450a88c3dc5fc64.json | 28 + ...3a308fb32f7439a18c83d1e16d3e537a43e7d.json | 28 + ...a273edc5729633aeaa87f6239667d2f861e68.json | 14 + ...ded611ff58d94461404182942210492e8010.json} | 7 +- ...b3bd0bec55f38807eded9130b932929f2ebe.json} | 7 +- ...0ba391ced72b38be97d462cdfe60048e947db.json | 15 + ...3b118a0eaa26e2851bcc3f1920ae0140b48ae.json | 28 + ...d9fb58ded7171b21e91973d1f13c91eab9b37.json | 15 + Cargo.lock | 7 + Cargo.toml | 1 + .../20240131224610_moderation_packs.sql | 19 + .../20240221215354_moderation_pack_fixes.sql | 2 + src/auth/checks.rs | 27 +- src/auth/mod.rs | 2 +- src/auth/oauth/errors.rs | 2 +- src/database/models/ids.rs | 4 +- src/database/models/team_item.rs | 10 +- src/lib.rs | 57 +- src/models/error.rs | 2 +- src/models/v2/threads.rs | 4 + src/models/v3/pack.rs | 6 +- src/models/v3/projects.rs | 2 +- src/models/v3/threads.rs | 2 + src/queue/mod.rs | 1 + src/queue/moderation.rs | 879 ++++++++++++++++++ src/queue/payouts.rs | 12 +- src/ratelimit/errors.rs | 4 +- src/routes/internal/mod.rs | 5 +- src/routes/internal/moderation.rs | 313 +++++++ src/routes/mod.rs | 70 +- src/routes/not_found.rs | 2 +- src/routes/v2/moderation.rs | 6 +- src/routes/v2/projects.rs | 4 + src/routes/v3/mod.rs | 2 - src/routes/v3/moderation.rs | 65 -- src/routes/v3/project_creation.rs | 2 +- src/routes/v3/projects.rs | 7 + src/routes/v3/threads.rs | 7 + src/search/mod.rs | 2 +- 43 files changed, 1597 insertions(+), 158 deletions(-) create mode 100644 .sqlx/query-3151ef71738a1f0d097aa14967d7b9eb1f24d4de1f81b80c4bd186427edc1399.json create mode 100644 .sqlx/query-3c875a8a1c03432f258040c436e19dbab6e78bd1789dc70f445578c779c7b995.json create mode 100644 .sqlx/query-3d535886d8a239967e6556fb0cd0588b79a7787b9b3cbbd4f8968cd0d99ed49d.json create mode 100644 .sqlx/query-4cb9fe3dbb2cbfe30a49487f896fb7890f726af2ff11da53f450a88c3dc5fc64.json create mode 100644 .sqlx/query-6e4ff5010b19890e26867611a243a308fb32f7439a18c83d1e16d3e537a43e7d.json create mode 100644 .sqlx/query-95e17b2512494ffcbfe6278b87aa273edc5729633aeaa87f6239667d2f861e68.json rename .sqlx/{query-b768d9db6c785d6a701324ea746794d33e94121403163a774b6ef775640fd3d3.json => query-a8bfce13de871daf0bb1cf73b4c5ded611ff58d94461404182942210492e8010.json} (76%) rename .sqlx/{query-03cd8926d18aa8c11934fdc0da32ccbbbccf2527c523336f230c0e344c471a0f.json => query-b82d35429e009e515ae1e0332142b3bd0bec55f38807eded9130b932929f2ebe.json} (81%) create mode 100644 .sqlx/query-c2924fff035e92f7bd2279517310ba391ced72b38be97d462cdfe60048e947db.json create mode 100644 .sqlx/query-cc1f2f568a0ba1d285a95fd9b6e3b118a0eaa26e2851bcc3f1920ae0140b48ae.json create mode 100644 .sqlx/query-ccf57f9c1026927afc940a20ebad9fb58ded7171b21e91973d1f13c91eab9b37.json create mode 100644 migrations/20240131224610_moderation_packs.sql create mode 100644 migrations/20240221215354_moderation_pack_fixes.sql create mode 100644 src/queue/moderation.rs create mode 100644 src/routes/internal/moderation.rs delete mode 100644 src/routes/v3/moderation.rs diff --git a/.env b/.env index 21aa0d99..85f578ac 100644 --- a/.env +++ b/.env @@ -98,3 +98,5 @@ CLICKHOUSE_DATABASE=staging_ariadne MAXMIND_LICENSE_KEY=none PAYOUTS_BUDGET=100 + +FLAME_ANVIL_URL=none \ No newline at end of file diff --git a/.sqlx/query-3151ef71738a1f0d097aa14967d7b9eb1f24d4de1f81b80c4bd186427edc1399.json b/.sqlx/query-3151ef71738a1f0d097aa14967d7b9eb1f24d4de1f81b80c4bd186427edc1399.json new file mode 100644 index 00000000..057d8602 --- /dev/null +++ b/.sqlx/query-3151ef71738a1f0d097aa14967d7b9eb1f24d4de1f81b80c4bd186427edc1399.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT mel.id, mel.flame_project_id, mel.status status\n FROM moderation_external_licenses mel\n WHERE mel.flame_project_id = ANY($1)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "flame_project_id", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "status", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int4Array" + ] + }, + "nullable": [ + false, + true, + false + ] + }, + "hash": "3151ef71738a1f0d097aa14967d7b9eb1f24d4de1f81b80c4bd186427edc1399" +} diff --git a/.sqlx/query-3c875a8a1c03432f258040c436e19dbab6e78bd1789dc70f445578c779c7b995.json b/.sqlx/query-3c875a8a1c03432f258040c436e19dbab6e78bd1789dc70f445578c779c7b995.json new file mode 100644 index 00000000..5d919103 --- /dev/null +++ b/.sqlx/query-3c875a8a1c03432f258040c436e19dbab6e78bd1789dc70f445578c779c7b995.json @@ -0,0 +1,34 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT mel.id, mel.flame_project_id, mel.status status\n FROM moderation_external_licenses mel\n WHERE mel.flame_project_id = ANY($1)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int8" + }, + { + "ordinal": 1, + "name": "flame_project_id", + "type_info": "Int4" + }, + { + "ordinal": 2, + "name": "status", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "Int4Array" + ] + }, + "nullable": [ + false, + true, + false + ] + }, + "hash": "3c875a8a1c03432f258040c436e19dbab6e78bd1789dc70f445578c779c7b995" +} diff --git a/.sqlx/query-3d535886d8a239967e6556fb0cd0588b79a7787b9b3cbbd4f8968cd0d99ed49d.json b/.sqlx/query-3d535886d8a239967e6556fb0cd0588b79a7787b9b3cbbd4f8968cd0d99ed49d.json new file mode 100644 index 00000000..371132b5 --- /dev/null +++ b/.sqlx/query-3d535886d8a239967e6556fb0cd0588b79a7787b9b3cbbd4f8968cd0d99ed49d.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n INSERT INTO moderation_external_files (sha1, external_license_id)\n SELECT * FROM UNNEST ($1::bytea[], $2::bigint[])\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "ByteaArray", + "Int8Array" + ] + }, + "nullable": [] + }, + "hash": "3d535886d8a239967e6556fb0cd0588b79a7787b9b3cbbd4f8968cd0d99ed49d" +} diff --git a/.sqlx/query-4cb9fe3dbb2cbfe30a49487f896fb7890f726af2ff11da53f450a88c3dc5fc64.json b/.sqlx/query-4cb9fe3dbb2cbfe30a49487f896fb7890f726af2ff11da53f450a88c3dc5fc64.json new file mode 100644 index 00000000..0397073b --- /dev/null +++ b/.sqlx/query-4cb9fe3dbb2cbfe30a49487f896fb7890f726af2ff11da53f450a88c3dc5fc64.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT encode(mef.sha1, 'escape') sha1, mel.status status\n FROM moderation_external_files mef\n INNER JOIN moderation_external_licenses mel ON mef.external_license_id = mel.id\n WHERE mef.sha1 = ANY($1)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "sha1", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "status", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "ByteaArray" + ] + }, + "nullable": [ + null, + false + ] + }, + "hash": "4cb9fe3dbb2cbfe30a49487f896fb7890f726af2ff11da53f450a88c3dc5fc64" +} diff --git a/.sqlx/query-6e4ff5010b19890e26867611a243a308fb32f7439a18c83d1e16d3e537a43e7d.json b/.sqlx/query-6e4ff5010b19890e26867611a243a308fb32f7439a18c83d1e16d3e537a43e7d.json new file mode 100644 index 00000000..feafe67d --- /dev/null +++ b/.sqlx/query-6e4ff5010b19890e26867611a243a308fb32f7439a18c83d1e16d3e537a43e7d.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT encode(mef.sha1, 'escape') sha1, mel.status status\n FROM moderation_external_files mef\n INNER JOIN moderation_external_licenses mel ON mef.external_license_id = mel.id\n WHERE mef.sha1 = ANY($1)\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "sha1", + "type_info": "Text" + }, + { + "ordinal": 1, + "name": "status", + "type_info": "Text" + } + ], + "parameters": { + "Left": [ + "ByteaArray" + ] + }, + "nullable": [ + null, + false + ] + }, + "hash": "6e4ff5010b19890e26867611a243a308fb32f7439a18c83d1e16d3e537a43e7d" +} diff --git a/.sqlx/query-95e17b2512494ffcbfe6278b87aa273edc5729633aeaa87f6239667d2f861e68.json b/.sqlx/query-95e17b2512494ffcbfe6278b87aa273edc5729633aeaa87f6239667d2f861e68.json new file mode 100644 index 00000000..063c2e0e --- /dev/null +++ b/.sqlx/query-95e17b2512494ffcbfe6278b87aa273edc5729633aeaa87f6239667d2f861e68.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE mods\n SET status = 'rejected'\n WHERE id = $1\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [] + }, + "hash": "95e17b2512494ffcbfe6278b87aa273edc5729633aeaa87f6239667d2f861e68" +} diff --git a/.sqlx/query-b768d9db6c785d6a701324ea746794d33e94121403163a774b6ef775640fd3d3.json b/.sqlx/query-a8bfce13de871daf0bb1cf73b4c5ded611ff58d94461404182942210492e8010.json similarity index 76% rename from .sqlx/query-b768d9db6c785d6a701324ea746794d33e94121403163a774b6ef775640fd3d3.json rename to .sqlx/query-a8bfce13de871daf0bb1cf73b4c5ded611ff58d94461404182942210492e8010.json index 7d789042..f762fb0e 100644 --- a/.sqlx/query-b768d9db6c785d6a701324ea746794d33e94121403163a774b6ef775640fd3d3.json +++ b/.sqlx/query-a8bfce13de871daf0bb1cf73b4c5ded611ff58d94461404182942210492e8010.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT m.id id, tm.user_id user_id, tm.payouts_split payouts_split\n FROM mods m\n INNER JOIN team_members tm on m.team_id = tm.team_id AND tm.accepted = TRUE\n WHERE m.id = ANY($1) AND m.monetization_status = $2\n ", + "query": "\n SELECT m.id id, tm.user_id user_id, tm.payouts_split payouts_split\n FROM mods m\n INNER JOIN team_members tm on m.team_id = tm.team_id AND tm.accepted = TRUE\n WHERE m.id = ANY($1) AND m.monetization_status = $2 AND m.status = ANY($3)\n ", "describe": { "columns": [ { @@ -22,7 +22,8 @@ "parameters": { "Left": [ "Int8Array", - "Text" + "Text", + "TextArray" ] }, "nullable": [ @@ -31,5 +32,5 @@ false ] }, - "hash": "b768d9db6c785d6a701324ea746794d33e94121403163a774b6ef775640fd3d3" + "hash": "a8bfce13de871daf0bb1cf73b4c5ded611ff58d94461404182942210492e8010" } diff --git a/.sqlx/query-03cd8926d18aa8c11934fdc0da32ccbbbccf2527c523336f230c0e344c471a0f.json b/.sqlx/query-b82d35429e009e515ae1e0332142b3bd0bec55f38807eded9130b932929f2ebe.json similarity index 81% rename from .sqlx/query-03cd8926d18aa8c11934fdc0da32ccbbbccf2527c523336f230c0e344c471a0f.json rename to .sqlx/query-b82d35429e009e515ae1e0332142b3bd0bec55f38807eded9130b932929f2ebe.json index 6671362a..d78e5d15 100644 --- a/.sqlx/query-03cd8926d18aa8c11934fdc0da32ccbbbccf2527c523336f230c0e344c471a0f.json +++ b/.sqlx/query-b82d35429e009e515ae1e0332142b3bd0bec55f38807eded9130b932929f2ebe.json @@ -1,6 +1,6 @@ { "db_name": "PostgreSQL", - "query": "\n SELECT m.id id, tm.user_id user_id, tm.payouts_split payouts_split\n FROM mods m\n INNER JOIN organizations o ON m.organization_id = o.id\n INNER JOIN team_members tm on o.team_id = tm.team_id AND tm.accepted = TRUE\n WHERE m.id = ANY($1) AND m.monetization_status = $2 AND m.organization_id IS NOT NULL\n ", + "query": "\n SELECT m.id id, tm.user_id user_id, tm.payouts_split payouts_split\n FROM mods m\n INNER JOIN organizations o ON m.organization_id = o.id\n INNER JOIN team_members tm on o.team_id = tm.team_id AND tm.accepted = TRUE\n WHERE m.id = ANY($1) AND m.monetization_status = $2 AND m.status = ANY($3) AND m.organization_id IS NOT NULL\n ", "describe": { "columns": [ { @@ -22,7 +22,8 @@ "parameters": { "Left": [ "Int8Array", - "Text" + "Text", + "TextArray" ] }, "nullable": [ @@ -31,5 +32,5 @@ false ] }, - "hash": "03cd8926d18aa8c11934fdc0da32ccbbbccf2527c523336f230c0e344c471a0f" + "hash": "b82d35429e009e515ae1e0332142b3bd0bec55f38807eded9130b932929f2ebe" } diff --git a/.sqlx/query-c2924fff035e92f7bd2279517310ba391ced72b38be97d462cdfe60048e947db.json b/.sqlx/query-c2924fff035e92f7bd2279517310ba391ced72b38be97d462cdfe60048e947db.json new file mode 100644 index 00000000..fba958d5 --- /dev/null +++ b/.sqlx/query-c2924fff035e92f7bd2279517310ba391ced72b38be97d462cdfe60048e947db.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE files\n SET metadata = $1\n WHERE id = $2\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "c2924fff035e92f7bd2279517310ba391ced72b38be97d462cdfe60048e947db" +} diff --git a/.sqlx/query-cc1f2f568a0ba1d285a95fd9b6e3b118a0eaa26e2851bcc3f1920ae0140b48ae.json b/.sqlx/query-cc1f2f568a0ba1d285a95fd9b6e3b118a0eaa26e2851bcc3f1920ae0140b48ae.json new file mode 100644 index 00000000..953a6002 --- /dev/null +++ b/.sqlx/query-cc1f2f568a0ba1d285a95fd9b6e3b118a0eaa26e2851bcc3f1920ae0140b48ae.json @@ -0,0 +1,28 @@ +{ + "db_name": "PostgreSQL", + "query": "\n SELECT\n f.metadata, v.id version_id\n FROM versions v\n INNER JOIN files f ON f.version_id = v.id\n WHERE v.mod_id = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "metadata", + "type_info": "Jsonb" + }, + { + "ordinal": 1, + "name": "version_id", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [ + "Int8" + ] + }, + "nullable": [ + true, + false + ] + }, + "hash": "cc1f2f568a0ba1d285a95fd9b6e3b118a0eaa26e2851bcc3f1920ae0140b48ae" +} diff --git a/.sqlx/query-ccf57f9c1026927afc940a20ebad9fb58ded7171b21e91973d1f13c91eab9b37.json b/.sqlx/query-ccf57f9c1026927afc940a20ebad9fb58ded7171b21e91973d1f13c91eab9b37.json new file mode 100644 index 00000000..8b28b3d9 --- /dev/null +++ b/.sqlx/query-ccf57f9c1026927afc940a20ebad9fb58ded7171b21e91973d1f13c91eab9b37.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE files\n SET metadata = $1\n WHERE id = $2\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Jsonb", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "ccf57f9c1026927afc940a20ebad9fb58ded7171b21e91973d1f13c91eab9b37" +} diff --git a/Cargo.lock b/Cargo.lock index cbdd72c6..e32b9b09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2343,6 +2343,7 @@ dependencies = [ "log", "maxminddb", "meilisearch-sdk", + "murmur2", "rand", "rand_chacha", "redis", @@ -2695,6 +2696,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "murmur2" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb585ade2549a017db2e35978b77c319214fa4b37cede841e27954dd6e8f3ca8" + [[package]] name = "native-tls" version = "0.2.11" diff --git a/Cargo.toml b/Cargo.toml index 90da500a..d2deedac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -51,6 +51,7 @@ sha1 = { version = "0.6.1", features = ["std"] } sha2 = "0.9.9" hmac = "0.11.0" argon2 = { version = "0.5.0", features = ["std"] } +murmur2 = "0.1.0" bitflags = "2.4.0" hex = "0.4.3" zxcvbn = "2.2.2" diff --git a/migrations/20240131224610_moderation_packs.sql b/migrations/20240131224610_moderation_packs.sql new file mode 100644 index 00000000..49040ec5 --- /dev/null +++ b/migrations/20240131224610_moderation_packs.sql @@ -0,0 +1,19 @@ +CREATE TABLE moderation_external_licenses ( + id bigint PRIMARY KEY, + title text not null, + status text not null, + link text null, + exceptions text null, + proof text null, + flame_project_id integer null +); + +CREATE TABLE moderation_external_files ( + sha1 bytea PRIMARY KEY, + external_license_id bigint references moderation_external_licenses not null +); + +ALTER TABLE files ADD COLUMN metadata jsonb NULL; + +INSERT INTO users (id, username, name, email, avatar_url, bio, role, badges, balance) +VALUES (0, 'AutoMod', 'AutoMod', 'support@modrinth.com', 'https://cdn.modrinth.com/user/2REoufqX/6aabaf2d1fca2935662eca4ce451cd9775054c22.png', 'An automated account performing moderation utilities for Modrinth.', 'moderator', 0, 0) \ No newline at end of file diff --git a/migrations/20240221215354_moderation_pack_fixes.sql b/migrations/20240221215354_moderation_pack_fixes.sql new file mode 100644 index 00000000..67eff677 --- /dev/null +++ b/migrations/20240221215354_moderation_pack_fixes.sql @@ -0,0 +1,2 @@ +ALTER TABLE moderation_external_files ALTER COLUMN sha1 SET NOT NULL; +ALTER TABLE moderation_external_licenses ALTER COLUMN title DROP NOT NULL; diff --git a/src/auth/checks.rs b/src/auth/checks.rs index 8697f714..fe3a7878 100644 --- a/src/auth/checks.rs +++ b/src/auth/checks.rs @@ -6,7 +6,6 @@ use crate::database::redis::RedisPool; use crate::database::{models, Project, Version}; use crate::models::users::User; use crate::routes::ApiError; -use actix_web::web; use itertools::Itertools; use sqlx::PgPool; @@ -32,7 +31,7 @@ where pub async fn is_visible_project( project_data: &Project, user_option: &Option, - pool: &web::Data, + pool: &PgPool, hide_unlisted: bool, ) -> Result { filter_visible_project_ids(vec![project_data], user_option, pool, hide_unlisted) @@ -43,7 +42,7 @@ pub async fn is_visible_project( pub async fn is_team_member_project( project_data: &Project, user_option: &Option, - pool: &web::Data, + pool: &PgPool, ) -> Result { filter_enlisted_projects_ids(vec![project_data], user_option, pool) .await @@ -53,7 +52,7 @@ pub async fn is_team_member_project( pub async fn filter_visible_projects( mut projects: Vec, user_option: &Option, - pool: &web::Data, + pool: &PgPool, hide_unlisted: bool, ) -> Result, ApiError> { let filtered_project_ids = filter_visible_project_ids( @@ -76,7 +75,7 @@ pub async fn filter_visible_projects( pub async fn filter_visible_project_ids( projects: Vec<&Project>, user_option: &Option, - pool: &web::Data, + pool: &PgPool, hide_unlisted: bool, ) -> Result, ApiError> { let mut return_projects = Vec::new(); @@ -114,7 +113,7 @@ pub async fn filter_visible_project_ids( pub async fn filter_enlisted_projects_ids( projects: Vec<&Project>, user_option: &Option, - pool: &web::Data, + pool: &PgPool, ) -> Result, ApiError> { let mut return_projects = vec![]; @@ -142,7 +141,7 @@ pub async fn filter_enlisted_projects_ids( .collect::>(), user_id as database::models::ids::UserId, ) - .fetch_many(&***pool) + .fetch_many(pool) .try_for_each(|e| { if let Some(row) = e.right() { for x in projects.iter() { @@ -163,7 +162,7 @@ pub async fn filter_enlisted_projects_ids( pub async fn is_visible_version( version_data: &Version, user_option: &Option, - pool: &web::Data, + pool: &PgPool, redis: &RedisPool, ) -> Result { filter_visible_version_ids(vec![version_data], user_option, pool, redis) @@ -174,7 +173,7 @@ pub async fn is_visible_version( pub async fn is_team_member_version( version_data: &Version, user_option: &Option, - pool: &web::Data, + pool: &PgPool, redis: &RedisPool, ) -> Result { filter_enlisted_version_ids(vec![version_data], user_option, pool, redis) @@ -185,7 +184,7 @@ pub async fn is_team_member_version( pub async fn filter_visible_versions( mut versions: Vec, user_option: &Option, - pool: &web::Data, + pool: &PgPool, redis: &RedisPool, ) -> Result, ApiError> { let filtered_version_ids = filter_visible_version_ids( @@ -220,7 +219,7 @@ impl ValidateAuthorized for models::OAuthClient { pub async fn filter_visible_version_ids( versions: Vec<&Version>, user_option: &Option, - pool: &web::Data, + pool: &PgPool, redis: &RedisPool, ) -> Result, ApiError> { let mut return_versions = Vec::new(); @@ -233,7 +232,7 @@ pub async fn filter_visible_version_ids( // Get visible projects- ones we are allowed to see public versions for. let visible_project_ids = filter_visible_project_ids( - Project::get_many_ids(&project_ids, &***pool, redis) + Project::get_many_ids(&project_ids, pool, redis) .await? .iter() .map(|x| &x.inner) @@ -273,7 +272,7 @@ pub async fn filter_visible_version_ids( pub async fn filter_enlisted_version_ids( versions: Vec<&Version>, user_option: &Option, - pool: &web::Data, + pool: &PgPool, redis: &RedisPool, ) -> Result, ApiError> { let mut return_versions = Vec::new(); @@ -283,7 +282,7 @@ pub async fn filter_enlisted_version_ids( // Get enlisted projects- ones we are allowed to see hidden versions for. let authorized_project_ids = filter_enlisted_projects_ids( - Project::get_many_ids(&project_ids, &***pool, redis) + Project::get_many_ids(&project_ids, pool, redis) .await? .iter() .map(|x| &x.inner) diff --git a/src/auth/mod.rs b/src/auth/mod.rs index 305743c3..bd7ac0ef 100644 --- a/src/auth/mod.rs +++ b/src/auth/mod.rs @@ -72,7 +72,7 @@ impl actix_web::ResponseError for AuthenticationError { fn error_response(&self) -> HttpResponse { HttpResponse::build(self.status_code()).json(ApiError { error: self.error_name(), - description: &self.to_string(), + description: self.to_string(), }) } } diff --git a/src/auth/oauth/errors.rs b/src/auth/oauth/errors.rs index 72a65abb..744d507c 100644 --- a/src/auth/oauth/errors.rs +++ b/src/auth/oauth/errors.rs @@ -100,7 +100,7 @@ impl actix_web::ResponseError for OAuthError { } else { HttpResponse::build(self.status_code()).json(ApiError { error: &self.error_type.error_name(), - description: &self.error_type.to_string(), + description: self.error_type.to_string(), }) } } diff --git a/src/database/models/ids.rs b/src/database/models/ids.rs index 2b640c69..1adfac19 100644 --- a/src/database/models/ids.rs +++ b/src/database/models/ids.rs @@ -2,6 +2,8 @@ use super::DatabaseError; use crate::models::ids::base62_impl::to_base62; use crate::models::ids::{random_base62_rng, random_base62_rng_range}; use censor::Censor; +use rand::SeedableRng; +use rand_chacha::ChaCha20Rng; use serde::{Deserialize, Serialize}; use sqlx::sqlx_macros::Type; @@ -12,7 +14,7 @@ macro_rules! generate_ids { $vis async fn $function_name( con: &mut sqlx::Transaction<'_, sqlx::Postgres>, ) -> Result<$return_type, DatabaseError> { - let mut rng = rand::thread_rng(); + let mut rng = ChaCha20Rng::from_entropy(); let length = $id_length; let mut id = random_base62_rng(&mut rng, length); let mut retry_count = 0; diff --git a/src/database/models/team_item.rs b/src/database/models/team_item.rs index 2dd0a2f7..b6353d45 100644 --- a/src/database/models/team_item.rs +++ b/src/database/models/team_item.rs @@ -269,9 +269,13 @@ impl TeamMember { .try_collect::>() .await?; - for (id, members) in &teams.into_iter().group_by(|x| x.team_id) { - let mut members = members.collect::>(); - + for (id, mut members) in teams + .into_iter() + .group_by(|x| x.team_id) + .into_iter() + .map(|(key, group)| (key, group.collect::>())) + .collect::>() + { redis .set_serialized_to_json(TEAMS_NAMESPACE, id.0, &members, None) .await?; diff --git a/src/lib.rs b/src/lib.rs index cac75903..4c74d3ab 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,8 +14,9 @@ extern crate clickhouse as clickhouse_crate; use clickhouse_crate::Client; use util::cors::default_cors; +use crate::queue::moderation::AutomatedModerationQueue; use crate::{ - queue::payouts::process_payout, + // queue::payouts::process_payout, search::indexing::index_projects, util::env::{parse_strings_from_var, parse_var}, }; @@ -52,6 +53,7 @@ pub struct LabrinthConfig { pub payouts_queue: web::Data, pub analytics_queue: Arc, pub active_sockets: web::Data>, + pub automated_moderation_queue: web::Data, } pub fn app_setup( @@ -67,6 +69,17 @@ pub fn app_setup( dotenvy::var("BIND_ADDR").unwrap() ); + let automated_moderation_queue = web::Data::new(AutomatedModerationQueue::default()); + + let automated_moderation_queue_ref = automated_moderation_queue.clone(); + let pool_ref = pool.clone(); + let redis_pool_ref = redis_pool.clone(); + actix_rt::spawn(async move { + automated_moderation_queue_ref + .task(pool_ref, redis_pool_ref) + .await; + }); + let mut scheduler = scheduler::Scheduler::new(); // The interval in seconds at which the local database is indexed @@ -201,25 +214,25 @@ pub fn app_setup( }); } - { - let pool_ref = pool.clone(); - let redis_ref = redis_pool.clone(); - let client_ref = clickhouse.clone(); - scheduler.run(std::time::Duration::from_secs(60 * 60 * 6), move || { - let pool_ref = pool_ref.clone(); - let redis_ref = redis_ref.clone(); - let client_ref = client_ref.clone(); - - async move { - info!("Started running payouts"); - let result = process_payout(&pool_ref, &redis_ref, &client_ref).await; - if let Err(e) = result { - warn!("Payouts run failed: {:?}", e); - } - info!("Done running payouts"); - } - }); - } + // { + // let pool_ref = pool.clone(); + // let redis_ref = redis_pool.clone(); + // let client_ref = clickhouse.clone(); + // scheduler.run(std::time::Duration::from_secs(60 * 60 * 6), move || { + // let pool_ref = pool_ref.clone(); + // let redis_ref = redis_ref.clone(); + // let client_ref = client_ref.clone(); + // + // async move { + // info!("Started running payouts"); + // let result = process_payout(&pool_ref, &redis_ref, &client_ref).await; + // if let Err(e) = result { + // warn!("Payouts run failed: {:?}", e); + // } + // info!("Done running payouts"); + // } + // }); + // } let ip_salt = Pepper { pepper: models::ids::Base62Id(models::ids::random_base62(11)).to_string(), @@ -241,6 +254,7 @@ pub fn app_setup( payouts_queue, analytics_queue, active_sockets, + automated_moderation_queue, } } @@ -272,6 +286,7 @@ pub fn app_config(cfg: &mut web::ServiceConfig, labrinth_config: LabrinthConfig) .app_data(web::Data::new(labrinth_config.clickhouse.clone())) .app_data(web::Data::new(labrinth_config.maxmind.clone())) .app_data(labrinth_config.active_sockets.clone()) + .app_data(labrinth_config.automated_moderation_queue.clone()) .configure(routes::v2::config) .configure(routes::v3::config) .configure(routes::internal::config) @@ -397,5 +412,7 @@ pub fn check_env_vars() -> bool { failed |= check_var::("PAYOUTS_BUDGET"); + failed |= check_var::("FLAME_ANVIL_URL"); + failed } diff --git a/src/models/error.rs b/src/models/error.rs index 5ac3c607..28f737c1 100644 --- a/src/models/error.rs +++ b/src/models/error.rs @@ -4,5 +4,5 @@ use serde::{Deserialize, Serialize}; #[derive(Serialize, Deserialize)] pub struct ApiError<'a> { pub error: &'a str, - pub description: &'a str, + pub description: String, } diff --git a/src/models/v2/threads.rs b/src/models/v2/threads.rs index 70654b84..98200eed 100644 --- a/src/models/v2/threads.rs +++ b/src/models/v2/threads.rs @@ -30,6 +30,8 @@ pub enum LegacyMessageBody { body: String, #[serde(default)] private: bool, + #[serde(default)] + hide_identity: bool, replying_to: Option, #[serde(default)] associated_images: Vec, @@ -74,11 +76,13 @@ impl From for LegacyMessageBody { private, replying_to, associated_images, + hide_identity, } => LegacyMessageBody::Text { body, private, replying_to, associated_images, + hide_identity, }, crate::models::v3::threads::MessageBody::StatusChange { new_status, diff --git a/src/models/v3/pack.rs b/src/models/v3/pack.rs index c73def00..49e22ca3 100644 --- a/src/models/v3/pack.rs +++ b/src/models/v3/pack.rs @@ -18,7 +18,7 @@ pub struct PackFormat { pub dependencies: std::collections::HashMap, } -#[derive(Serialize, Deserialize, Validate, Eq, PartialEq, Debug)] +#[derive(Serialize, Deserialize, Validate, Eq, PartialEq, Debug, Clone)] #[serde(rename_all = "camelCase")] pub struct PackFile { pub path: String, @@ -54,7 +54,7 @@ fn validate_download_url(values: &[String]) -> Result<(), validator::ValidationE Ok(()) } -#[derive(Serialize, Deserialize, Eq, PartialEq, Hash, Debug)] +#[derive(Serialize, Deserialize, Eq, PartialEq, Hash, Debug, Clone)] #[serde(rename_all = "camelCase", from = "String")] pub enum PackFileHash { Sha1, @@ -72,7 +72,7 @@ impl From for PackFileHash { } } -#[derive(Serialize, Deserialize, Eq, PartialEq, Hash, Debug)] +#[derive(Serialize, Deserialize, Eq, PartialEq, Hash, Debug, Clone)] #[serde(rename_all = "camelCase")] pub enum EnvType { Client, diff --git a/src/models/v3/projects.rs b/src/models/v3/projects.rs index 5bb0710b..8e75d079 100644 --- a/src/models/v3/projects.rs +++ b/src/models/v3/projects.rs @@ -12,7 +12,7 @@ use serde::{Deserialize, Serialize}; use validator::Validate; /// The ID of a specific project, encoded as base62 for usage in the API -#[derive(Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Debug)] +#[derive(Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Debug, Hash)] #[serde(from = "Base62Id")] #[serde(into = "Base62Id")] pub struct ProjectId(pub u64); diff --git a/src/models/v3/threads.rs b/src/models/v3/threads.rs index 5a18de7f..c77c8ad4 100644 --- a/src/models/v3/threads.rs +++ b/src/models/v3/threads.rs @@ -41,6 +41,8 @@ pub enum MessageBody { body: String, #[serde(default)] private: bool, + #[serde(default)] + hide_identity: bool, replying_to: Option, #[serde(default)] associated_images: Vec, diff --git a/src/queue/mod.rs b/src/queue/mod.rs index 9501640b..7ccf81c0 100644 --- a/src/queue/mod.rs +++ b/src/queue/mod.rs @@ -1,5 +1,6 @@ pub mod analytics; pub mod maxmind; +pub mod moderation; pub mod payouts; pub mod session; pub mod socket; diff --git a/src/queue/moderation.rs b/src/queue/moderation.rs new file mode 100644 index 00000000..beeb51e3 --- /dev/null +++ b/src/queue/moderation.rs @@ -0,0 +1,879 @@ +use crate::auth::checks::filter_visible_versions; +use crate::database; +use crate::database::models::notification_item::NotificationBuilder; +use crate::database::models::thread_item::ThreadMessageBuilder; +use crate::database::redis::RedisPool; +use crate::models::ids::ProjectId; +use crate::models::notifications::NotificationBody; +use crate::models::pack::{PackFile, PackFileHash, PackFormat}; +use crate::models::projects::ProjectStatus; +use crate::models::threads::MessageBody; +use crate::routes::ApiError; +use dashmap::DashSet; +use itertools::Itertools; +use serde::{Deserialize, Serialize}; +use sqlx::PgPool; +use std::collections::HashMap; +use std::io::{Cursor, Read}; +use std::time::Duration; +use zip::ZipArchive; + +const AUTOMOD_ID: i64 = 0; + +pub struct ModerationMessages { + pub messages: Vec, + pub version_specific: HashMap>, +} + +impl ModerationMessages { + pub fn is_empty(&self) -> bool { + self.messages.is_empty() && self.version_specific.is_empty() + } + + pub fn markdown(&self, auto_mod: bool) -> String { + let mut str = "".to_string(); + + for message in &self.messages { + str.push_str(&format!("## {}\n", message.header())); + str.push_str(&format!("{}\n", message.body())); + str.push('\n'); + } + + for (version_num, messages) in &self.version_specific { + for message in messages { + str.push_str(&format!( + "## Version {}: {}\n", + version_num, + message.header() + )); + str.push_str(&format!("{}\n", message.body())); + str.push('\n'); + } + } + + if auto_mod { + str.push_str("
\n\n"); + str.push_str("🤖 This is an automated message generated by AutoMod (BETA). If you are facing issues, please [contact support](https://support.modrinth.com)."); + } + + str + } + + pub fn should_reject(&self, first_time: bool) -> bool { + self.messages.iter().any(|x| x.rejectable(first_time)) + || self + .version_specific + .values() + .any(|x| x.iter().any(|x| x.rejectable(first_time))) + } + + pub fn approvable(&self) -> bool { + self.messages.iter().all(|x| x.approvable()) + && self + .version_specific + .values() + .all(|x| x.iter().all(|x| x.approvable())) + } +} + +pub enum ModerationMessage { + MissingGalleryImage, + NoPrimaryFile, + NoSideTypes, + PackFilesNotAllowed { + files: HashMap, + incomplete: bool, + }, + MissingLicense, + MissingCustomLicenseUrl { + license: String, + }, +} + +impl ModerationMessage { + pub fn rejectable(&self, first_time: bool) -> bool { + match self { + ModerationMessage::NoPrimaryFile => true, + ModerationMessage::PackFilesNotAllowed { files, incomplete } => { + (!incomplete || first_time) + && files.values().any(|x| match x.status { + ApprovalType::Yes => false, + ApprovalType::WithAttributionAndSource => false, + ApprovalType::WithAttribution => false, + ApprovalType::No => first_time, + ApprovalType::PermanentNo => true, + ApprovalType::Unidentified => first_time, + }) + } + ModerationMessage::MissingGalleryImage => true, + ModerationMessage::MissingLicense => true, + ModerationMessage::MissingCustomLicenseUrl { .. } => true, + ModerationMessage::NoSideTypes => true, + } + } + + pub fn approvable(&self) -> bool { + match self { + ModerationMessage::NoPrimaryFile => false, + ModerationMessage::PackFilesNotAllowed { files, .. } => { + files.values().all(|x| x.status.approved()) + } + ModerationMessage::MissingGalleryImage => false, + ModerationMessage::MissingLicense => false, + ModerationMessage::MissingCustomLicenseUrl { .. } => false, + ModerationMessage::NoSideTypes => false, + } + } + + pub fn header(&self) -> &'static str { + match self { + ModerationMessage::NoPrimaryFile => "No primary files", + ModerationMessage::PackFilesNotAllowed { .. } => "Copyrighted Content", + ModerationMessage::MissingGalleryImage => "Missing Gallery Images", + ModerationMessage::MissingLicense => "Missing License", + ModerationMessage::MissingCustomLicenseUrl { .. } => "Missing License URL", + ModerationMessage::NoSideTypes => "Missing Environment Information", + } + } + + pub fn body(&self) -> String { + match self { + ModerationMessage::NoPrimaryFile => "Please attach a file to this version. All files on Modrinth must have files associated with their versions.\n".to_string(), + ModerationMessage::PackFilesNotAllowed { files, .. } => { + let mut str = "".to_string(); + str.push_str("This pack redistributes copyrighted material. Please refer to [Modrinth's guide on obtaining modpack permissions](https://docs.modrinth.com/modpacks/permissions) for more information.\n\n"); + + let mut attribute_mods = Vec::new(); + let mut no_mods = Vec::new(); + let mut permanent_no_mods = Vec::new(); + let mut unidentified_mods = Vec::new(); + for (_, approval) in files.iter() { + match approval.status { + ApprovalType::Yes | ApprovalType::WithAttributionAndSource => {} + ApprovalType::WithAttribution => attribute_mods.push(&approval.file_name), + ApprovalType::No => no_mods.push(&approval.file_name), + ApprovalType::PermanentNo => permanent_no_mods.push(&approval.file_name), + ApprovalType::Unidentified => unidentified_mods.push(&approval.file_name), + } + } + + fn print_mods(projects: Vec<&String>, headline: &str, val: &mut String) { + if projects.is_empty() { return } + + val.push_str(&format!("{headline}\n\n")); + + for project in &projects { + let additional_text = if project.contains("ftb-quests") { + Some("Heracles") + } else if project.contains("ftb-ranks") || project.contains("ftb-essentials") { + Some("Prometheus") + } else if project.contains("ftb-teams") { + Some("Argonauts") + } else if project.contains("ftb-chunks") { + Some("Cadmus") + } else { + None + }; + + val.push_str(&if let Some(additional_text) = additional_text { + format!("- {project}(consider using [{additional_text}](https://modrinth.com/mod/{}) instead)\n", additional_text.to_lowercase()) + } else { + format!("- {project}\n") + }) + } + + if !projects.is_empty() { + val.push('\n'); + } + } + + print_mods(attribute_mods, "The following content has attribution requirements, meaning that you must link back to the page where you originally found this content in your modpack description or version changelog (e.g. linking a mod's CurseForge page if you got it from CurseForge):", &mut str); + print_mods(no_mods, "The following content is not allowed in Modrinth modpacks due to licensing restrictions. Please contact the author(s) directly for permission or remove the content from your modpack:", &mut str); + print_mods(permanent_no_mods, "The following content is not allowed in Modrinth modpacks, regardless of permission obtained. This may be because it breaks Modrinth's content rules or because the authors, upon being contacted for permission, have declined. Please remove the content from your modpack:", &mut str); + print_mods(unidentified_mods, "The following content could not be identified. Please provide proof of its origin along with proof that you have permission to include it:", &mut str); + + str + }, + ModerationMessage::MissingGalleryImage => "We ask that resource packs like yours show off their content using images in the Gallery, or optionally in the Description, in order to effectively and clearly inform users of the content in your pack per section 2.1 of [Modrinth's content rules](https://modrinth.com/legal/rules#general-expectations).\n +Keep in mind that you should:\n +- Set a featured image that best represents your pack. +- Ensure all your images have titles that accurately label the image, and optionally, details on the contents of the image in the images Description. +- Upload any relevant images in your Description to your Gallery tab for best results.".to_string(), + ModerationMessage::MissingLicense => "You must select a License before your project can be published publicly, having a License associated with your project is important to protecting your rights and allowing others to use your content as you intend. For more information, you can see our [Guide to Licensing Mods]().".to_string(), + ModerationMessage::MissingCustomLicenseUrl { license } => format!("It looks like you've selected the License \"{license}\" without providing a valid License link. When using a custom License you must provide a link directly to the License in the License Link field."), + ModerationMessage::NoSideTypes => "Your project's side types are currently set to Unknown on both sides. Please set accurate side types!".to_string(), + } + } +} + +pub struct AutomatedModerationQueue { + pub projects: DashSet, +} + +impl Default for AutomatedModerationQueue { + fn default() -> Self { + Self { + projects: DashSet::new(), + } + } +} + +impl AutomatedModerationQueue { + pub async fn task(&self, pool: PgPool, redis: RedisPool) { + loop { + let projects = self.projects.clone(); + self.projects.clear(); + + for project in projects { + async { + let project = + database::Project::get_id((project).into(), &pool, &redis).await?; + + if let Some(project) = project { + let res = async { + let mut mod_messages = ModerationMessages { + messages: vec![], + version_specific: HashMap::new(), + }; + + if project.project_types.iter().any(|x| ["mod", "modpack"].contains(&&**x)) && !project.aggregate_version_fields.iter().any(|x| ["server_only", "client_only", "client_and_server", "singleplayer"].contains(&&*x.field_name)) { + mod_messages.messages.push(ModerationMessage::NoSideTypes); + } + + if project.inner.license == "LicenseRef-Unknown" || project.inner.license == "LicenseRef-" { + mod_messages.messages.push(ModerationMessage::MissingLicense); + } else if project.inner.license.starts_with("LicenseRef-") && project.inner.license != "LicenseRef-All-Rights-Reserved" && project.inner.license_url.is_none() { + mod_messages.messages.push(ModerationMessage::MissingCustomLicenseUrl { license: project.inner.license.clone() }); + } + + if (project.project_types.contains(&"resourcepack".to_string()) || project.project_types.contains(&"shader".to_string())) && project.gallery_items.is_empty() { + mod_messages.messages.push(ModerationMessage::MissingGalleryImage); + } + + let versions = + database::Version::get_many(&project.versions, &pool, &redis) + .await? + .into_iter() + // we only support modpacks at this time + .filter(|x| x.project_types.contains(&"modpack".to_string())) + .collect::>(); + + for version in versions { + let primary_file = version.files.iter().find_or_first(|x| x.primary); + + if let Some(primary_file) = primary_file { + let data = reqwest::get(&primary_file.url).await?.bytes().await?; + + let reader = Cursor::new(data); + let mut zip = ZipArchive::new(reader)?; + + let pack: PackFormat = { + let mut file = + if let Ok(file) = zip.by_name("modrinth.index.json") { + file + } else { + continue; + }; + + let mut contents = String::new(); + file.read_to_string(&mut contents)?; + + serde_json::from_str(&contents)? + }; + + // sha1, pack file, file path, murmur + let mut hashes: Vec<( + String, + Option, + String, + Option, + )> = pack + .files + .clone() + .into_iter() + .flat_map(|x| { + let hash = x.hashes.get(&PackFileHash::Sha1); + + if let Some(hash) = hash { + let path = x.path.clone(); + Some((hash.clone(), Some(x), path, None)) + } else { + None + } + }) + .collect(); + + for i in 0..zip.len() { + let mut file = zip.by_index(i)?; + + if file.name().starts_with("overrides/mods") + || file.name().starts_with("client-overrides/mods") + || file.name().starts_with("server-overrides/mods") + || file.name().starts_with("overrides/shaderpacks") + || file.name().starts_with("client-overrides/shaderpacks") + || file.name().starts_with("overrides/resourcepacks") + || file.name().starts_with("client-overrides/resourcepacks") + { + if file.name().matches('/').count() > 2 || file.name().ends_with(".txt") { + continue; + } + + let mut contents = Vec::new(); + file.read_to_end(&mut contents)?; + + let hash = sha1::Sha1::from(&contents).hexdigest(); + let murmur = hash_flame_murmur32(contents); + + hashes.push(( + hash, + None, + file.name().to_string(), + Some(murmur), + )); + } + } + + let files = database::models::Version::get_files_from_hash( + "sha1".to_string(), + &hashes.iter().map(|x| x.0.clone()).collect::>(), + &pool, + &redis, + ) + .await?; + + let version_ids = + files.iter().map(|x| x.version_id).collect::>(); + let versions_data = filter_visible_versions( + database::models::Version::get_many( + &version_ids, + &pool, + &redis, + ) + .await?, + &None, + &pool, + &redis, + ) + .await?; + + let mut final_hashes = HashMap::new(); + + for version in versions_data { + for file in + files.iter().filter(|x| x.version_id == version.id.into()) + { + if let Some(hash) = file.hashes.get(&"sha1".to_string()) { + if let Some((index, (sha1, _, file_name, _))) = hashes + .iter() + .enumerate() + .find(|(_, (value, _, _, _))| value == hash) + { + final_hashes + .insert(sha1.clone(), IdentifiedFile { status: ApprovalType::Yes, file_name: file_name.clone() }); + + hashes.remove(index); + } + } + } + } + + // All files are on Modrinth, so we don't send any messages + if hashes.is_empty() { + sqlx::query!( + " + UPDATE files + SET metadata = $1 + WHERE id = $2 + ", + serde_json::to_value(&MissingMetadata { + identified: final_hashes, + flame_files: Default::default(), + unknown_files: Default::default(), + })?, + primary_file.id.0 + ) + .execute(&pool) + .await?; + + continue; + } + + let rows = sqlx::query!( + " + SELECT encode(mef.sha1, 'escape') sha1, mel.status status + FROM moderation_external_files mef + INNER JOIN moderation_external_licenses mel ON mef.external_license_id = mel.id + WHERE mef.sha1 = ANY($1) + ", + &hashes.iter().map(|x| x.0.as_bytes().to_vec()).collect::>() + ) + .fetch_all(&pool) + .await?; + + for row in rows { + if let Some(sha1) = row.sha1 { + if let Some((index, (sha1, _, file_name, _))) = hashes.iter().enumerate().find(|(_, (value, _, _, _))| value == &sha1) { + final_hashes.insert(sha1.clone(), IdentifiedFile { file_name: file_name.clone(), status: ApprovalType::from_string(&row.status).unwrap_or(ApprovalType::Unidentified) }); + hashes.remove(index); + } + } + } + + if hashes.is_empty() { + let metadata = MissingMetadata { + identified: final_hashes, + flame_files: Default::default(), + unknown_files: Default::default(), + }; + + sqlx::query!( + " + UPDATE files + SET metadata = $1 + WHERE id = $2 + ", + serde_json::to_value(&metadata)?, + primary_file.id.0 + ) + .execute(&pool) + .await?; + + if metadata.identified.values().any(|x| x.status != ApprovalType::Yes && x.status != ApprovalType::WithAttributionAndSource) { + let val = mod_messages.version_specific.entry(version.inner.version_number).or_default(); + val.push(ModerationMessage::PackFilesNotAllowed {files: metadata.identified, incomplete: false }); + } + continue; + } + + let client = reqwest::Client::new(); + let res = client + .post(format!("{}/v1/fingerprints", dotenvy::var("FLAME_ANVIL_URL")?)) + .json(&serde_json::json!({ + "fingerprints": hashes.iter().filter_map(|x| x.3).collect::>() + })) + .send() + .await?.text() + .await?; + + let flame_hashes = serde_json::from_str::>(&res)? + .data + .exact_matches + .into_iter() + .map(|x| x.file) + .collect::>(); + + let mut flame_files = Vec::new(); + + for file in flame_hashes { + let hash = file + .hashes + .iter() + .find(|x| x.algo == 1) + .map(|x| x.value.clone()); + + if let Some(hash) = hash { + flame_files.push((hash, file.mod_id)) + } + } + + let rows = sqlx::query!( + " + SELECT mel.id, mel.flame_project_id, mel.status status + FROM moderation_external_licenses mel + WHERE mel.flame_project_id = ANY($1) + ", + &flame_files.iter().map(|x| x.1 as i32).collect::>() + ) + .fetch_all(&pool).await?; + + let mut insert_hashes = Vec::new(); + let mut insert_ids = Vec::new(); + + for row in rows { + if let Some((curse_index, (hash, _flame_id))) = flame_files.iter().enumerate().find(|(_, x)| Some(x.1 as i32) == row.flame_project_id) { + if let Some((index, (sha1, _, file_name, _))) = hashes.iter().enumerate().find(|(_, (value, _, _, _))| value == hash) { + final_hashes.insert(sha1.clone(), IdentifiedFile { + file_name: file_name.clone(), + status: ApprovalType::from_string(&row.status).unwrap_or(ApprovalType::Unidentified), + }); + + insert_hashes.push(hash.clone().as_bytes().to_vec()); + insert_ids.push(row.id); + + hashes.remove(index); + flame_files.remove(curse_index); + } + } + } + + if !insert_ids.is_empty() && !insert_hashes.is_empty() { + sqlx::query!( + " + INSERT INTO moderation_external_files (sha1, external_license_id) + SELECT * FROM UNNEST ($1::bytea[], $2::bigint[]) + ", + &insert_hashes[..], + &insert_ids[..] + ) + .execute(&pool) + .await?; + } + + if hashes.is_empty() { + let metadata = MissingMetadata { + identified: final_hashes, + flame_files: Default::default(), + unknown_files: Default::default(), + }; + + sqlx::query!( + " + UPDATE files + SET metadata = $1 + WHERE id = $2 + ", + serde_json::to_value(&metadata)?, + primary_file.id.0 + ) + .execute(&pool) + .await?; + + if metadata.identified.values().any(|x| x.status != ApprovalType::Yes && x.status != ApprovalType::WithAttributionAndSource) { + let val = mod_messages.version_specific.entry(version.inner.version_number).or_default(); + val.push(ModerationMessage::PackFilesNotAllowed {files: metadata.identified, incomplete: false }); + } + + continue; + } + + let flame_projects = if flame_files.is_empty() { + Vec::new() + } else { + let res = client + .post(format!("{}v1/mods", dotenvy::var("FLAME_ANVIL_URL")?)) + .json(&serde_json::json!({ + "modIds": flame_files.iter().map(|x| x.1).collect::>() + })) + .send() + .await? + .text() + .await?; + + serde_json::from_str::>>(&res)?.data + }; + + let mut missing_metadata = MissingMetadata { + identified: final_hashes, + flame_files: HashMap::new(), + unknown_files: HashMap::new(), + }; + + for (sha1, _pack_file, file_name, _mumur2) in hashes { + let flame_file = flame_files.iter().find(|x| x.0 == sha1); + + if let Some((_, flame_project_id)) = flame_file { + if let Some(project) = flame_projects.iter().find(|x| &x.id == flame_project_id) { + missing_metadata.flame_files.insert(sha1, MissingMetadataFlame { + title: project.name.clone(), + file_name, + url: project.links.website_url.clone(), + id: *flame_project_id, + }); + + continue; + } + } + + missing_metadata.unknown_files.insert(sha1, file_name); + } + + sqlx::query!( + " + UPDATE files + SET metadata = $1 + WHERE id = $2 + ", + serde_json::to_value(&missing_metadata)?, + primary_file.id.0 + ) + .execute(&pool) + .await?; + + if missing_metadata.identified.values().any(|x| x.status != ApprovalType::Yes && x.status != ApprovalType::WithAttributionAndSource) { + let val = mod_messages.version_specific.entry(version.inner.version_number).or_default(); + val.push(ModerationMessage::PackFilesNotAllowed {files: missing_metadata.identified, incomplete: true }); + } + } else { + let val = mod_messages.version_specific.entry(version.inner.version_number).or_default(); + val.push(ModerationMessage::NoPrimaryFile); + } + } + + if !mod_messages.is_empty() { + let first_time = database::models::Thread::get(project.thread_id, &pool).await? + .map(|x| x.messages.iter().all(|x| match x.body { MessageBody::Text { hide_identity, .. } => x.author_id == Some(database::models::UserId(AUTOMOD_ID)) || hide_identity, _ => true})) + .unwrap_or(true); + + let mut transaction = pool.begin().await?; + let id = ThreadMessageBuilder { + author_id: Some(database::models::UserId(AUTOMOD_ID)), + body: MessageBody::Text { + body: mod_messages.markdown(true), + private: false, + hide_identity: false, + replying_to: None, + associated_images: vec![], + }, + thread_id: project.thread_id, + } + .insert(&mut transaction) + .await?; + + let members = database::models::TeamMember::get_from_team_full( + project.inner.team_id, + &pool, + &redis, + ) + .await?; + + if mod_messages.should_reject(first_time) { + ThreadMessageBuilder { + author_id: Some(database::models::UserId(AUTOMOD_ID)), + body: MessageBody::StatusChange { + new_status: ProjectStatus::Rejected, + old_status: project.inner.status, + }, + thread_id: project.thread_id, + } + .insert(&mut transaction) + .await?; + + NotificationBuilder { + body: NotificationBody::StatusChange { + project_id: project.inner.id.into(), + old_status: project.inner.status, + new_status: ProjectStatus::Rejected, + }, + } + .insert_many(members.into_iter().map(|x| x.user_id).collect(), &mut transaction, &redis) + .await?; + + if let Ok(webhook_url) = dotenvy::var("MODERATION_DISCORD_WEBHOOK") { + crate::util::webhook::send_discord_webhook( + project.inner.id.into(), + &pool, + &redis, + webhook_url, + Some( + format!( + "**[AutoMod]({}/user/AutoMod)** changed project status from **{}** to **Rejected**", + dotenvy::var("SITE_URL")?, + &project.inner.status.as_friendly_str(), + ) + .to_string(), + ), + ) + .await + .ok(); + } + + sqlx::query!( + " + UPDATE mods + SET status = 'rejected' + WHERE id = $1 + ", + project.inner.id.0 + ) + .execute(&pool) + .await?; + + database::models::Project::clear_cache( + project.inner.id, + project.inner.slug.clone(), + None, + &redis, + ) + .await?; + } else { + NotificationBuilder { + body: NotificationBody::ModeratorMessage { + thread_id: project.thread_id.into(), + message_id: id.into(), + project_id: Some(project.inner.id.into()), + report_id: None, + }, + } + .insert_many( + members.into_iter().map(|x| x.user_id).collect(), + &mut transaction, + &redis, + ) + .await?; + } + + transaction.commit().await?; + } + + Ok::<(), ApiError>(()) + }.await; + + if let Err(err) = res { + let err = err.as_api_error(); + + let mut str = String::new(); + str.push_str("## Internal AutoMod Error\n\n"); + str.push_str(&format!("Error code: {}\n\n", err.error)); + str.push_str(&format!("Error description: {}\n\n", err.description)); + + let mut transaction = pool.begin().await?; + ThreadMessageBuilder { + author_id: Some(database::models::UserId(AUTOMOD_ID)), + body: MessageBody::Text { + body: str, + private: true, + hide_identity: false, + replying_to: None, + associated_images: vec![], + }, + thread_id: project.thread_id, + } + .insert(&mut transaction) + .await?; + transaction.commit().await?; + } + } + + Ok::<(), ApiError>(()) + }.await.ok(); + } + + tokio::time::sleep(Duration::from_secs(5)).await + } + } +} + +#[derive(Serialize, Deserialize)] +pub struct MissingMetadata { + pub identified: HashMap, + pub flame_files: HashMap, + pub unknown_files: HashMap, +} + +#[derive(Serialize, Deserialize)] +pub struct IdentifiedFile { + pub file_name: String, + pub status: ApprovalType, +} + +#[derive(Serialize, Deserialize)] +pub struct MissingMetadataFlame { + pub title: String, + pub file_name: String, + pub url: String, + pub id: u32, +} + +#[derive(Deserialize, Serialize, Copy, Clone, PartialEq, Eq, Debug)] +#[serde(rename_all = "kebab-case")] +pub enum ApprovalType { + Yes, + WithAttributionAndSource, + WithAttribution, + No, + PermanentNo, + Unidentified, +} + +impl ApprovalType { + fn approved(&self) -> bool { + match self { + ApprovalType::Yes => true, + ApprovalType::WithAttributionAndSource => true, + ApprovalType::WithAttribution => true, + ApprovalType::No => false, + ApprovalType::PermanentNo => false, + ApprovalType::Unidentified => false, + } + } + + pub fn from_string(string: &str) -> Option { + match string { + "yes" => Some(ApprovalType::Yes), + "with-attribution-and-source" => Some(ApprovalType::WithAttributionAndSource), + "with-attribution" => Some(ApprovalType::WithAttribution), + "no" => Some(ApprovalType::No), + "permanent-no" => Some(ApprovalType::PermanentNo), + "unidentified" => Some(ApprovalType::Unidentified), + _ => None, + } + } + + pub(crate) fn as_str(&self) -> &'static str { + match self { + ApprovalType::Yes => "yes", + ApprovalType::WithAttributionAndSource => "with-attribution-and-source", + ApprovalType::WithAttribution => "with-attribution", + ApprovalType::No => "no", + ApprovalType::PermanentNo => "permanent-no", + ApprovalType::Unidentified => "unidentified", + } + } +} + +#[derive(Deserialize, Serialize)] +pub struct FlameResponse { + pub data: T, +} + +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct FingerprintResponse { + pub exact_matches: Vec, +} + +#[derive(Deserialize, Serialize)] +pub struct FingerprintMatch { + pub id: u32, + pub file: FlameFile, +} + +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct FlameFile { + pub id: u32, + pub mod_id: u32, + pub hashes: Vec, + pub file_fingerprint: u32, +} + +#[derive(Deserialize, Serialize, Debug)] +pub struct FlameFileHash { + pub value: String, + pub algo: u32, +} + +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct FlameProject { + pub id: u32, + pub name: String, + pub slug: String, + pub links: FlameLinks, +} + +#[derive(Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct FlameLinks { + pub website_url: String, +} + +fn hash_flame_murmur32(input: Vec) -> u32 { + murmur2::murmur2( + &input + .into_iter() + .filter(|x| *x != 9 && *x != 10 && *x != 13 && *x != 32) + .collect::>(), + 1, + ) +} diff --git a/src/queue/payouts.rs b/src/queue/payouts.rs index d16988a9..53ba2005 100644 --- a/src/queue/payouts.rs +++ b/src/queue/payouts.rs @@ -627,10 +627,14 @@ pub async fn process_payout( FROM mods m INNER JOIN organizations o ON m.organization_id = o.id INNER JOIN team_members tm on o.team_id = tm.team_id AND tm.accepted = TRUE - WHERE m.id = ANY($1) AND m.monetization_status = $2 AND m.organization_id IS NOT NULL + WHERE m.id = ANY($1) AND m.monetization_status = $2 AND m.status = ANY($3) AND m.organization_id IS NOT NULL ", &project_ids, MonetizationStatus::Monetized.as_str(), + &*crate::models::projects::ProjectStatus::iterator() + .filter(|x| !x.is_hidden()) + .map(|x| x.to_string()) + .collect::>(), ) .fetch_all(&mut *transaction) .await?; @@ -640,10 +644,14 @@ pub async fn process_payout( SELECT m.id id, tm.user_id user_id, tm.payouts_split payouts_split FROM mods m INNER JOIN team_members tm on m.team_id = tm.team_id AND tm.accepted = TRUE - WHERE m.id = ANY($1) AND m.monetization_status = $2 + WHERE m.id = ANY($1) AND m.monetization_status = $2 AND m.status = ANY($3) ", &project_ids, MonetizationStatus::Monetized.as_str(), + &*crate::models::projects::ProjectStatus::iterator() + .filter(|x| !x.is_hidden()) + .map(|x| x.to_string()) + .collect::>(), ) .fetch_all(&mut *transaction) .await?; diff --git a/src/ratelimit/errors.rs b/src/ratelimit/errors.rs index ef103117..f06ba48c 100644 --- a/src/ratelimit/errors.rs +++ b/src/ratelimit/errors.rs @@ -40,12 +40,12 @@ impl ResponseError for ARError { response.insert_header(("x-ratelimit-reset", reset.to_string())); response.json(ApiError { error: "ratelimit_error", - description: &self.to_string(), + description: self.to_string(), }) } _ => actix_web::HttpResponse::build(self.status_code()).json(ApiError { error: "ratelimit_error", - description: &self.to_string(), + description: self.to_string(), }), } } diff --git a/src/routes/internal/mod.rs b/src/routes/internal/mod.rs index 81ac4c9b..5c1d782a 100644 --- a/src/routes/internal/mod.rs +++ b/src/routes/internal/mod.rs @@ -1,5 +1,6 @@ pub(crate) mod admin; pub mod flows; +pub mod moderation; pub mod pats; pub mod session; @@ -12,10 +13,10 @@ pub fn config(cfg: &mut actix_web::web::ServiceConfig) { actix_web::web::scope("_internal") .wrap(default_cors()) .configure(admin::config) - // TODO: write tests that catch these .configure(oauth_clients::config) .configure(session::config) .configure(flows::config) - .configure(pats::config), + .configure(pats::config) + .configure(moderation::config), ); } diff --git a/src/routes/internal/moderation.rs b/src/routes/internal/moderation.rs new file mode 100644 index 00000000..0918c638 --- /dev/null +++ b/src/routes/internal/moderation.rs @@ -0,0 +1,313 @@ +use super::ApiError; +use crate::database; +use crate::database::redis::RedisPool; +use crate::models::ids::random_base62; +use crate::models::projects::ProjectStatus; +use crate::queue::moderation::{ApprovalType, IdentifiedFile, MissingMetadata}; +use crate::queue::session::AuthQueue; +use crate::{auth::check_is_moderator_from_headers, models::pats::Scopes}; +use actix_web::{web, HttpRequest, HttpResponse}; +use serde::Deserialize; +use sqlx::PgPool; +use std::collections::HashMap; + +pub fn config(cfg: &mut web::ServiceConfig) { + cfg.route("moderation/projects", web::get().to(get_projects)); + cfg.route("moderation/project/{id}", web::get().to(get_project_meta)); + cfg.route("moderation/project", web::post().to(set_project_meta)); +} + +#[derive(Deserialize)] +pub struct ResultCount { + #[serde(default = "default_count")] + pub count: i16, +} + +fn default_count() -> i16 { + 100 +} + +pub async fn get_projects( + req: HttpRequest, + pool: web::Data, + redis: web::Data, + count: web::Query, + session_queue: web::Data, +) -> Result { + check_is_moderator_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Some(&[Scopes::PROJECT_READ]), + ) + .await?; + + use futures::stream::TryStreamExt; + + let project_ids = sqlx::query!( + " + SELECT id FROM mods + WHERE status = $1 + ORDER BY queued ASC + LIMIT $2; + ", + ProjectStatus::Processing.as_str(), + count.count as i64 + ) + .fetch_many(&**pool) + .try_filter_map(|e| async { Ok(e.right().map(|m| database::models::ProjectId(m.id))) }) + .try_collect::>() + .await?; + + let projects: Vec<_> = database::Project::get_many_ids(&project_ids, &**pool, &redis) + .await? + .into_iter() + .map(crate::models::projects::Project::from) + .collect(); + + Ok(HttpResponse::Ok().json(projects)) +} + +pub async fn get_project_meta( + req: HttpRequest, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, + info: web::Path<(String,)>, +) -> Result { + check_is_moderator_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Some(&[Scopes::PROJECT_READ]), + ) + .await?; + + let project_id = info.into_inner().0; + let project = database::models::Project::get(&project_id, &**pool, &redis).await?; + + if let Some(project) = project { + let rows = sqlx::query!( + " + SELECT + f.metadata, v.id version_id + FROM versions v + INNER JOIN files f ON f.version_id = v.id + WHERE v.mod_id = $1 + ", + project.inner.id.0 + ) + .fetch_all(&**pool) + .await?; + + let mut merged = MissingMetadata { + identified: HashMap::new(), + flame_files: HashMap::new(), + unknown_files: HashMap::new(), + }; + + let mut check_hashes = Vec::new(); + let mut check_flames = Vec::new(); + + for row in rows { + if let Some(metadata) = row + .metadata + .and_then(|x| serde_json::from_value::(x).ok()) + { + merged.identified.extend(metadata.identified); + merged.flame_files.extend(metadata.flame_files); + merged.unknown_files.extend(metadata.unknown_files); + + check_hashes.extend(merged.flame_files.keys().cloned()); + check_hashes.extend(merged.unknown_files.keys().cloned()); + check_flames.extend(merged.flame_files.values().map(|x| x.id as i32)); + } + } + + let rows = sqlx::query!( + " + SELECT encode(mef.sha1, 'escape') sha1, mel.status status + FROM moderation_external_files mef + INNER JOIN moderation_external_licenses mel ON mef.external_license_id = mel.id + WHERE mef.sha1 = ANY($1) + ", + &check_hashes + .iter() + .map(|x| x.as_bytes().to_vec()) + .collect::>() + ) + .fetch_all(&**pool) + .await?; + + for row in rows { + if let Some(sha1) = row.sha1 { + if let Some(val) = merged.flame_files.remove(&sha1) { + merged.identified.insert( + sha1, + IdentifiedFile { + file_name: val.file_name, + status: ApprovalType::from_string(&row.status) + .unwrap_or(ApprovalType::Unidentified), + }, + ); + } else if let Some(val) = merged.unknown_files.remove(&sha1) { + merged.identified.insert( + sha1, + IdentifiedFile { + file_name: val, + status: ApprovalType::from_string(&row.status) + .unwrap_or(ApprovalType::Unidentified), + }, + ); + } + } + } + + let rows = sqlx::query!( + " + SELECT mel.id, mel.flame_project_id, mel.status status + FROM moderation_external_licenses mel + WHERE mel.flame_project_id = ANY($1) + ", + &check_flames, + ) + .fetch_all(&**pool) + .await?; + + for row in rows { + if let Some(sha1) = merged + .flame_files + .iter() + .find(|x| Some(x.1.id as i32) == row.flame_project_id) + .map(|x| x.0.clone()) + { + if let Some(val) = merged.flame_files.remove(&sha1) { + merged.identified.insert( + sha1, + IdentifiedFile { + file_name: val.file_name.clone(), + status: ApprovalType::from_string(&row.status) + .unwrap_or(ApprovalType::Unidentified), + }, + ); + } + } + } + + Ok(HttpResponse::Ok().json(merged)) + } else { + Err(ApiError::NotFound) + } +} + +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum Judgement { + Flame { + id: i32, + status: ApprovalType, + link: String, + title: String, + }, + Unknown { + status: ApprovalType, + proof: Option, + link: Option, + title: Option, + }, +} + +pub async fn set_project_meta( + req: HttpRequest, + pool: web::Data, + redis: web::Data, + session_queue: web::Data, + judgements: web::Json>, +) -> Result { + check_is_moderator_from_headers( + &req, + &**pool, + &redis, + &session_queue, + Some(&[Scopes::PROJECT_READ]), + ) + .await?; + + let mut transaction = pool.begin().await?; + + let mut ids = Vec::new(); + let mut titles = Vec::new(); + let mut statuses = Vec::new(); + let mut links = Vec::new(); + let mut proofs = Vec::new(); + let mut flame_ids = Vec::new(); + + let mut file_hashes = Vec::new(); + + for (hash, judgement) in judgements.0 { + let id = random_base62(8); + + let (title, status, link, proof, flame_id) = match judgement { + Judgement::Flame { + id, + status, + link, + title, + } => ( + Some(title), + status, + Some(link), + Some("See Flame page/license for permission".to_string()), + Some(id), + ), + Judgement::Unknown { + status, + proof, + link, + title, + } => (title, status, link, proof, None), + }; + + ids.push(id as i64); + titles.push(title); + statuses.push(status.as_str()); + links.push(link); + proofs.push(proof); + flame_ids.push(flame_id); + file_hashes.push(hash); + } + + sqlx::query( + " + INSERT INTO moderation_external_licenses (id, title, status, link, proof, flame_project_id) + SELECT * FROM UNNEST ($1::bigint[], $2::varchar[], $3::varchar[], $4::varchar[], $5::varchar[], $6::integer[]) + " + ) + .bind(&ids[..]) + .bind(&titles[..]) + .bind(&statuses[..]) + .bind(&links[..]) + .bind(&proofs[..]) + .bind(&flame_ids[..]) + .execute(&mut *transaction) + .await?; + + sqlx::query( + " + INSERT INTO moderation_external_files (sha1, external_license_id) + SELECT * FROM UNNEST ($1::bytea[], $2::bigint[]) + ON CONFLICT (sha1) + DO NOTHING + ", + ) + .bind(&file_hashes[..]) + .bind(&ids[..]) + .execute(&mut *transaction) + .await?; + + transaction.commit().await?; + + Ok(HttpResponse::NoContent().finish()) +} diff --git a/src/routes/mod.rs b/src/routes/mod.rs index 18581eaa..1b77062f 100644 --- a/src/routes/mod.rs +++ b/src/routes/mod.rs @@ -123,10 +123,49 @@ pub enum ApiError { Mail(#[from] crate::auth::email::MailError), #[error("Error while rerouting request: {0}")] Reroute(#[from] reqwest::Error), + #[error("Unable to read Zip Archive: {0}")] + Zip(#[from] zip::result::ZipError), + #[error("IO Error: {0}")] + Io(#[from] std::io::Error), #[error("Resource not found")] NotFound, } +impl ApiError { + pub fn as_api_error<'a>(&self) -> crate::models::error::ApiError<'a> { + crate::models::error::ApiError { + error: match self { + ApiError::Env(..) => "environment_error", + ApiError::SqlxDatabase(..) => "database_error", + ApiError::Database(..) => "database_error", + ApiError::Authentication(..) => "unauthorized", + ApiError::CustomAuthentication(..) => "unauthorized", + ApiError::Xml(..) => "xml_error", + ApiError::Json(..) => "json_error", + ApiError::Search(..) => "search_error", + ApiError::Indexing(..) => "indexing_error", + ApiError::FileHosting(..) => "file_hosting_error", + ApiError::InvalidInput(..) => "invalid_input", + ApiError::Validation(..) => "invalid_input", + ApiError::Payments(..) => "payments_error", + ApiError::Discord(..) => "discord_error", + ApiError::Turnstile => "turnstile_error", + ApiError::Decoding(..) => "decoding_error", + ApiError::ImageParse(..) => "invalid_image", + ApiError::PasswordHashing(..) => "password_hashing_error", + ApiError::PasswordStrengthCheck(..) => "strength_check_error", + ApiError::Mail(..) => "mail_error", + ApiError::Clickhouse(..) => "clickhouse_error", + ApiError::Reroute(..) => "reroute_error", + ApiError::NotFound => "not_found", + ApiError::Zip(..) => "zip_error", + ApiError::Io(..) => "io_error", + }, + description: self.to_string(), + } + } +} + impl actix_web::ResponseError for ApiError { fn status_code(&self) -> StatusCode { match self { @@ -153,37 +192,12 @@ impl actix_web::ResponseError for ApiError { ApiError::Mail(..) => StatusCode::INTERNAL_SERVER_ERROR, ApiError::Reroute(..) => StatusCode::INTERNAL_SERVER_ERROR, ApiError::NotFound => StatusCode::NOT_FOUND, + ApiError::Zip(..) => StatusCode::BAD_REQUEST, + ApiError::Io(..) => StatusCode::BAD_REQUEST, } } fn error_response(&self) -> HttpResponse { - HttpResponse::build(self.status_code()).json(crate::models::error::ApiError { - error: match self { - ApiError::Env(..) => "environment_error", - ApiError::SqlxDatabase(..) => "database_error", - ApiError::Database(..) => "database_error", - ApiError::Authentication(..) => "unauthorized", - ApiError::CustomAuthentication(..) => "unauthorized", - ApiError::Xml(..) => "xml_error", - ApiError::Json(..) => "json_error", - ApiError::Search(..) => "search_error", - ApiError::Indexing(..) => "indexing_error", - ApiError::FileHosting(..) => "file_hosting_error", - ApiError::InvalidInput(..) => "invalid_input", - ApiError::Validation(..) => "invalid_input", - ApiError::Payments(..) => "payments_error", - ApiError::Discord(..) => "discord_error", - ApiError::Turnstile => "turnstile_error", - ApiError::Decoding(..) => "decoding_error", - ApiError::ImageParse(..) => "invalid_image", - ApiError::PasswordHashing(..) => "password_hashing_error", - ApiError::PasswordStrengthCheck(..) => "strength_check_error", - ApiError::Mail(..) => "mail_error", - ApiError::Clickhouse(..) => "clickhouse_error", - ApiError::Reroute(..) => "reroute_error", - ApiError::NotFound => "not_found", - }, - description: &self.to_string(), - }) + HttpResponse::build(self.status_code()).json(self.as_api_error()) } } diff --git a/src/routes/not_found.rs b/src/routes/not_found.rs index aa01aac9..2da930bd 100644 --- a/src/routes/not_found.rs +++ b/src/routes/not_found.rs @@ -4,7 +4,7 @@ use actix_web::{HttpResponse, Responder}; pub async fn not_found() -> impl Responder { let data = ApiError { error: "not_found", - description: "the requested route does not exist", + description: "the requested route does not exist".to_string(), }; HttpResponse::NotFound().json(data) diff --git a/src/routes/v2/moderation.rs b/src/routes/v2/moderation.rs index fac02f15..db4517a8 100644 --- a/src/routes/v2/moderation.rs +++ b/src/routes/v2/moderation.rs @@ -2,7 +2,7 @@ use super::ApiError; use crate::models::projects::Project; use crate::models::v2::projects::LegacyProject; use crate::queue::session::AuthQueue; -use crate::routes::v3; +use crate::routes::internal; use crate::{database::redis::RedisPool, routes::v2_reroute}; use actix_web::{get, web, HttpRequest, HttpResponse}; use serde::Deserialize; @@ -30,11 +30,11 @@ pub async fn get_projects( count: web::Query, session_queue: web::Data, ) -> Result { - let response = v3::moderation::get_projects( + let response = internal::moderation::get_projects( req, pool.clone(), redis.clone(), - web::Query(v3::moderation::ResultCount { count: count.count }), + web::Query(internal::moderation::ResultCount { count: count.count }), session_queue, ) .await diff --git a/src/routes/v2/projects.rs b/src/routes/v2/projects.rs index 512ad920..f5ce258c 100644 --- a/src/routes/v2/projects.rs +++ b/src/routes/v2/projects.rs @@ -7,6 +7,7 @@ use crate::models::projects::{ }; use crate::models::v2::projects::{DonationLink, LegacyProject, LegacySideType, LegacyVersion}; use crate::models::v2::search::LegacySearchResults; +use crate::queue::moderation::AutomatedModerationQueue; use crate::queue::session::AuthQueue; use crate::routes::v3::projects::ProjectIds; use crate::routes::{v2_reroute, v3, ApiError}; @@ -380,6 +381,7 @@ pub struct EditProject { } #[patch("{id}")] +#[allow(clippy::too_many_arguments)] pub async fn project_edit( req: HttpRequest, info: web::Path<(String,)>, @@ -388,6 +390,7 @@ pub async fn project_edit( new_project: web::Json, redis: web::Data, session_queue: web::Data, + moderation_queue: web::Data, ) -> Result { let v2_new_project = new_project.into_inner(); let client_side = v2_new_project.client_side; @@ -494,6 +497,7 @@ pub async fn project_edit( web::Json(new_project), redis.clone(), session_queue.clone(), + moderation_queue, ) .await .or_else(v2_reroute::flatten_404_error)?; diff --git a/src/routes/v3/mod.rs b/src/routes/v3/mod.rs index a5165fec..23a92a00 100644 --- a/src/routes/v3/mod.rs +++ b/src/routes/v3/mod.rs @@ -6,7 +6,6 @@ use serde_json::json; pub mod analytics_get; pub mod collections; pub mod images; -pub mod moderation; pub mod notifications; pub mod organizations; pub mod payouts; @@ -31,7 +30,6 @@ pub fn config(cfg: &mut web::ServiceConfig) { .configure(analytics_get::config) .configure(collections::config) .configure(images::config) - .configure(moderation::config) .configure(notifications::config) .configure(organizations::config) .configure(project_creation::config) diff --git a/src/routes/v3/moderation.rs b/src/routes/v3/moderation.rs deleted file mode 100644 index 8b72e036..00000000 --- a/src/routes/v3/moderation.rs +++ /dev/null @@ -1,65 +0,0 @@ -use super::ApiError; -use crate::database; -use crate::database::redis::RedisPool; -use crate::models::projects::ProjectStatus; -use crate::queue::session::AuthQueue; -use crate::{auth::check_is_moderator_from_headers, models::pats::Scopes}; -use actix_web::{web, HttpRequest, HttpResponse}; -use serde::Deserialize; -use sqlx::PgPool; - -pub fn config(cfg: &mut web::ServiceConfig) { - cfg.route("moderation/projects", web::get().to(get_projects)); -} - -#[derive(Deserialize)] -pub struct ResultCount { - #[serde(default = "default_count")] - pub count: i16, -} - -fn default_count() -> i16 { - 100 -} - -pub async fn get_projects( - req: HttpRequest, - pool: web::Data, - redis: web::Data, - count: web::Query, - session_queue: web::Data, -) -> Result { - check_is_moderator_from_headers( - &req, - &**pool, - &redis, - &session_queue, - Some(&[Scopes::PROJECT_READ]), - ) - .await?; - - use futures::stream::TryStreamExt; - - let project_ids = sqlx::query!( - " - SELECT id FROM mods - WHERE status = $1 - ORDER BY queued ASC - LIMIT $2; - ", - ProjectStatus::Processing.as_str(), - count.count as i64 - ) - .fetch_many(&**pool) - .try_filter_map(|e| async { Ok(e.right().map(|m| database::models::ProjectId(m.id))) }) - .try_collect::>() - .await?; - - let projects: Vec<_> = database::Project::get_many_ids(&project_ids, &**pool, &redis) - .await? - .into_iter() - .map(crate::models::projects::Project::from) - .collect(); - - Ok(HttpResponse::Ok().json(projects)) -} diff --git a/src/routes/v3/project_creation.rs b/src/routes/v3/project_creation.rs index 4e5203bf..f6453c35 100644 --- a/src/routes/v3/project_creation.rs +++ b/src/routes/v3/project_creation.rs @@ -137,7 +137,7 @@ impl actix_web::ResponseError for CreateError { CreateError::ImageError(..) => "invalid_image", CreateError::RerouteError(..) => "reroute_error", }, - description: &self.to_string(), + description: self.to_string(), }) } } diff --git a/src/routes/v3/projects.rs b/src/routes/v3/projects.rs index 4ba06ea4..603db711 100644 --- a/src/routes/v3/projects.rs +++ b/src/routes/v3/projects.rs @@ -20,6 +20,7 @@ use crate::models::projects::{ }; use crate::models::teams::ProjectPermissions; use crate::models::threads::MessageBody; +use crate::queue::moderation::AutomatedModerationQueue; use crate::queue::session::AuthQueue; use crate::routes::ApiError; use crate::search::indexing::remove_documents; @@ -229,6 +230,7 @@ pub struct EditProject { pub monetization_status: Option, } +#[allow(clippy::too_many_arguments)] pub async fn project_edit( req: HttpRequest, info: web::Path<(String,)>, @@ -237,6 +239,7 @@ pub async fn project_edit( new_project: web::Json, redis: web::Data, session_queue: web::Data, + moderation_queue: web::Data, ) -> Result { let user = get_user_from_headers( &req, @@ -362,6 +365,10 @@ pub async fn project_edit( ) .execute(&mut *transaction) .await?; + + moderation_queue + .projects + .insert(project_item.inner.id.into()); } if status.is_approved() && !project_item.inner.status.is_approved() { diff --git a/src/routes/v3/threads.rs b/src/routes/v3/threads.rs index 87907878..9121656d 100644 --- a/src/routes/v3/threads.rs +++ b/src/routes/v3/threads.rs @@ -379,6 +379,7 @@ pub async fn thread_send_message( body, replying_to, private, + hide_identity, .. } = &new_message.body { @@ -394,6 +395,12 @@ pub async fn thread_send_message( )); } + if *hide_identity && !user.role.is_mod() { + return Err(ApiError::InvalidInput( + "You are not allowed to send masked messages!".to_string(), + )); + } + if let Some(replying_to) = replying_to { let thread_message = database::models::ThreadMessage::get((*replying_to).into(), &**pool).await?; diff --git a/src/search/mod.rs b/src/search/mod.rs index 725958e5..c7588d8c 100644 --- a/src/search/mod.rs +++ b/src/search/mod.rs @@ -52,7 +52,7 @@ impl actix_web::ResponseError for SearchError { SearchError::InvalidIndex(..) => "invalid_input", SearchError::FormatError(..) => "invalid_input", }, - description: &self.to_string(), + description: self.to_string(), }) } }