GDPR export route (#969)

* GDPR export route

* make users able to access

.sqlx/query-505543e3e6aa69a9b9d4ee50938a305e86949fefc8ba11f4b10992fa507d136c.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n            SELECT r.id FROM reports r\n            WHERE r.user_id = $1\n            ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "505543e3e6aa69a9b9d4ee50938a305e86949fefc8ba11f4b10992fa507d136c"
+}

.sqlx/query-5a13a79ebb1ab975f88b58e6deaba9685fe16e242c0fa4a5eea54f12f9448e6b.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n            DELETE FROM reports\n            WHERE version_id = $1\n            ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int8"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "5a13a79ebb1ab975f88b58e6deaba9685fe16e242c0fa4a5eea54f12f9448e6b"
-}

.sqlx/query-683e08f3b71aca0d004ebf83a9e6b7b0b30291d595e5ae9f7e0fd38d347c3f74.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "SELECT id FROM uploaded_images WHERE owner_id = $1",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "683e08f3b71aca0d004ebf83a9e6b7b0b30291d595e5ae9f7e0fd38d347c3f74"
+}

.sqlx/query-8a67a27f45a743f8679ec6021ef125c242cb339db8914afcc3e2c90b0c307053.json

@@ -0,0 +1,22 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n        SELECT id FROM threads_messages WHERE author_id = $1 AND hide_identity = FALSE\n        ",
+  "describe": {
+    "columns": [
+      {
+        "ordinal": 0,
+        "name": "id",
+        "type_info": "Int8"
+      }
+    ],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": [
+      false
+    ]
+  },
+  "hash": "8a67a27f45a743f8679ec6021ef125c242cb339db8914afcc3e2c90b0c307053"
+}

.sqlx/query-902d0803deb5eca7614f3a68ccae6c3b401fcaa0bcc304b9caf18afc20a3e52b.json

@@ -1,22 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                SELECT t.id\n                FROM threads t\n                INNER JOIN reports r ON t.report_id = r.id\n                WHERE r.mod_id = $1 AND report_id IS NOT NULL \n                ",
-  "describe": {
-    "columns": [
-      {
-        "ordinal": 0,
-        "name": "id",
-        "type_info": "Int8"
-      }
-    ],
-    "parameters": {
-      "Left": [
-        "Int8"
-      ]
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "902d0803deb5eca7614f3a68ccae6c3b401fcaa0bcc304b9caf18afc20a3e52b"
-}

.sqlx/query-b3475fbc7a4d8b353964e6c5f7d32c45947cfdc88be25ab04dff16eb289dcbcb.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n            UPDATE reports\n            SET version_id = NULL\n            WHERE version_id = $1\n            ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "b3475fbc7a4d8b353964e6c5f7d32c45947cfdc88be25ab04dff16eb289dcbcb"
+}

.sqlx/query-c3391aed338110205a170ba3032e54be0f2b753b5550d87d7b5ba3e17a57a202.json

@@ -1,14 +0,0 @@
-{
-  "db_name": "PostgreSQL",
-  "query": "\n                DELETE FROM reports\n                WHERE mod_id = $1\n                ",
-  "describe": {
-    "columns": [],
-    "parameters": {
-      "Left": [
-        "Int8"
-      ]
-    },
-    "nullable": []
-  },
-  "hash": "c3391aed338110205a170ba3032e54be0f2b753b5550d87d7b5ba3e17a57a202"
-}

.sqlx/query-f96967f8d0d7e4c7a9d424e075fe70b2a89efe74bde1db9730ac478749dc1b66.json

@@ -0,0 +1,14 @@
+{
+  "db_name": "PostgreSQL",
+  "query": "\n                UPDATE reports\n                SET mod_id = NULL\n                WHERE mod_id = $1\n                ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Left": [
+        "Int8"
+      ]
+    },
+    "nullable": []
+  },
+  "hash": "f96967f8d0d7e4c7a9d424e075fe70b2a89efe74bde1db9730ac478749dc1b66"
+}

src/auth/validate.rs

@@ -3,13 +3,12 @@ use crate::auth::AuthenticationError;
 use crate::database::models::user_item;
 use crate::database::redis::RedisPool;
 use crate::models::pats::Scopes;
-use crate::models::users::{Role, User, UserId, UserPayoutData};
+use crate::models::users::User;
 use crate::queue::session::AuthQueue;
 use crate::routes::internal::session::get_session_metadata;
 use actix_web::http::header::{HeaderValue, AUTHORIZATION};
 use actix_web::HttpRequest;
 use chrono::Utc;
-use rust_decimal::Decimal;
 
 pub async fn get_user_from_headers<'a, E>(
     req: &HttpRequest,
@@ -26,51 +25,8 @@ where
         get_user_record_from_bearer_token(req, None, executor, redis, session_queue)
             .await?
             .ok_or_else(|| AuthenticationError::InvalidCredentials)?;
-    let mut auth_providers = Vec::new();
-    if db_user.github_id.is_some() {
-        auth_providers.push(AuthProvider::GitHub)
-    }
-    if db_user.gitlab_id.is_some() {
-        auth_providers.push(AuthProvider::GitLab)
-    }
-    if db_user.discord_id.is_some() {
-        auth_providers.push(AuthProvider::Discord)
-    }
-    if db_user.google_id.is_some() {
-        auth_providers.push(AuthProvider::Google)
-    }
-    if db_user.microsoft_id.is_some() {
-        auth_providers.push(AuthProvider::Microsoft)
-    }
-    if db_user.steam_id.is_some() {
-        auth_providers.push(AuthProvider::Steam)
-    }
-    if db_user.paypal_id.is_some() {
-        auth_providers.push(AuthProvider::PayPal)
-    }
 
-    let user = User {
-        id: UserId::from(db_user.id),
-        username: db_user.username,
-        email: db_user.email,
-        email_verified: Some(db_user.email_verified),
-        avatar_url: db_user.avatar_url,
-        bio: db_user.bio,
-        created: db_user.created,
-        role: Role::from_string(&db_user.role),
-        badges: db_user.badges,
-        auth_providers: Some(auth_providers),
-        has_password: Some(db_user.password.is_some()),
-        has_totp: Some(db_user.totp_secret.is_some()),
-        github_id: None,
-        payout_data: Some(UserPayoutData {
-            paypal_address: db_user.paypal_email,
-            paypal_country: db_user.paypal_country,
-            venmo_handle: db_user.venmo_handle,
-            balance: Decimal::ZERO,
-        }),
-        stripe_customer_id: db_user.stripe_customer_id,
-    };
+    let user = User::from_full(db_user);
 
     if let Some(required_scopes) = required_scopes {
         for scope in required_scopes {

src/database/models/project_item.rs

@@ -355,31 +355,12 @@ impl Project {
             .execute(&mut **transaction)
             .await?;
 
-            // Notably joins with report id and not thread.mod_id directly, as
-            // this is set to null for threads that are reports.
-            let report_threads = sqlx::query!(
-                "
-                SELECT t.id
-                FROM threads t
-                INNER JOIN reports r ON t.report_id = r.id
-                WHERE r.mod_id = $1 AND report_id IS NOT NULL 
-                ",
-                id as ProjectId,
-            )
-            .fetch(&mut **transaction)
-            .map_ok(|x| ThreadId(x.id))
-            .try_collect::<Vec<_>>()
-            .await?;
-
-            for thread_id in report_threads {
-                models::Thread::remove_full(thread_id, transaction).await?;
-            }
-
             models::Thread::remove_full(project.thread_id, transaction).await?;
 
             sqlx::query!(
                 "
-                DELETE FROM reports
+                UPDATE reports
+                SET mod_id = NULL
                 WHERE mod_id = $1
                 ",
                 id as ProjectId,

src/database/models/user_item.rs

@@ -1,5 +1,5 @@
 use super::ids::{ProjectId, UserId};
-use super::{CollectionId, ThreadId};
+use super::{CollectionId, ReportId, ThreadId};
 use crate::database::models;
 use crate::database::models::{DatabaseError, OrganizationId};
 use crate::database::redis::RedisPool;
@@ -323,6 +323,48 @@ impl User {
         Ok(projects)
     }
 
+    pub async fn get_follows<'a, E>(user_id: UserId, exec: E) -> Result<Vec<ProjectId>, sqlx::Error>
+    where
+        E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy,
+    {
+        use futures::stream::TryStreamExt;
+
+        let projects = sqlx::query!(
+            "
+            SELECT mf.mod_id FROM mod_follows mf
+            WHERE mf.follower_id = $1
+            ",
+            user_id as UserId,
+        )
+        .fetch(exec)
+        .map_ok(|m| ProjectId(m.mod_id))
+        .try_collect::<Vec<ProjectId>>()
+        .await?;
+
+        Ok(projects)
+    }
+
+    pub async fn get_reports<'a, E>(user_id: UserId, exec: E) -> Result<Vec<ReportId>, sqlx::Error>
+    where
+        E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy,
+    {
+        use futures::stream::TryStreamExt;
+
+        let reports = sqlx::query!(
+            "
+            SELECT r.id FROM reports r
+            WHERE r.user_id = $1
+            ",
+            user_id as UserId,
+        )
+        .fetch(exec)
+        .map_ok(|m| ReportId(m.id))
+        .try_collect::<Vec<ReportId>>()
+        .await?;
+
+        Ok(reports)
+    }
+
     pub async fn get_backup_codes<'a, E>(
         user_id: UserId,
         exec: E,

src/database/models/version_item.rs

@@ -335,7 +335,8 @@ impl Version {
 
         sqlx::query!(
             "
-            DELETE FROM reports
+            UPDATE reports
+            SET version_id = NULL
             WHERE version_id = $1
             ",
             id as VersionId,

src/models/v3/billing.rs

@@ -88,6 +88,23 @@ pub struct UserSubscription {
     pub last_charge: Option<DateTime<Utc>>,
 }
 
+impl From<crate::database::models::user_subscription_item::UserSubscriptionItem>
+    for UserSubscription
+{
+    fn from(x: crate::database::models::user_subscription_item::UserSubscriptionItem) -> Self {
+        Self {
+            id: x.id.into(),
+            user_id: x.user_id.into(),
+            price_id: x.price_id.into(),
+            interval: x.interval,
+            status: x.status,
+            created: x.created,
+            expires: x.expires,
+            last_charge: x.last_charge,
+        }
+    }
+}
+
 #[derive(Serialize, Deserialize, Eq, PartialEq)]
 #[serde(rename_all = "kebab-case")]
 pub enum SubscriptionStatus {

src/models/v3/threads.rs

@@ -113,19 +113,25 @@ impl Thread {
                         true
                     }
                 })
-                .map(|x| ThreadMessage {
-                    id: x.id.into(),
-                    author_id: if x.hide_identity && !user.role.is_mod() {
-                        None
-                    } else {
-                        x.author_id.map(|x| x.into())
-                    },
-                    body: x.body,
-                    created: x.created,
-                    hide_identity: x.hide_identity,
-                })
+                .map(|x| ThreadMessage::from(x, user))
                 .collect(),
             members: users,
         }
     }
 }
+
+impl ThreadMessage {
+    pub fn from(data: crate::database::models::ThreadMessage, user: &User) -> Self {
+        Self {
+            id: data.id.into(),
+            author_id: if data.hide_identity && !user.role.is_mod() {
+                None
+            } else {
+                data.author_id.map(|x| x.into())
+            },
+            body: data.body,
+            created: data.created,
+            hide_identity: data.hide_identity,
+        }
+    }
+}

src/models/v3/users.rs

@@ -89,6 +89,57 @@ impl From<DBUser> for User {
     }
 }
 
+impl User {
+    pub fn from_full(db_user: DBUser) -> Self {
+        let mut auth_providers = Vec::new();
+
+        if db_user.github_id.is_some() {
+            auth_providers.push(AuthProvider::GitHub)
+        }
+        if db_user.gitlab_id.is_some() {
+            auth_providers.push(AuthProvider::GitLab)
+        }
+        if db_user.discord_id.is_some() {
+            auth_providers.push(AuthProvider::Discord)
+        }
+        if db_user.google_id.is_some() {
+            auth_providers.push(AuthProvider::Google)
+        }
+        if db_user.microsoft_id.is_some() {
+            auth_providers.push(AuthProvider::Microsoft)
+        }
+        if db_user.steam_id.is_some() {
+            auth_providers.push(AuthProvider::Steam)
+        }
+        if db_user.paypal_id.is_some() {
+            auth_providers.push(AuthProvider::PayPal)
+        }
+
+        Self {
+            id: UserId::from(db_user.id),
+            username: db_user.username,
+            email: db_user.email,
+            email_verified: Some(db_user.email_verified),
+            avatar_url: db_user.avatar_url,
+            bio: db_user.bio,
+            created: db_user.created,
+            role: Role::from_string(&db_user.role),
+            badges: db_user.badges,
+            auth_providers: Some(auth_providers),
+            has_password: Some(db_user.password.is_some()),
+            has_totp: Some(db_user.totp_secret.is_some()),
+            github_id: None,
+            payout_data: Some(UserPayoutData {
+                paypal_address: db_user.paypal_email,
+                paypal_country: db_user.paypal_country,
+                venmo_handle: db_user.venmo_handle,
+                balance: Decimal::ZERO,
+            }),
+            stripe_customer_id: db_user.stripe_customer_id,
+        }
+    }
+}
+
 #[derive(Serialize, Deserialize, PartialEq, Eq, Clone, Debug)]
 #[serde(rename_all = "lowercase")]
 pub enum Role {

src/routes/internal/billing.rs

@@ -95,16 +95,7 @@ pub async fn subscriptions(
         user_subscription_item::UserSubscriptionItem::get_all_user(user.id.into(), &**pool)
             .await?
             .into_iter()
-            .map(|x| UserSubscription {
-                id: x.id.into(),
-                user_id: x.user_id.into(),
-                price_id: x.price_id.into(),
-                interval: x.interval,
-                status: x.status,
-                created: x.created,
-                expires: x.expires,
-                last_charge: x.last_charge,
-            })
+            .map(UserSubscription::from)
             .collect::<Vec<_>>();
 
     Ok(HttpResponse::Ok().json(subscriptions))

src/routes/internal/flows.rs

@@ -1641,14 +1641,13 @@ async fn validate_2fa_code(
     }
 
     if input == token {
-        conn
-            .set(
-                TOTP_NAMESPACE,
-                &format!("{}-{}", token, user_id.0),
-                "",
-                Some(60),
-            )
-            .await?;
+        conn.set(
+            TOTP_NAMESPACE,
+            &format!("{}-{}", token, user_id.0),
+            "",
+            Some(60),
+        )
+        .await?;
 
         Ok(true)
     } else if allow_backup {

src/routes/internal/gdpr.rs

@@ -0,0 +1,177 @@
+use crate::auth::get_user_from_headers;
+use crate::database::redis::RedisPool;
+use crate::models::pats::Scopes;
+use crate::queue::session::AuthQueue;
+use crate::routes::ApiError;
+use actix_web::{post, web, HttpRequest, HttpResponse};
+use sqlx::PgPool;
+
+pub fn config(cfg: &mut web::ServiceConfig) {
+    cfg.service(web::scope("gdpr").service(export));
+}
+
+#[post("/export")]
+pub async fn export(
+    req: HttpRequest,
+    pool: web::Data<PgPool>,
+    redis: web::Data<RedisPool>,
+    session_queue: web::Data<AuthQueue>,
+) -> Result<HttpResponse, ApiError> {
+    let user = get_user_from_headers(
+        &req,
+        &**pool,
+        &redis,
+        &*session_queue,
+        Some(&[Scopes::SESSION_ACCESS]),
+    )
+    .await?
+    .1;
+
+    let user_id = user.id.into();
+
+    let collection_ids = crate::database::models::User::get_collections(user_id, &**pool).await?;
+    let collections =
+        crate::database::models::Collection::get_many(&collection_ids, &**pool, &redis)
+            .await?
+            .into_iter()
+            .map(|x| crate::models::collections::Collection::from(x))
+            .collect::<Vec<_>>();
+
+    let follows = crate::database::models::User::get_follows(user_id, &**pool)
+        .await?
+        .into_iter()
+        .map(|x| crate::models::ids::ProjectId::from(x))
+        .collect::<Vec<_>>();
+
+    let projects = crate::database::models::User::get_projects(user_id, &**pool, &redis)
+        .await?
+        .into_iter()
+        .map(|x| crate::models::ids::ProjectId::from(x))
+        .collect::<Vec<_>>();
+
+    let org_ids = crate::database::models::User::get_organizations(user_id, &**pool).await?;
+    let orgs = crate::database::models::organization_item::Organization::get_many_ids(
+        &org_ids, &**pool, &redis,
+    )
+    .await?
+    .into_iter()
+    // TODO: add team members
+    .map(|x| crate::models::organizations::Organization::from(x, vec![]))
+    .collect::<Vec<_>>();
+
+    let notifs = crate::database::models::notification_item::Notification::get_many_user(
+        user_id, &**pool, &redis,
+    )
+    .await?
+    .into_iter()
+    .map(|x| crate::models::notifications::Notification::from(x))
+    .collect::<Vec<_>>();
+
+    let oauth_clients =
+        crate::database::models::oauth_client_item::OAuthClient::get_all_user_clients(
+            user_id, &**pool,
+        )
+        .await?
+        .into_iter()
+        .map(|x| crate::models::oauth_clients::OAuthClient::from(x))
+        .collect::<Vec<_>>();
+
+    let oauth_authorizations = crate::database::models::oauth_client_authorization_item::OAuthClientAuthorization::get_all_for_user(
+        user_id, &**pool,
+    )
+        .await?
+        .into_iter()
+        .map(|x| crate::models::oauth_clients::OAuthClientAuthorization::from(x))
+        .collect::<Vec<_>>();
+
+    let pat_ids = crate::database::models::pat_item::PersonalAccessToken::get_user_pats(
+        user_id, &**pool, &redis,
+    )
+    .await?;
+    let pats = crate::database::models::pat_item::PersonalAccessToken::get_many_ids(
+        &pat_ids, &**pool, &redis,
+    )
+    .await?
+    .into_iter()
+    .map(|x| crate::models::pats::PersonalAccessToken::from(x, false))
+    .collect::<Vec<_>>();
+
+    let payout_ids =
+        crate::database::models::payout_item::Payout::get_all_for_user(user_id, &**pool).await?;
+
+    let payouts = crate::database::models::payout_item::Payout::get_many(&payout_ids, &**pool)
+        .await?
+        .into_iter()
+        .map(|x| crate::models::payouts::Payout::from(x))
+        .collect::<Vec<_>>();
+
+    let report_ids =
+        crate::database::models::user_item::User::get_reports(user_id, &**pool).await?;
+    let reports = crate::database::models::report_item::Report::get_many(&report_ids, &**pool)
+        .await?
+        .into_iter()
+        .map(|x| crate::models::reports::Report::from(x))
+        .collect::<Vec<_>>();
+
+    let message_ids = sqlx::query!(
+        "
+        SELECT id FROM threads_messages WHERE author_id = $1 AND hide_identity = FALSE
+        ",
+        user_id.0
+    )
+    .fetch_all(pool.as_ref())
+    .await?
+    .into_iter()
+    .map(|x| crate::database::models::ids::ThreadMessageId(x.id))
+    .collect::<Vec<_>>();
+
+    let messages =
+        crate::database::models::thread_item::ThreadMessage::get_many(&message_ids, &**pool)
+            .await?
+            .into_iter()
+            .map(|x| crate::models::threads::ThreadMessage::from(x, &user))
+            .collect::<Vec<_>>();
+
+    let uploaded_images_ids = sqlx::query!(
+        "SELECT id FROM uploaded_images WHERE owner_id = $1",
+        user_id.0
+    )
+    .fetch_all(pool.as_ref())
+    .await?
+    .into_iter()
+    .map(|x| crate::database::models::ids::ImageId(x.id))
+    .collect::<Vec<_>>();
+
+    let uploaded_images =
+        crate::database::models::image_item::Image::get_many(&uploaded_images_ids, &**pool, &redis)
+            .await?
+            .into_iter()
+            .map(|x| crate::models::images::Image::from(x))
+            .collect::<Vec<_>>();
+
+    let subscriptions =
+        crate::database::models::user_subscription_item::UserSubscriptionItem::get_all_user(
+            user_id, &**pool,
+        )
+        .await?
+        .into_iter()
+        .map(|x| crate::models::billing::UserSubscription::from(x))
+        .collect::<Vec<_>>();
+
+    Ok(HttpResponse::Ok().json(serde_json::json!({
+        "user": user,
+        "collections": collections,
+        "follows": follows,
+        "projects": projects,
+        "orgs": orgs,
+        "notifs": notifs,
+        "oauth_clients": oauth_clients,
+        "oauth_authorizations": oauth_authorizations,
+        "pats": pats,
+        "payouts": payouts,
+        "reports": reports,
+        "messages": messages,
+        "uploaded_images": uploaded_images,
+        "subscriptions": subscriptions,
+    })))
+}

src/routes/internal/mod.rs

@@ -1,6 +1,7 @@
 pub(crate) mod admin;
 pub mod billing;
 pub mod flows;
+pub mod gdpr;
 pub mod moderation;
 pub mod pats;
 pub mod session;
@@ -19,6 +20,7 @@ pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
             .configure(flows::config)
             .configure(pats::config)
             .configure(moderation::config)
-            .configure(billing::config),
+            .configure(billing::config)
+            .configure(gdpr::config),
     );
 }

src/routes/v3/users.rs

@@ -571,20 +571,7 @@ pub async fn user_follows(
             ));
         }
 
-        use futures::TryStreamExt;
-
-        let project_ids = sqlx::query!(
-            "
-            SELECT mf.mod_id FROM mod_follows mf
-            WHERE mf.follower_id = $1
-            ",
-            id as crate::database::models::ids::UserId,
-        )
-        .fetch(&**pool)
-        .map_ok(|m| crate::database::models::ProjectId(m.mod_id))
-        .try_collect::<Vec<crate::database::models::ProjectId>>()
-        .await?;
-
+        let project_ids = User::get_follows(id, &**pool).await?;
         let projects: Vec<_> =
             crate::database::Project::get_many_ids(&project_ids, &**pool, &redis)
                 .await?