eightequal/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Scoped PATs (#651)

Browse Source 
* Scoped PATs

* fix threads issues

* fix migration
This commit is contained in:
Geometrically
2023-07-10 16:44:40 -07:00
committed by GitHub
parent 366ea63209
commit 7fbb8838e7
42 changed files with 2560 additions and 1402 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
src/database/models/ids.rs
View File

@@ -83,13 +83,13 @@ generate_ids!(
"SELECT EXISTS(SELECT 1 FROM states WHERE id=$1)",
StateId
);
// generate_ids!(
// pub generate_pat_id,
// PatId,
// 8,
// "SELECT EXISTS(SELECT 1 FROM pats WHERE id=$1)",
// PatId
// );
generate_ids!(
pub generate_pat_id,
PatId,
8,
"SELECT EXISTS(SELECT 1 FROM pats WHERE id=$1)",
PatId
);
generate_ids!(
pub generate_user_id,
@@ -193,7 +193,7 @@ pub struct FileId(pub i64);
#[sqlx(transparent)]
pub struct StateId(pub i64);
#[derive(Copy, Clone, Debug, Type)]
#[derive(Copy, Clone, Debug, Type, Deserialize, Serialize)]
#[sqlx(transparent)]
pub struct PatId(pub i64);
@@ -302,3 +302,8 @@ impl From<SessionId> for ids::SessionId {
ids::SessionId(id.0 as u64)
}
}
impl From<PatId> for ids::PatId {
fn from(id: PatId) -> Self {
ids::PatId(id.0 as u64)
}
}

1
src/database/models/mod.rs
View File

@@ -3,6 +3,7 @@ use thiserror::Error;
pub mod categories;
pub mod ids;
pub mod notification_item;
pub mod pat_item;
pub mod project_item;
pub mod report_item;
pub mod session_item;

289
src/database/models/pat_item.rs Normal file
View File

@@ -0,0 +1,289 @@
use super::ids::*;
use crate::database::models::DatabaseError;
use crate::models::ids::base62_impl::{parse_base62, to_base62};
use crate::models::pats::Scopes;
use chrono::{DateTime, Utc};
use redis::cmd;
use serde::{Deserialize, Serialize};
const PATS_NAMESPACE: &str = "pats";
const PATS_TOKENS_NAMESPACE: &str = "pats_tokens";
const PATS_USERS_NAMESPACE: &str = "pats_users";
const DEFAULT_EXPIRY: i64 = 1800; // 30 minutes
#[derive(Deserialize, Serialize)]
pub struct PersonalAccessToken {
pub id: PatId,
pub name: String,
pub access_token: String,
pub scopes: Scopes,
pub user_id: UserId,
pub created: DateTime<Utc>,
pub expires: DateTime<Utc>,
pub last_used: Option<DateTime<Utc>>,
}
impl PersonalAccessToken {
pub async fn insert(
&self,
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<(), DatabaseError> {
sqlx::query!(
"
INSERT INTO pats (
id, name, access_token, scopes, user_id,
expires
)
VALUES (
$1, $2, $3, $4, $5,
$6
)
",
self.id as PatId,
self.name,
self.access_token,
self.scopes.bits() as i64,
self.user_id as UserId,
self.expires
)
.execute(&mut *transaction)
.await?;
Ok(())
}
pub async fn get<'a, E, T: ToString>(
id: T,
exec: E,
redis: &deadpool_redis::Pool,
) -> Result<Option<PersonalAccessToken>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
Self::get_many(&[id], exec, redis)
.await
.map(|x| x.into_iter().next())
}
pub async fn get_many_ids<'a, E>(
pat_ids: &[PatId],
exec: E,
redis: &deadpool_redis::Pool,
) -> Result<Vec<PersonalAccessToken>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let ids = pat_ids
.iter()
.map(|x| crate::models::ids::PatId::from(*x))
.collect::<Vec<_>>();
PersonalAccessToken::get_many(&ids, exec, redis).await
}
pub async fn get_many<'a, E, T: ToString>(
pat_strings: &[T],
exec: E,
redis: &deadpool_redis::Pool,
) -> Result<Vec<PersonalAccessToken>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
use futures::TryStreamExt;
if pat_strings.is_empty() {
return Ok(Vec::new());
}
let mut redis = redis.get().await?;
let mut found_pats = Vec::new();
let mut remaining_strings = pat_strings
.iter()
.map(|x| x.to_string())
.collect::<Vec<_>>();
let mut pat_ids = pat_strings
.iter()
.flat_map(|x| parse_base62(&x.to_string()).map(|x| x as i64))
.collect::<Vec<_>>();
pat_ids.append(
&mut cmd("MGET")
.arg(
pat_strings
.iter()
.map(|x| format!("{}:{}", PATS_TOKENS_NAMESPACE, x.to_string()))
.collect::<Vec<_>>(),
)
.query_async::<_, Vec<Option<i64>>>(&mut redis)
.await?
.into_iter()
.flatten()
.collect(),
);
if !pat_ids.is_empty() {
let pats = cmd("MGET")
.arg(
pat_ids
.iter()
.map(|x| format!("{}:{}", PATS_NAMESPACE, x))
.collect::<Vec<_>>(),
)
.query_async::<_, Vec<Option<String>>>(&mut redis)
.await?;
for pat in pats {
if let Some(pat) =
pat.and_then(|x| serde_json::from_str::<PersonalAccessToken>(&x).ok())
{
remaining_strings
.retain(|x| &to_base62(pat.id.0 as u64) != x && &pat.access_token != x);
found_pats.push(pat);
continue;
}
}
}
if !remaining_strings.is_empty() {
let pat_ids_parsed: Vec<i64> = pat_strings
.iter()
.flat_map(|x| parse_base62(&x.to_string()).ok())
.map(|x| x as i64)
.collect();
let db_pats: Vec<PersonalAccessToken> = sqlx::query!(
"
SELECT id, name, access_token, scopes, user_id, created, expires, last_used
FROM pats
WHERE id = ANY($1) OR access_token = ANY($2)
ORDER BY created DESC
",
&pat_ids_parsed,
&pat_strings
.into_iter()
.map(|x| x.to_string())
.collect::<Vec<_>>(),
)
.fetch_many(exec)
.try_filter_map(|e| async {
Ok(e.right().map(|x| PersonalAccessToken {
id: PatId(x.id),
name: x.name,
access_token: x.access_token,
scopes: Scopes::from_bits(x.scopes as u64).unwrap_or(Scopes::NONE),
user_id: UserId(x.user_id),
created: x.created,
expires: x.expires,
last_used: x.last_used,
}))
})
.try_collect::<Vec<PersonalAccessToken>>()
.await?;
for pat in db_pats {
cmd("SET")
.arg(format!("{}:{}", PATS_NAMESPACE, pat.id.0))
.arg(serde_json::to_string(&pat)?)
.arg("EX")
.arg(DEFAULT_EXPIRY)
.query_async::<_, ()>(&mut redis)
.await?;
cmd("SET")
.arg(format!("{}:{}", PATS_TOKENS_NAMESPACE, pat.access_token))
.arg(pat.id.0)
.arg("EX")
.arg(DEFAULT_EXPIRY)
.query_async::<_, ()>(&mut redis)
.await?;
found_pats.push(pat);
}
}
Ok(found_pats)
}
pub async fn get_user_pats<'a, E>(
user_id: UserId,
exec: E,
redis: &deadpool_redis::Pool,
) -> Result<Vec<PatId>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let mut redis = redis.get().await?;
let res = cmd("GET")
.arg(format!("{}:{}", PATS_USERS_NAMESPACE, user_id.0))
.query_async::<_, Option<Vec<i64>>>(&mut redis)
.await?;
if let Some(res) = res {
return Ok(res.into_iter().map(PatId).collect());
}
use futures::TryStreamExt;
let db_pats: Vec<PatId> = sqlx::query!(
"
SELECT id
FROM pats
WHERE user_id = $1
ORDER BY created DESC
",
user_id.0,
)
.fetch_many(exec)
.try_filter_map(|e| async { Ok(e.right().map(|x| PatId(x.id))) })
.try_collect::<Vec<PatId>>()
.await?;
cmd("SET")
.arg(format!("{}:{}", PATS_USERS_NAMESPACE, user_id.0))
.arg(serde_json::to_string(&db_pats)?)
.arg("EX")
.arg(DEFAULT_EXPIRY)
.query_async::<_, ()>(&mut redis)
.await?;
Ok(db_pats)
}
pub async fn clear_cache(
clear_pats: Vec<(Option<PatId>, Option<String>, Option<UserId>)>,
redis: &deadpool_redis::Pool,
) -> Result<(), DatabaseError> {
let mut redis = redis.get().await?;
let mut cmd = cmd("DEL");
for (id, token, user_id) in clear_pats {
if let Some(id) = id {
cmd.arg(format!("{}:{}", PATS_NAMESPACE, id.0));
}
if let Some(token) = token {
cmd.arg(format!("{}:{}", PATS_TOKENS_NAMESPACE, token));
}
if let Some(user_id) = user_id {
cmd.arg(format!("{}:{}", PATS_USERS_NAMESPACE, user_id.0));
}
}
cmd.query_async::<_, ()>(&mut redis).await?;
Ok(())
}
pub async fn remove(
id: PatId,
transaction: &mut sqlx::Transaction<'_, sqlx::Postgres>,
) -> Result<Option<()>, sqlx::error::Error> {
sqlx::query!(
"
DELETE FROM pats WHERE id = $1
",
id as PatId,
)
.execute(&mut *transaction)
.await?;
Ok(Some(()))
}
}

27
src/database/models/project_item.rs
View File

@@ -85,6 +85,7 @@ impl GalleryItem {
}
}
#[derive(Clone)]
pub struct ProjectBuilder {
pub project_id: ProjectId,
pub project_type_id: ProjectTypeId,
@@ -110,7 +111,6 @@ pub struct ProjectBuilder {
pub donation_urls: Vec<DonationUrl>,
pub gallery_items: Vec<GalleryItem>,
pub color: Option<u32>,
pub thread_id: ThreadId,
pub monetization_status: MonetizationStatus,
}
@@ -153,7 +153,6 @@ impl ProjectBuilder {
moderation_message_body: None,
webhook_sent: false,
color: self.color,
thread_id: Some(self.thread_id),
monetization_status: self.monetization_status,
};
project_struct.insert(&mut *transaction).await?;
@@ -231,7 +230,6 @@ pub struct Project {
pub moderation_message_body: Option<String>,
pub webhook_sent: bool,
pub color: Option<u32>,
pub thread_id: Option<ThreadId>,
pub monetization_status: MonetizationStatus,
}
@@ -247,14 +245,14 @@ impl Project {
published, downloads, icon_url, issues_url,
source_url, wiki_url, status, requested_status, discord_url,
client_side, server_side, license_url, license,
slug, project_type, color, thread_id, monetization_status
slug, project_type, color, monetization_status
)
VALUES (
$1, $2, $3, $4, $5,
$6, $7, $8, $9,
$10, $11, $12, $13, $14,
$15, $16, $17, $18,
LOWER($19), $20, $21, $22, $23
LOWER($19), $20, $21, $22
)
",
self.id as ProjectId,
@@ -278,7 +276,6 @@ impl Project {
self.slug.as_ref(),
self.project_type as ProjectTypeId,
self.color.map(|x| x as i32),
self.thread_id.map(|x| x.0),
self.monetization_status.as_str(),
)
.execute(&mut *transaction)
@@ -381,6 +378,8 @@ impl Project {
.execute(&mut *transaction)
.await?;
models::Thread::remove_full(project.thread_id, transaction).await?;
sqlx::query!(
"
DELETE FROM mods
@@ -413,10 +412,6 @@ impl Project {
.execute(&mut *transaction)
.await?;
if let Some(thread_id) = project.inner.thread_id {
models::Thread::remove_full(thread_id, transaction).await?;
}
Ok(Some(()))
} else {
Ok(None)
@@ -551,7 +546,7 @@ impl Project {
m.issues_url issues_url, m.source_url source_url, m.wiki_url wiki_url, m.discord_url discord_url, m.license_url license_url,
m.team_id team_id, m.client_side client_side, m.server_side server_side, m.license license, m.slug slug, m.moderation_message moderation_message, m.moderation_message_body moderation_message_body,
cs.name client_side_type, ss.name server_side_type, pt.name project_type_name, m.webhook_sent, m.color,
m.thread_id thread_id, m.monetization_status monetization_status,
t.id thread_id, m.monetization_status monetization_status,
ARRAY_AGG(DISTINCT l.loader) filter (where l.loader is not null) loaders,
JSONB_AGG(DISTINCT jsonb_build_object('id', gv.version, 'created', gv.created)) filter (where gv.version is not null) game_versions,
ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is false) categories,
@@ -563,6 +558,7 @@ impl Project {
INNER JOIN project_types pt ON pt.id = m.project_type
INNER JOIN side_types cs ON m.client_side = cs.id
INNER JOIN side_types ss ON m.server_side = ss.id
INNER JOIN threads t ON t.mod_id = m.id
LEFT JOIN mods_gallery mg ON mg.mod_id = m.id
LEFT JOIN mods_donations md ON md.joining_mod_id = m.id
LEFT JOIN donation_platforms dp ON md.joining_platform_id = dp.id
@@ -574,7 +570,7 @@ impl Project {
LEFT JOIN game_versions_versions gvv ON v.id = gvv.joining_version_id
LEFT JOIN game_versions gv ON gvv.game_version_id = gv.id
WHERE m.id = ANY($1) OR m.slug = ANY($2)
GROUP BY pt.id, cs.id, ss.id, m.id;
GROUP BY pt.id, cs.id, ss.id, t.id, m.id;
",
&project_ids_parsed,
&remaining_strings.into_iter().map(|x| x.to_string().to_lowercase()).collect::<Vec<_>>(),
@@ -620,7 +616,6 @@ impl Project {
webhook_sent: m.webhook_sent,
color: m.color.map(|x| x as u32),
queued: m.queued,
thread_id: m.thread_id.map(ThreadId),
monetization_status: MonetizationStatus::from_str(
&m.monetization_status,
),
@@ -676,8 +671,9 @@ impl Project {
game_versions.sort_by(|a, b| a.created.cmp(&b.created));
game_versions.into_iter().map(|x| x.id).collect()
}
}}))
},
thread_id: ThreadId(m.thread_id),
}}))
})
.try_collect::<Vec<QueryProject>>()
.await?;
@@ -814,4 +810,5 @@ pub struct QueryProject {
pub server_side: crate::models::projects::SideType,
pub loaders: Vec<String>,
pub game_versions: Vec<String>,
pub thread_id: ThreadId,
}

29
src/database/models/report_item.rs
View File

@@ -11,7 +11,6 @@ pub struct Report {
pub reporter: UserId,
pub created: DateTime<Utc>,
pub closed: bool,
pub thread_id: ThreadId,
}
pub struct QueryReport {
@@ -24,7 +23,7 @@ pub struct QueryReport {
pub reporter: UserId,
pub created: DateTime<Utc>,
pub closed: bool,
pub thread_id: Option<ThreadId>,
pub thread_id: ThreadId,
}
impl Report {
@@ -36,11 +35,11 @@ impl Report {
"
INSERT INTO reports (
id, report_type_id, mod_id, version_id, user_id,
body, reporter, thread_id
body, reporter
)
VALUES (
$1, $2, $3, $4, $5,
$6, $7, $8
$6, $7
)
",
self.id as ReportId,
@@ -49,8 +48,7 @@ impl Report {
self.version_id.map(|x| x.0 as i64),
self.user_id.map(|x| x.0 as i64),
self.body,
self.reporter as UserId,
self.thread_id as ThreadId,
self.reporter as UserId
)
.execute(&mut *transaction)
.await?;
@@ -79,9 +77,10 @@ impl Report {
let report_ids_parsed: Vec<i64> = report_ids.iter().map(|x| x.0).collect();
let reports = sqlx::query!(
"
SELECT r.id, rt.name, r.mod_id, r.version_id, r.user_id, r.body, r.reporter, r.created, r.thread_id, r.closed
SELECT r.id, rt.name, r.mod_id, r.version_id, r.user_id, r.body, r.reporter, r.created, t.id thread_id, r.closed
FROM reports r
INNER JOIN report_types rt ON rt.id = r.report_type_id
INNER JOIN threads t ON t.report_id = r.id
WHERE r.id = ANY($1)
ORDER BY r.created DESC
",
@@ -99,7 +98,7 @@ impl Report {
reporter: UserId(x.reporter),
created: x.created,
closed: x.closed,
thread_id: x.thread_id.map(ThreadId),
thread_id: ThreadId(x.thread_id)
}))
})
.try_collect::<Vec<QueryReport>>()
@@ -127,14 +126,18 @@ impl Report {
let thread_id = sqlx::query!(
"
SELECT thread_id FROM reports
WHERE id = $1
SELECT id FROM threads
WHERE report_id = $1
",
id as ReportId
)
.fetch_optional(&mut *transaction)
.await?;
if let Some(thread_id) = thread_id {
crate::database::models::Thread::remove_full(ThreadId(thread_id.id), transaction).await?;
}
sqlx::query!(
"
DELETE FROM reports WHERE id = $1
@@ -144,12 +147,6 @@ impl Report {
.execute(&mut *transaction)
.await?;
if let Some(thread_id) = thread_id {
if let Some(id) = thread_id.thread_id {
crate::database::models::Thread::remove_full(ThreadId(id), transaction).await?;
}
}
Ok(Some(()))
}
}

4
src/database/models/session_item.rs
View File

@@ -107,14 +107,14 @@ impl Session {
}
pub async fn get_many_ids<'a, E>(
user_ids: &[SessionId],
session_ids: &[SessionId],
exec: E,
redis: &deadpool_redis::Pool,
) -> Result<Vec<Session>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let ids = user_ids
let ids = session_ids
.iter()
.map(|x| crate::models::ids::SessionId::from(*x))
.collect::<Vec<_>>();

18
src/database/models/thread_item.rs
View File

@@ -7,12 +7,18 @@ use serde::Deserialize;
pub struct ThreadBuilder {
pub type_: ThreadType,
pub members: Vec<UserId>,
pub project_id: Option<ProjectId>,
pub report_id: Option<ReportId>,
}
#[derive(Clone)]
pub struct Thread {
pub id: ThreadId,
pub project_id: Option<ProjectId>,
pub report_id: Option<ReportId>,
pub type_: ThreadType,
pub messages: Vec<ThreadMessage>,
pub members: Vec<UserId>,
pub show_in_mod_inbox: bool,
@@ -70,14 +76,16 @@ impl ThreadBuilder {
sqlx::query!(
"
INSERT INTO threads (
id, thread_type
id, thread_type, mod_id, report_id
)
VALUES (
$1, $2
$1, $2, $3, $4
)
",
thread_id as ThreadId,
self.type_.as_str()
self.type_.as_str(),
self.project_id.map(|x| x.0),
self.report_id.map(|x| x.0),
)
.execute(&mut *transaction)
.await?;
@@ -125,7 +133,7 @@ impl Thread {
let thread_ids_parsed: Vec<i64> = thread_ids.iter().map(|x| x.0).collect();
let threads = sqlx::query!(
"
SELECT t.id, t.thread_type, t.show_in_mod_inbox,
SELECT t.id, t.thread_type, t.mod_id, t.report_id, t.show_in_mod_inbox,
ARRAY_AGG(DISTINCT tm.user_id) filter (where tm.user_id is not null) members,
JSONB_AGG(DISTINCT jsonb_build_object('id', tmsg.id, 'author_id', tmsg.author_id, 'thread_id', tmsg.thread_id, 'body', tmsg.body, 'created', tmsg.created)) filter (where tmsg.id is not null) messages
FROM threads t
@@ -140,6 +148,8 @@ impl Thread {
.try_filter_map(|e| async {
Ok(e.right().map(|x| Thread {
id: ThreadId(x.id),
project_id: x.mod_id.map(ProjectId),
report_id: x.report_id.map(ReportId),
type_: ThreadType::from_str(&x.thread_type),
messages: {
let mut messages: Vec<ThreadMessage> = serde_json::from_value(

6
src/database/models/version_item.rs
View File

@@ -10,11 +10,10 @@ use std::cmp::Ordering;
use std::collections::HashMap;
const VERSIONS_NAMESPACE: &str = "versions";
// TODO: Cache version slugs call
// const VERSIONS_SLUGS_NAMESPACE: &str = "versions_slugs";
const VERSION_FILES_NAMESPACE: &str = "versions_files";
const DEFAULT_EXPIRY: i64 = 1800; // 30 minutes
#[derive(Clone)]
pub struct VersionBuilder {
pub version_id: VersionId,
pub project_id: ProjectId,
@@ -32,6 +31,7 @@ pub struct VersionBuilder {
pub requested_status: Option<VersionStatus>,
}
#[derive(Clone)]
pub struct DependencyBuilder {
pub project_id: Option<ProjectId>,
pub version_id: Option<VersionId>,
@@ -79,6 +79,7 @@ impl DependencyBuilder {
}
}
#[derive(Clone)]
pub struct VersionFileBuilder {
pub url: String,
pub filename: String,
@@ -130,6 +131,7 @@ impl VersionFileBuilder {
}
}
#[derive(Clone)]
pub struct HashBuilder {
pub algorithm: String,
pub hash: Vec<u8>,