You've already forked pages
forked from didirus/AstralRinth
Fix cache stampede issues + generalize cache (#884)
* caching changes * fix cache stampede issues * Use pub/sub for better DB fetches * remove pubsub * remove debugs * Fix caches not working * fix search indexing removal
This commit is contained in:
Geometrically
@@ -1,10 +1,14 @@
|
||||
use super::ids::*;
|
||||
use crate::database::models::DatabaseError;
|
||||
use crate::database::redis::RedisPool;
|
||||
use crate::models::ids::base62_impl::{parse_base62, to_base62};
|
||||
use crate::models::ids::base62_impl::parse_base62;
|
||||
use crate::models::pats::Scopes;
|
||||
use chrono::{DateTime, Utc};
|
||||
use dashmap::DashMap;
|
||||
use futures::TryStreamExt;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::fmt::{Debug, Display};
|
||||
use std::hash::Hash;
|
||||
|
||||
const PATS_NAMESPACE: &str = "pats";
|
||||
const PATS_TOKENS_NAMESPACE: &str = "pats_tokens";
|
||||
@@ -51,7 +55,7 @@ impl PersonalAccessToken {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn get<'a, E, T: ToString>(
|
||||
pub async fn get<'a, E, T: Display + Hash + Eq + PartialEq + Clone + Debug>(
|
||||
id: T,
|
||||
exec: E,
|
||||
redis: &RedisPool,
|
||||
@@ -79,7 +83,7 @@ impl PersonalAccessToken {
|
||||
PersonalAccessToken::get_many(&ids, exec, redis).await
|
||||
}
|
||||
|
||||
pub async fn get_many<'a, E, T: ToString>(
|
||||
pub async fn get_many<'a, E, T: Display + Hash + Eq + PartialEq + Clone + Debug>(
|
||||
pat_strings: &[T],
|
||||
exec: E,
|
||||
redis: &RedisPool,
|
||||
@@ -87,105 +91,53 @@ impl PersonalAccessToken {
|
||||
where
|
||||
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
|
||||
{
|
||||
use futures::TryStreamExt;
|
||||
let val = redis
|
||||
.get_cached_keys_with_slug(
|
||||
PATS_NAMESPACE,
|
||||
PATS_TOKENS_NAMESPACE,
|
||||
true,
|
||||
pat_strings,
|
||||
|ids| async move {
|
||||
let pat_ids: Vec<i64> = ids
|
||||
.iter()
|
||||
.flat_map(|x| parse_base62(&x.to_string()).ok())
|
||||
.map(|x| x as i64)
|
||||
.collect();
|
||||
let slugs = ids.into_iter().map(|x| x.to_string()).collect::<Vec<_>>();
|
||||
|
||||
let mut redis = redis.connect().await?;
|
||||
let pats = sqlx::query!(
|
||||
"
|
||||
SELECT id, name, access_token, scopes, user_id, created, expires, last_used
|
||||
FROM pats
|
||||
WHERE id = ANY($1) OR access_token = ANY($2)
|
||||
ORDER BY created DESC
|
||||
",
|
||||
&pat_ids,
|
||||
&slugs,
|
||||
)
|
||||
.fetch(exec)
|
||||
.try_fold(DashMap::new(), |acc, x| {
|
||||
let pat = PersonalAccessToken {
|
||||
id: PatId(x.id),
|
||||
name: x.name,
|
||||
access_token: x.access_token.clone(),
|
||||
scopes: Scopes::from_bits(x.scopes as u64).unwrap_or(Scopes::NONE),
|
||||
user_id: UserId(x.user_id),
|
||||
created: x.created,
|
||||
expires: x.expires,
|
||||
last_used: x.last_used,
|
||||
};
|
||||
|
||||
if pat_strings.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let mut found_pats = Vec::new();
|
||||
let mut remaining_strings = pat_strings
|
||||
.iter()
|
||||
.map(|x| x.to_string())
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
let mut pat_ids = pat_strings
|
||||
.iter()
|
||||
.flat_map(|x| parse_base62(&x.to_string()).map(|x| x as i64))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
pat_ids.append(
|
||||
&mut redis
|
||||
.multi_get::<i64>(
|
||||
PATS_TOKENS_NAMESPACE,
|
||||
pat_strings.iter().map(|x| x.to_string()),
|
||||
)
|
||||
.await?
|
||||
.into_iter()
|
||||
.flatten()
|
||||
.collect(),
|
||||
);
|
||||
|
||||
if !pat_ids.is_empty() {
|
||||
let pats = redis
|
||||
.multi_get::<String>(PATS_NAMESPACE, pat_ids.iter().map(|x| x.to_string()))
|
||||
.await?;
|
||||
for pat in pats {
|
||||
if let Some(pat) =
|
||||
pat.and_then(|x| serde_json::from_str::<PersonalAccessToken>(&x).ok())
|
||||
{
|
||||
remaining_strings
|
||||
.retain(|x| &to_base62(pat.id.0 as u64) != x && &pat.access_token != x);
|
||||
found_pats.push(pat);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !remaining_strings.is_empty() {
|
||||
let pat_ids_parsed: Vec<i64> = remaining_strings
|
||||
.iter()
|
||||
.flat_map(|x| parse_base62(&x.to_string()).ok())
|
||||
.map(|x| x as i64)
|
||||
.collect();
|
||||
let db_pats: Vec<PersonalAccessToken> = sqlx::query!(
|
||||
"
|
||||
SELECT id, name, access_token, scopes, user_id, created, expires, last_used
|
||||
FROM pats
|
||||
WHERE id = ANY($1) OR access_token = ANY($2)
|
||||
ORDER BY created DESC
|
||||
",
|
||||
&pat_ids_parsed,
|
||||
&remaining_strings
|
||||
.into_iter()
|
||||
.map(|x| x.to_string())
|
||||
.collect::<Vec<_>>(),
|
||||
acc.insert(x.id, (Some(x.access_token), pat));
|
||||
async move { Ok(acc) }
|
||||
})
|
||||
.await?;
|
||||
Ok(pats)
|
||||
},
|
||||
)
|
||||
.fetch_many(exec)
|
||||
.try_filter_map(|e| async {
|
||||
Ok(e.right().map(|x| PersonalAccessToken {
|
||||
id: PatId(x.id),
|
||||
name: x.name,
|
||||
access_token: x.access_token,
|
||||
scopes: Scopes::from_bits(x.scopes as u64).unwrap_or(Scopes::NONE),
|
||||
user_id: UserId(x.user_id),
|
||||
created: x.created,
|
||||
expires: x.expires,
|
||||
last_used: x.last_used,
|
||||
}))
|
||||
})
|
||||
.try_collect::<Vec<PersonalAccessToken>>()
|
||||
.await?;
|
||||
|
||||
for pat in db_pats {
|
||||
redis
|
||||
.set_serialized_to_json(PATS_NAMESPACE, pat.id.0, &pat, None)
|
||||
.await?;
|
||||
redis
|
||||
.set(
|
||||
PATS_TOKENS_NAMESPACE,
|
||||
&pat.access_token,
|
||||
&pat.id.0.to_string(),
|
||||
None,
|
||||
)
|
||||
.await?;
|
||||
found_pats.push(pat);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(found_pats)
|
||||
Ok(val)
|
||||
}
|
||||
|
||||
pub async fn get_user_pats<'a, E>(
|
||||
@@ -206,14 +158,13 @@ impl PersonalAccessToken {
|
||||
return Ok(res.into_iter().map(PatId).collect());
|
||||
}
|
||||
|
||||
use futures::TryStreamExt;
|
||||
let db_pats: Vec<PatId> = sqlx::query!(
|
||||
"
|
||||
SELECT id
|
||||
FROM pats
|
||||
WHERE user_id = $1
|
||||
ORDER BY created DESC
|
||||
",
|
||||
SELECT id
|
||||
FROM pats
|
||||
WHERE user_id = $1
|
||||
ORDER BY created DESC
|
||||
",
|
||||
user_id.0,
|
||||
)
|
||||
.fetch_many(exec)
|
||||
|
||||
Reference in New Issue
Block a user