tomasalias/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

PG/RedisPool configuration/o11y improvements (#5032)

Browse Source 
* Don't retain Redis connections while doing database queries

* Optional REDIS_WAIT_TIMEOUT_MS

* Attach more data to CacheTimeout errrors

* Fix locks_released

* Fmt

* Set default REDIS_WAIT_TIMEOUT_MS to 15s

* Fix lint

* Close Redis connections idle for > 5 minutes

* Exponential backoff on cache spin lock
This commit is contained in:
François-Xavier Talbot
2026-01-08 14:47:13 -05:00
committed by GitHub
parent ea17534f77
commit ff222aa168
13 changed files with 269 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
apps/labrinth/src/database/models/categories.rs
View File

@@ -93,14 +93,16 @@ impl Category {
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let mut redis = redis.connect().await?;
{
let mut redis = redis.connect().await?;
let res: Option<Vec<Category>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "category")
.await?;
let res: Option<Vec<Category>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "category")
.await?;
if let Some(res) = res {
return Ok(res);
if let Some(res) = res {
return Ok(res);
}
}
let result = sqlx::query!(
@@ -122,6 +124,8 @@ impl Category {
.try_collect::<Vec<Category>>()
.await?;
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(TAGS_NAMESPACE, "category", &result, None)
.await?;
@@ -158,14 +162,16 @@ impl LinkPlatform {
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let mut redis = redis.connect().await?;
{
let mut redis = redis.connect().await?;
let res: Option<Vec<LinkPlatform>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "link_platform")
.await?;
let res: Option<Vec<LinkPlatform>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "link_platform")
.await?;
if let Some(res) = res {
return Ok(res);
if let Some(res) = res {
return Ok(res);
}
}
let result = sqlx::query!(
@@ -182,6 +188,8 @@ impl LinkPlatform {
.try_collect::<Vec<LinkPlatform>>()
.await?;
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
TAGS_NAMESPACE,
@@ -223,14 +231,16 @@ impl ReportType {
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let mut redis = redis.connect().await?;
{
let mut redis = redis.connect().await?;
let res: Option<Vec<String>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "report_type")
.await?;
let res: Option<Vec<String>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "report_type")
.await?;
if let Some(res) = res {
return Ok(res);
if let Some(res) = res {
return Ok(res);
}
}
let result = sqlx::query!(
@@ -243,6 +253,8 @@ impl ReportType {
.try_collect::<Vec<String>>()
.await?;
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
TAGS_NAMESPACE,
@@ -284,14 +296,16 @@ impl ProjectType {
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let mut redis = redis.connect().await?;
{
let mut redis = redis.connect().await?;
let res: Option<Vec<String>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "project_type")
.await?;
let res: Option<Vec<String>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "project_type")
.await?;
if let Some(res) = res {
return Ok(res);
if let Some(res) = res {
return Ok(res);
}
}
let result = sqlx::query!(
@@ -304,6 +318,8 @@ impl ProjectType {
.try_collect::<Vec<String>>()
.await?;
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
TAGS_NAMESPACE,

85
apps/labrinth/src/database/models/loader_fields.rs
View File

@@ -50,12 +50,14 @@ impl Game {
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let mut redis = redis.connect().await?;
let cached_games: Option<Vec<Game>> = redis
.get_deserialized_from_json(GAMES_LIST_NAMESPACE, "games")
.await?;
if let Some(cached_games) = cached_games {
return Ok(cached_games);
{
let mut redis = redis.connect().await?;
let cached_games: Option<Vec<Game>> = redis
.get_deserialized_from_json(GAMES_LIST_NAMESPACE, "games")
.await?;
if let Some(cached_games) = cached_games {
return Ok(cached_games);
}
}
let result = sqlx::query!(
@@ -74,6 +76,8 @@ impl Game {
.try_collect::<Vec<Game>>()
.await?;
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
GAMES_LIST_NAMESPACE,
@@ -106,11 +110,13 @@ impl Loader {
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let mut redis = redis.connect().await?;
let cached_id: Option<i32> =
redis.get_deserialized_from_json(LOADER_ID, name).await?;
if let Some(cached_id) = cached_id {
return Ok(Some(LoaderId(cached_id)));
{
let mut redis = redis.connect().await?;
let cached_id: Option<i32> =
redis.get_deserialized_from_json(LOADER_ID, name).await?;
if let Some(cached_id) = cached_id {
return Ok(Some(LoaderId(cached_id)));
}
}
let result = sqlx::query!(
@@ -125,6 +131,7 @@ impl Loader {
.map(|r| LoaderId(r.id));
if let Some(result) = result {
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(LOADER_ID, name, &result.0, None)
.await?;
@@ -140,12 +147,14 @@ impl Loader {
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let mut redis = redis.connect().await?;
let cached_loaders: Option<Vec<Loader>> = redis
.get_deserialized_from_json(LOADERS_LIST_NAMESPACE, "all")
.await?;
if let Some(cached_loaders) = cached_loaders {
return Ok(cached_loaders);
{
let mut redis = redis.connect().await?;
let cached_loaders: Option<Vec<Loader>> = redis
.get_deserialized_from_json(LOADERS_LIST_NAMESPACE, "all")
.await?;
if let Some(cached_loaders) = cached_loaders {
return Ok(cached_loaders);
}
}
let result = sqlx::query!(
@@ -180,6 +189,8 @@ impl Loader {
.try_collect::<Vec<_>>()
.await?;
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
LOADERS_LIST_NAMESPACE,
@@ -455,15 +466,17 @@ impl LoaderField {
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let mut redis = redis.connect().await?;
{
let mut redis = redis.connect().await?;
let cached_fields: Option<Vec<LoaderField>> = redis
.get(LOADER_FIELDS_NAMESPACE_ALL, "")
.await?
.and_then(|x| serde_json::from_str::<Vec<LoaderField>>(&x).ok());
let cached_fields: Option<Vec<LoaderField>> =
redis.get(LOADER_FIELDS_NAMESPACE_ALL, "").await?.and_then(
|x| serde_json::from_str::<Vec<LoaderField>>(&x).ok(),
);
if let Some(cached_fields) = cached_fields {
return Ok(cached_fields);
if let Some(cached_fields) = cached_fields {
return Ok(cached_fields);
}
}
let result = sqlx::query!(
@@ -489,6 +502,8 @@ impl LoaderField {
.flatten()
.collect();
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
LOADER_FIELDS_NAMESPACE_ALL,
@@ -510,16 +525,18 @@ impl LoaderFieldEnum {
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let mut redis = redis.connect().await?;
{
let mut redis = redis.connect().await?;
let cached_enum = redis
.get_deserialized_from_json(
LOADER_FIELD_ENUMS_ID_NAMESPACE,
enum_name,
)
.await?;
if let Some(cached_enum) = cached_enum {
return Ok(cached_enum);
let cached_enum = redis
.get_deserialized_from_json(
LOADER_FIELD_ENUMS_ID_NAMESPACE,
enum_name,
)
.await?;
if let Some(cached_enum) = cached_enum {
return Ok(cached_enum);
}
}
let result = sqlx::query!(
@@ -540,6 +557,8 @@ impl LoaderFieldEnum {
hidable: l.hidable,
});
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
LOADER_FIELD_ENUMS_ID_NAMESPACE,

11
apps/labrinth/src/database/models/mod.rs
View File

@@ -67,6 +67,13 @@ pub enum DatabaseError {
SerdeCacheError(#[from] serde_json::Error),
#[error("Schema error: {0}")]
SchemaError(String),
#[error("Timeout when waiting for cache subscriber")]
CacheTimeout,
#[error(
"Timeout waiting on Redis cache lock ({locks_released}/{locks_waiting} released, spent {time_spent_pool_wait_ms}ms/{time_spent_total_ms}ms waiting on connections from pool)"
)]
CacheTimeout {
locks_released: usize,
locks_waiting: usize,
time_spent_pool_wait_ms: u64,
time_spent_total_ms: u64,
},
}

22
apps/labrinth/src/database/models/notification_item.rs
View File

@@ -380,17 +380,19 @@ impl DBNotification {
where
E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy,
{
let mut redis = redis.connect().await?;
{
let mut redis = redis.connect().await?;
let cached_notifications: Option<Vec<DBNotification>> = redis
.get_deserialized_from_json(
USER_NOTIFICATIONS_NAMESPACE,
&user_id.0.to_string(),
)
.await?;
let cached_notifications: Option<Vec<DBNotification>> = redis
.get_deserialized_from_json(
USER_NOTIFICATIONS_NAMESPACE,
&user_id.0.to_string(),
)
.await?;
if let Some(notifications) = cached_notifications {
return Ok(notifications);
if let Some(notifications) = cached_notifications {
return Ok(notifications);
}
}
let db_notifications = sqlx::query!(
@@ -437,6 +439,8 @@ impl DBNotification {
.try_collect::<Vec<DBNotification>>()
.await?;
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
USER_NOTIFICATIONS_NAMESPACE,

19
apps/labrinth/src/database/models/notifications_template_item.rs
View File

@@ -52,14 +52,19 @@ impl NotificationTemplate {
exec: impl sqlx::Executor<'_, Database = sqlx::Postgres>,
redis: &RedisPool,
) -> Result<Vec<NotificationTemplate>, DatabaseError> {
let mut redis = redis.connect().await?;
{
let mut redis = redis.connect().await?;
let maybe_cached_templates = redis
.get_deserialized_from_json(TEMPLATES_NAMESPACE, channel.as_str())
.await?;
let maybe_cached_templates = redis
.get_deserialized_from_json(
TEMPLATES_NAMESPACE,
channel.as_str(),
)
.await?;
if let Some(cached) = maybe_cached_templates {
return Ok(cached);
if let Some(cached) = maybe_cached_templates {
return Ok(cached);
}
}
let results = sqlx::query_as!(
@@ -74,6 +79,8 @@ impl NotificationTemplate {
let templates = results.into_iter().map(Into::into).collect();
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
TEMPLATES_NAMESPACE,

16
apps/labrinth/src/database/models/notifications_type_item.rs
View File

@@ -39,14 +39,16 @@ impl NotificationTypeItem {
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let mut redis = redis.connect().await?;
{
let mut redis = redis.connect().await?;
let cached_types = redis
.get_deserialized_from_json(NOTIFICATION_TYPES_NAMESPACE, "all")
.await?;
let cached_types = redis
.get_deserialized_from_json(NOTIFICATION_TYPES_NAMESPACE, "all")
.await?;
if let Some(types) = cached_types {
return Ok(types);
if let Some(types) = cached_types {
return Ok(types);
}
}
let results = sqlx::query_as!(
@@ -58,6 +60,8 @@ impl NotificationTypeItem {
let types = results.into_iter().map(Into::into).collect();
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
NOTIFICATION_TYPES_NAMESPACE,

22
apps/labrinth/src/database/models/pat_item.rs
View File

@@ -156,17 +156,19 @@ impl DBPersonalAccessToken {
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let mut redis = redis.connect().await?;
{
let mut redis = redis.connect().await?;
let res = redis
.get_deserialized_from_json::<Vec<i64>>(
PATS_USERS_NAMESPACE,
&user_id.0.to_string(),
)
.await?;
let res = redis
.get_deserialized_from_json::<Vec<i64>>(
PATS_USERS_NAMESPACE,
&user_id.0.to_string(),
)
.await?;
if let Some(res) = res {
return Ok(res.into_iter().map(DBPatId).collect());
if let Some(res) = res {
return Ok(res.into_iter().map(DBPatId).collect());
}
}
let db_pats: Vec<DBPatId> = sqlx::query!(
@@ -183,6 +185,8 @@ impl DBPersonalAccessToken {
.try_collect::<Vec<DBPatId>>()
.await?;
let mut redis = redis.connect().await?;
redis
.set(
PATS_USERS_NAMESPACE,

16
apps/labrinth/src/database/models/product_item.rs
View File

@@ -150,14 +150,16 @@ impl QueryProductWithPrices {
where
E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy,
{
let mut redis = redis.connect().await?;
{
let mut redis = redis.connect().await?;
let res: Option<Vec<QueryProductWithPrices>> = redis
.get_deserialized_from_json(PRODUCTS_NAMESPACE, "all")
.await?;
let res: Option<Vec<QueryProductWithPrices>> = redis
.get_deserialized_from_json(PRODUCTS_NAMESPACE, "all")
.await?;
if let Some(res) = res {
return Ok(res);
if let Some(res) = res {
return Ok(res);
}
}
let all_products = product_item::DBProduct::get_all(exec).await?;
@@ -191,6 +193,8 @@ impl QueryProductWithPrices {
})
.collect::<Vec<_>>();
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(PRODUCTS_NAMESPACE, "all", &products, None)
.await?;

22
apps/labrinth/src/database/models/project_item.rs
View File

@@ -893,16 +893,18 @@ impl DBProject {
Option<DBProjectId>,
)>;
let mut redis = redis.connect().await?;
{
let mut redis = redis.connect().await?;
let dependencies = redis
.get_deserialized_from_json::<Dependencies>(
PROJECTS_DEPENDENCIES_NAMESPACE,
&id.0.to_string(),
)
.await?;
if let Some(dependencies) = dependencies {
return Ok(dependencies);
let dependencies = redis
.get_deserialized_from_json::<Dependencies>(
PROJECTS_DEPENDENCIES_NAMESPACE,
&id.0.to_string(),
)
.await?;
if let Some(dependencies) = dependencies {
return Ok(dependencies);
}
}
let dependencies: Dependencies = sqlx::query!(
@@ -930,6 +932,8 @@ impl DBProject {
.try_collect::<Dependencies>()
.await?;
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
PROJECTS_DEPENDENCIES_NAMESPACE,

22
apps/labrinth/src/database/models/session_item.rs
View File

@@ -209,17 +209,19 @@ impl DBSession {
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let mut redis = redis.connect().await?;
{
let mut redis = redis.connect().await?;
let res = redis
.get_deserialized_from_json::<Vec<i64>>(
SESSIONS_USERS_NAMESPACE,
&user_id.0.to_string(),
)
.await?;
let res = redis
.get_deserialized_from_json::<Vec<i64>>(
SESSIONS_USERS_NAMESPACE,
&user_id.0.to_string(),
)
.await?;
if let Some(res) = res {
return Ok(res.into_iter().map(DBSessionId).collect());
if let Some(res) = res {
return Ok(res.into_iter().map(DBSessionId).collect());
}
}
use futures::TryStreamExt;
@@ -237,6 +239,8 @@ impl DBSession {
.try_collect::<Vec<DBSessionId>>()
.await?;
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
SESSIONS_USERS_NAMESPACE,

22
apps/labrinth/src/database/models/user_item.rs
View File

@@ -299,17 +299,19 @@ impl DBUser {
{
use futures::stream::TryStreamExt;
let mut redis = redis.connect().await?;
{
let mut redis = redis.connect().await?;
let cached_projects = redis
.get_deserialized_from_json::<Vec<DBProjectId>>(
USERS_PROJECTS_NAMESPACE,
&user_id.0.to_string(),
)
.await?;
let cached_projects = redis
.get_deserialized_from_json::<Vec<DBProjectId>>(
USERS_PROJECTS_NAMESPACE,
&user_id.0.to_string(),
)
.await?;
if let Some(projects) = cached_projects {
return Ok(projects);
if let Some(projects) = cached_projects {
return Ok(projects);
}
}
let db_projects = sqlx::query!(
@@ -326,6 +328,8 @@ impl DBUser {
.try_collect::<Vec<DBProjectId>>()
.await?;
let mut redis = redis.connect().await?;
redis
.set_serialized_to_json(
USERS_PROJECTS_NAMESPACE,