PG/RedisPool configuration/o11y improvements (#5032)

* Don't retain Redis connections while doing database queries

* Optional REDIS_WAIT_TIMEOUT_MS

* Attach more data to CacheTimeout errrors

* Fix locks_released

* Fmt

* Set default REDIS_WAIT_TIMEOUT_MS to 15s

* Fix lint

* Close Redis connections idle for > 5 minutes

* Exponential backoff on cache spin lock

apps/labrinth/src/database/models/notifications_template_item.rs

@@ -52,14 +52,19 @@ impl NotificationTemplate {
         exec: impl sqlx::Executor<'_, Database = sqlx::Postgres>,
         redis: &RedisPool,
     ) -> Result<Vec<NotificationTemplate>, DatabaseError> {
-        let mut redis = redis.connect().await?;
+        {
+            let mut redis = redis.connect().await?;
 
-        let maybe_cached_templates = redis
-            .get_deserialized_from_json(TEMPLATES_NAMESPACE, channel.as_str())
-            .await?;
+            let maybe_cached_templates = redis
+                .get_deserialized_from_json(
+                    TEMPLATES_NAMESPACE,
+                    channel.as_str(),
+                )
+                .await?;
 
-        if let Some(cached) = maybe_cached_templates {
-            return Ok(cached);
+            if let Some(cached) = maybe_cached_templates {
+                return Ok(cached);
+            }
         }
 
         let results = sqlx::query_as!(
@@ -74,6 +79,8 @@ impl NotificationTemplate {
 
         let templates = results.into_iter().map(Into::into).collect();
 
+        let mut redis = redis.connect().await?;
+
         redis
             .set_serialized_to_json(
                 TEMPLATES_NAMESPACE,