tomasalias/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Compiler improvements (#753)

Browse Source 
* basic redis add

* toml; reverted unnecessary changes

* merge issues

* increased test connections

---------

Co-authored-by: Geometrically <18202329+Geometrically@users.noreply.github.com>
This commit is contained in:
Wyatt Verchere
2023-11-19 19:10:13 -08:00
committed by GitHub
parent e06a77af28
commit dfba6c7c91
22 changed files with 307 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
Cargo.toml
View File

@@ -109,3 +109,9 @@ derive-new = "0.5.9"
[dev-dependencies] [dev-dependencies]
actix-http = "3.4.0" actix-http = "3.4.0"
[profile.dev]
opt-level = 0 # Minimal optimization, speeds up compilation
lto = false # Disables Link Time Optimization
incremental = true # Enables incremental compilation
codegen-units = 16 # Higher number can improve compile times but reduce runtime performance

8
src/database/models/categories.rs
View File

@@ -90,6 +90,8 @@ impl Category {
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{ {
let mut redis = redis.connect().await?;
let res: Option<Vec<Category>> = redis let res: Option<Vec<Category>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "category") .get_deserialized_from_json(TAGS_NAMESPACE, "category")
.await?; .await?;
@@ -155,6 +157,8 @@ impl DonationPlatform {
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{ {
let mut redis = redis.connect().await?;
let res: Option<Vec<DonationPlatform>> = redis let res: Option<Vec<DonationPlatform>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "donation_platform") .get_deserialized_from_json(TAGS_NAMESPACE, "donation_platform")
.await?; .await?;
@@ -209,6 +213,8 @@ impl ReportType {
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{ {
let mut redis = redis.connect().await?;
let res: Option<Vec<String>> = redis let res: Option<Vec<String>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "report_type") .get_deserialized_from_json(TAGS_NAMESPACE, "report_type")
.await?; .await?;
@@ -257,6 +263,8 @@ impl ProjectType {
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{ {
let mut redis = redis.connect().await?;
let res: Option<Vec<String>> = redis let res: Option<Vec<String>> = redis
.get_deserialized_from_json(TAGS_NAMESPACE, "project_type") .get_deserialized_from_json(TAGS_NAMESPACE, "project_type")
.await?; .await?;

9
src/database/models/collection_item.rs
View File

@@ -157,6 +157,8 @@ impl Collection {
{ {
use futures::TryStreamExt; use futures::TryStreamExt;
let mut redis = redis.connect().await?;
if collection_ids.is_empty() { if collection_ids.is_empty() {
return Ok(Vec::new()); return Ok(Vec::new());
} }
@@ -166,7 +168,10 @@ impl Collection {
if !collection_ids.is_empty() { if !collection_ids.is_empty() {
let collections = redis let collections = redis
.multi_get::<String, _>(COLLECTIONS_NAMESPACE, collection_ids.iter().map(|x| x.0)) .multi_get::<String>(
COLLECTIONS_NAMESPACE,
collection_ids.iter().map(|x| x.0.to_string()),
)
.await?; .await?;
for collection in collections { for collection in collections {
@@ -240,6 +245,8 @@ impl Collection {
} }
pub async fn clear_cache(id: CollectionId, redis: &RedisPool) -> Result<(), DatabaseError> { pub async fn clear_cache(id: CollectionId, redis: &RedisPool) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
redis.delete(COLLECTIONS_NAMESPACE, id.0).await?; redis.delete(COLLECTIONS_NAMESPACE, id.0).await?;
Ok(()) Ok(())
} }

6
src/database/models/flow_item.rs
View File

@@ -58,6 +58,8 @@ impl Flow {
expires: Duration, expires: Duration,
redis: &RedisPool, redis: &RedisPool,
) -> Result<String, DatabaseError> { ) -> Result<String, DatabaseError> {
let mut redis = redis.connect().await?;
let flow = ChaCha20Rng::from_entropy() let flow = ChaCha20Rng::from_entropy()
.sample_iter(&Alphanumeric) .sample_iter(&Alphanumeric)
.take(32) .take(32)
@@ -71,6 +73,8 @@ impl Flow {
} }
pub async fn get(id: &str, redis: &RedisPool) -> Result<Option<Flow>, DatabaseError> { pub async fn get(id: &str, redis: &RedisPool) -> Result<Option<Flow>, DatabaseError> {
let mut redis = redis.connect().await?;
redis.get_deserialized_from_json(FLOWS_NAMESPACE, id).await redis.get_deserialized_from_json(FLOWS_NAMESPACE, id).await
} }
@@ -91,6 +95,8 @@ impl Flow {
} }
pub async fn remove(id: &str, redis: &RedisPool) -> Result<Option<()>, DatabaseError> { pub async fn remove(id: &str, redis: &RedisPool) -> Result<Option<()>, DatabaseError> {
let mut redis = redis.connect().await?;
redis.delete(FLOWS_NAMESPACE, id).await?; redis.delete(FLOWS_NAMESPACE, id).await?;
Ok(Some(())) Ok(Some(()))
} }

5
src/database/models/image_item.rs
View File

@@ -180,6 +180,7 @@ impl Image {
{ {
use futures::TryStreamExt; use futures::TryStreamExt;
let mut redis = redis.connect().await?;
if image_ids.is_empty() { if image_ids.is_empty() {
return Ok(Vec::new()); return Ok(Vec::new());
} }
@@ -191,7 +192,7 @@ impl Image {
if !image_ids.is_empty() { if !image_ids.is_empty() {
let images = redis let images = redis
.multi_get::<String, _>(IMAGES_NAMESPACE, image_ids) .multi_get::<String>(IMAGES_NAMESPACE, image_ids.iter().map(|x| x.to_string()))
.await?; .await?;
for image in images { for image in images {
if let Some(image) = image.and_then(|x| serde_json::from_str::<Image>(&x).ok()) { if let Some(image) = image.and_then(|x| serde_json::from_str::<Image>(&x).ok()) {
@@ -246,6 +247,8 @@ impl Image {
} }
pub async fn clear_cache(id: ImageId, redis: &RedisPool) -> Result<(), DatabaseError> { pub async fn clear_cache(id: ImageId, redis: &RedisPool) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
redis.delete(IMAGES_NAMESPACE, id.0).await?; redis.delete(IMAGES_NAMESPACE, id.0).await?;
Ok(()) Ok(())
} }

12
src/database/models/loader_fields.rs
View File

@@ -44,6 +44,7 @@ impl Game {
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{ {
let mut redis = redis.connect().await?;
let cached_games: Option<Vec<Game>> = redis let cached_games: Option<Vec<Game>> = redis
.get_deserialized_from_json(GAMES_LIST_NAMESPACE, "games") .get_deserialized_from_json(GAMES_LIST_NAMESPACE, "games")
.await?; .await?;
@@ -95,6 +96,7 @@ impl Loader {
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{ {
let mut redis = redis.connect().await?;
let cached_id: Option<i32> = redis.get_deserialized_from_json(LOADER_ID, name).await?; let cached_id: Option<i32> = redis.get_deserialized_from_json(LOADER_ID, name).await?;
if let Some(cached_id) = cached_id { if let Some(cached_id) = cached_id {
return Ok(Some(LoaderId(cached_id))); return Ok(Some(LoaderId(cached_id)));
@@ -124,6 +126,7 @@ impl Loader {
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{ {
let mut redis = redis.connect().await?;
let cached_loaders: Option<Vec<Loader>> = redis let cached_loaders: Option<Vec<Loader>> = redis
.get_deserialized_from_json(LOADERS_LIST_NAMESPACE, "all") .get_deserialized_from_json(LOADERS_LIST_NAMESPACE, "all")
.await?; .await?;
@@ -318,9 +321,11 @@ impl LoaderField {
{ {
type RedisLoaderFieldTuple = (LoaderId, Vec<LoaderField>); type RedisLoaderFieldTuple = (LoaderId, Vec<LoaderField>);
let mut redis = redis.connect().await?;
let mut loader_ids = loader_ids.to_vec(); let mut loader_ids = loader_ids.to_vec();
let cached_fields: Vec<RedisLoaderFieldTuple> = redis let cached_fields: Vec<RedisLoaderFieldTuple> = redis
.multi_get::<String, _>(LOADER_FIELDS_NAMESPACE, loader_ids.iter().map(|x| x.0)) .multi_get::<String>(LOADER_FIELDS_NAMESPACE, loader_ids.iter().map(|x| x.0))
.await? .await?
.into_iter() .into_iter()
.flatten() .flatten()
@@ -399,6 +404,8 @@ impl LoaderFieldEnum {
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{ {
let mut redis = redis.connect().await?;
let cached_enum = redis let cached_enum = redis
.get_deserialized_from_json(LOADER_FIELD_ENUMS_ID_NAMESPACE, enum_name) .get_deserialized_from_json(LOADER_FIELD_ENUMS_ID_NAMESPACE, enum_name)
.await?; .await?;
@@ -488,12 +495,13 @@ impl LoaderFieldEnumValue {
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{ {
let mut redis = redis.connect().await?;
let mut found_enums = Vec::new(); let mut found_enums = Vec::new();
let mut remaining_enums: Vec<LoaderFieldEnumId> = loader_field_enum_ids.to_vec(); let mut remaining_enums: Vec<LoaderFieldEnumId> = loader_field_enum_ids.to_vec();
if !remaining_enums.is_empty() { if !remaining_enums.is_empty() {
let enums = redis let enums = redis
.multi_get::<String, _>( .multi_get::<String>(
LOADER_FIELD_ENUM_VALUES_NAMESPACE, LOADER_FIELD_ENUM_VALUES_NAMESPACE,
loader_field_enum_ids.iter().map(|x| x.0), loader_field_enum_ids.iter().map(|x| x.0),
) )

6
src/database/models/notification_item.rs
View File

@@ -174,8 +174,10 @@ impl Notification {
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy, E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy,
{ {
let mut redis = redis.connect().await?;
let cached_notifications: Option<Vec<Notification>> = redis let cached_notifications: Option<Vec<Notification>> = redis
.get_deserialized_from_json(USER_NOTIFICATIONS_NAMESPACE, user_id.0) .get_deserialized_from_json(USER_NOTIFICATIONS_NAMESPACE, &user_id.0.to_string())
.await?; .await?;
if let Some(notifications) = cached_notifications { if let Some(notifications) = cached_notifications {
@@ -319,6 +321,8 @@ impl Notification {
user_ids: impl IntoIterator<Item = &UserId>, user_ids: impl IntoIterator<Item = &UserId>,
redis: &RedisPool, redis: &RedisPool,
) -> Result<(), DatabaseError> { ) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
redis redis
.delete_many( .delete_many(
user_ids user_ids

18
src/database/models/organization_item.rs
View File

@@ -103,6 +103,8 @@ impl Organization {
{ {
use futures::stream::TryStreamExt; use futures::stream::TryStreamExt;
let mut redis = redis.connect().await?;
if organization_strings.is_empty() { if organization_strings.is_empty() {
return Ok(Vec::new()); return Ok(Vec::new());
} }
@@ -120,11 +122,12 @@ impl Organization {
organization_ids.append( organization_ids.append(
&mut redis &mut redis
.multi_get::<i64, _>( .multi_get::<i64>(
ORGANIZATIONS_TITLES_NAMESPACE, ORGANIZATIONS_TITLES_NAMESPACE,
organization_strings organization_strings
.iter() .iter()
.map(|x| x.to_string().to_lowercase()), .map(|x| x.to_string().to_lowercase())
.collect::<Vec<_>>(),
) )
.await? .await?
.into_iter() .into_iter()
@@ -134,7 +137,10 @@ impl Organization {
if !organization_ids.is_empty() { if !organization_ids.is_empty() {
let organizations = redis let organizations = redis
.multi_get::<String, _>(ORGANIZATIONS_NAMESPACE, organization_ids) .multi_get::<String>(
ORGANIZATIONS_NAMESPACE,
organization_ids.iter().map(|x| x.to_string()),
)
.await?; .await?;
for organization in organizations { for organization in organizations {
@@ -197,8 +203,8 @@ impl Organization {
redis redis
.set( .set(
ORGANIZATIONS_TITLES_NAMESPACE, ORGANIZATIONS_TITLES_NAMESPACE,
organization.title.to_lowercase(), &organization.title.to_lowercase(),
organization.id.0, &organization.id.0.to_string(),
None, None,
) )
.await?; .await?;
@@ -318,6 +324,8 @@ impl Organization {
title: Option<String>, title: Option<String>,
redis: &RedisPool, redis: &RedisPool,
) -> Result<(), super::DatabaseError> { ) -> Result<(), super::DatabaseError> {
let mut redis = redis.connect().await?;
redis redis
.delete_many([ .delete_many([
(ORGANIZATIONS_NAMESPACE, Some(id.0.to_string())), (ORGANIZATIONS_NAMESPACE, Some(id.0.to_string())),

20
src/database/models/pat_item.rs
View File

@@ -89,6 +89,8 @@ impl PersonalAccessToken {
{ {
use futures::TryStreamExt; use futures::TryStreamExt;
let mut redis = redis.connect().await?;
if pat_strings.is_empty() { if pat_strings.is_empty() {
return Ok(Vec::new()); return Ok(Vec::new());
} }
@@ -106,7 +108,7 @@ impl PersonalAccessToken {
pat_ids.append( pat_ids.append(
&mut redis &mut redis
.multi_get::<i64, _>( .multi_get::<i64>(
PATS_TOKENS_NAMESPACE, PATS_TOKENS_NAMESPACE,
pat_strings.iter().map(|x| x.to_string()), pat_strings.iter().map(|x| x.to_string()),
) )
@@ -118,7 +120,7 @@ impl PersonalAccessToken {
if !pat_ids.is_empty() { if !pat_ids.is_empty() {
let pats = redis let pats = redis
.multi_get::<String, _>(PATS_NAMESPACE, pat_ids) .multi_get::<String>(PATS_NAMESPACE, pat_ids.iter().map(|x| x.to_string()))
.await?; .await?;
for pat in pats { for pat in pats {
if let Some(pat) = if let Some(pat) =
@@ -174,8 +176,8 @@ impl PersonalAccessToken {
redis redis
.set( .set(
PATS_TOKENS_NAMESPACE, PATS_TOKENS_NAMESPACE,
pat.access_token.clone(), &pat.access_token,
pat.id.0, &pat.id.0.to_string(),
None, None,
) )
.await?; .await?;
@@ -194,8 +196,10 @@ impl PersonalAccessToken {
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{ {
let mut redis = redis.connect().await?;
let res = redis let res = redis
.get_deserialized_from_json::<Vec<i64>, _>(PATS_USERS_NAMESPACE, user_id.0) .get_deserialized_from_json::<Vec<i64>>(PATS_USERS_NAMESPACE, &user_id.0.to_string())
.await?; .await?;
if let Some(res) = res { if let Some(res) = res {
@@ -220,8 +224,8 @@ impl PersonalAccessToken {
redis redis
.set( .set(
PATS_USERS_NAMESPACE, PATS_USERS_NAMESPACE,
user_id.0, &user_id.0.to_string(),
serde_json::to_string(&db_pats)?, &serde_json::to_string(&db_pats)?,
None, None,
) )
.await?; .await?;
@@ -232,6 +236,8 @@ impl PersonalAccessToken {
clear_pats: Vec<(Option<PatId>, Option<String>, Option<UserId>)>, clear_pats: Vec<(Option<PatId>, Option<String>, Option<UserId>)>,
redis: &RedisPool, redis: &RedisPool,
) -> Result<(), DatabaseError> { ) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
if clear_pats.is_empty() { if clear_pats.is_empty() {
return Ok(()); return Ok(());
} }

22
src/database/models/project_item.rs
View File

@@ -513,6 +513,8 @@ impl Project {
return Ok(Vec::new()); return Ok(Vec::new());
} }
let mut redis = redis.connect().await?;
let mut found_projects = Vec::new(); let mut found_projects = Vec::new();
let mut remaining_strings = project_strings let mut remaining_strings = project_strings
.iter() .iter()
@@ -526,7 +528,7 @@ impl Project {
project_ids.append( project_ids.append(
&mut redis &mut redis
.multi_get::<i64, _>( .multi_get::<i64>(
PROJECTS_SLUGS_NAMESPACE, PROJECTS_SLUGS_NAMESPACE,
project_strings.iter().map(|x| x.to_string().to_lowercase()), project_strings.iter().map(|x| x.to_string().to_lowercase()),
) )
@@ -537,7 +539,10 @@ impl Project {
); );
if !project_ids.is_empty() { if !project_ids.is_empty() {
let projects = redis let projects = redis
.multi_get::<String, _>(PROJECTS_NAMESPACE, project_ids) .multi_get::<String>(
PROJECTS_NAMESPACE,
project_ids.iter().map(|x| x.to_string()),
)
.await?; .await?;
for project in projects { for project in projects {
if let Some(project) = if let Some(project) =
@@ -686,8 +691,8 @@ impl Project {
redis redis
.set( .set(
PROJECTS_SLUGS_NAMESPACE, PROJECTS_SLUGS_NAMESPACE,
slug.to_lowercase(), &slug.to_lowercase(),
project.inner.id.0, &project.inner.id.0.to_string(),
None, None,
) )
.await?; .await?;
@@ -709,8 +714,13 @@ impl Project {
{ {
type Dependencies = Vec<(Option<VersionId>, Option<ProjectId>, Option<ProjectId>)>; type Dependencies = Vec<(Option<VersionId>, Option<ProjectId>, Option<ProjectId>)>;
let mut redis = redis.connect().await?;
let dependencies = redis let dependencies = redis
.get_deserialized_from_json::<Dependencies, _>(PROJECTS_DEPENDENCIES_NAMESPACE, id.0) .get_deserialized_from_json::<Dependencies>(
PROJECTS_DEPENDENCIES_NAMESPACE,
&id.0.to_string(),
)
.await?; .await?;
if let Some(dependencies) = dependencies { if let Some(dependencies) = dependencies {
return Ok(dependencies); return Ok(dependencies);
@@ -755,6 +765,8 @@ impl Project {
clear_dependencies: Option<bool>, clear_dependencies: Option<bool>,
redis: &RedisPool, redis: &RedisPool,
) -> Result<(), DatabaseError> { ) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
redis redis
.delete_many([ .delete_many([
(PROJECTS_NAMESPACE, Some(id.0.to_string())), (PROJECTS_NAMESPACE, Some(id.0.to_string())),

22
src/database/models/session_item.rs
View File

@@ -130,6 +130,8 @@ impl Session {
{ {
use futures::TryStreamExt; use futures::TryStreamExt;
let mut redis = redis.connect().await?;
if session_strings.is_empty() { if session_strings.is_empty() {
return Ok(Vec::new()); return Ok(Vec::new());
} }
@@ -147,7 +149,7 @@ impl Session {
session_ids.append( session_ids.append(
&mut redis &mut redis
.multi_get::<i64, _>( .multi_get::<i64>(
SESSIONS_IDS_NAMESPACE, SESSIONS_IDS_NAMESPACE,
session_strings.iter().map(|x| x.to_string()), session_strings.iter().map(|x| x.to_string()),
) )
@@ -159,7 +161,10 @@ impl Session {
if !session_ids.is_empty() { if !session_ids.is_empty() {
let sessions = redis let sessions = redis
.multi_get::<String, _>(SESSIONS_NAMESPACE, session_ids) .multi_get::<String>(
SESSIONS_NAMESPACE,
session_ids.iter().map(|x| x.to_string()),
)
.await?; .await?;
for session in sessions { for session in sessions {
if let Some(session) = if let Some(session) =
@@ -218,8 +223,8 @@ impl Session {
redis redis
.set( .set(
SESSIONS_IDS_NAMESPACE, SESSIONS_IDS_NAMESPACE,
session.session.clone(), &session.session,
session.id.0, &session.id.0.to_string(),
None, None,
) )
.await?; .await?;
@@ -238,8 +243,13 @@ impl Session {
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{ {
let mut redis = redis.connect().await?;
let res = redis let res = redis
.get_deserialized_from_json::<Vec<i64>, _>(SESSIONS_USERS_NAMESPACE, user_id.0) .get_deserialized_from_json::<Vec<i64>>(
SESSIONS_USERS_NAMESPACE,
&user_id.0.to_string(),
)
.await?; .await?;
if let Some(res) = res { if let Some(res) = res {
@@ -272,6 +282,8 @@ impl Session {
clear_sessions: Vec<(Option<SessionId>, Option<String>, Option<UserId>)>, clear_sessions: Vec<(Option<SessionId>, Option<String>, Option<UserId>)>,
redis: &RedisPool, redis: &RedisPool,
) -> Result<(), DatabaseError> { ) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
if clear_sessions.is_empty() { if clear_sessions.is_empty() {
return Ok(()); return Ok(());
} }

10
src/database/models/team_item.rs
View File

@@ -197,18 +197,23 @@ impl TeamMember {
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy, E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy,
{ {
use futures::stream::TryStreamExt;
if team_ids.is_empty() { if team_ids.is_empty() {
return Ok(Vec::new()); return Ok(Vec::new());
} }
use futures::stream::TryStreamExt; let mut redis = redis.connect().await?;
let mut team_ids_parsed: Vec<i64> = team_ids.iter().map(|x| x.0).collect(); let mut team_ids_parsed: Vec<i64> = team_ids.iter().map(|x| x.0).collect();
let mut found_teams = Vec::new(); let mut found_teams = Vec::new();
let teams = redis let teams = redis
.multi_get::<String, _>(TEAMS_NAMESPACE, team_ids_parsed.clone()) .multi_get::<String>(
TEAMS_NAMESPACE,
team_ids_parsed.iter().map(|x| x.to_string()),
)
.await?; .await?;
for team_raw in teams { for team_raw in teams {
@@ -271,6 +276,7 @@ impl TeamMember {
} }
pub async fn clear_cache(id: TeamId, redis: &RedisPool) -> Result<(), super::DatabaseError> { pub async fn clear_cache(id: TeamId, redis: &RedisPool) -> Result<(), super::DatabaseError> {
let mut redis = redis.connect().await?;
redis.delete(TEAMS_NAMESPACE, id.0).await?; redis.delete(TEAMS_NAMESPACE, id.0).await?;
Ok(()) Ok(())
} }

21
src/database/models/user_item.rs
View File

@@ -134,6 +134,8 @@ impl User {
{ {
use futures::TryStreamExt; use futures::TryStreamExt;
let mut redis = redis.connect().await?;
if users_strings.is_empty() { if users_strings.is_empty() {
return Ok(Vec::new()); return Ok(Vec::new());
} }
@@ -151,7 +153,7 @@ impl User {
user_ids.append( user_ids.append(
&mut redis &mut redis
.multi_get::<i64, _>( .multi_get::<i64>(
USER_USERNAMES_NAMESPACE, USER_USERNAMES_NAMESPACE,
users_strings.iter().map(|x| x.to_string().to_lowercase()), users_strings.iter().map(|x| x.to_string().to_lowercase()),
) )
@@ -163,7 +165,7 @@ impl User {
if !user_ids.is_empty() { if !user_ids.is_empty() {
let users = redis let users = redis
.multi_get::<String, _>(USERS_NAMESPACE, user_ids) .multi_get::<String>(USERS_NAMESPACE, user_ids.iter().map(|x| x.to_string()))
.await?; .await?;
for user in users { for user in users {
if let Some(user) = user.and_then(|x| serde_json::from_str::<User>(&x).ok()) { if let Some(user) = user.and_then(|x| serde_json::from_str::<User>(&x).ok()) {
@@ -239,8 +241,8 @@ impl User {
redis redis
.set( .set(
USER_USERNAMES_NAMESPACE, USER_USERNAMES_NAMESPACE,
user.username.to_lowercase(), &user.username.to_lowercase(),
user.id.0, &user.id.0.to_string(),
None, None,
) )
.await?; .await?;
@@ -278,8 +280,13 @@ impl User {
{ {
use futures::stream::TryStreamExt; use futures::stream::TryStreamExt;
let mut redis = redis.connect().await?;
let cached_projects = redis let cached_projects = redis
.get_deserialized_from_json::<Vec<ProjectId>, _>(USERS_PROJECTS_NAMESPACE, user_id.0) .get_deserialized_from_json::<Vec<ProjectId>>(
USERS_PROJECTS_NAMESPACE,
&user_id.0.to_string(),
)
.await?; .await?;
if let Some(projects) = cached_projects { if let Some(projects) = cached_projects {
@@ -384,6 +391,8 @@ impl User {
user_ids: &[(UserId, Option<String>)], user_ids: &[(UserId, Option<String>)],
redis: &RedisPool, redis: &RedisPool,
) -> Result<(), DatabaseError> { ) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
redis redis
.delete_many(user_ids.iter().flat_map(|(id, username)| { .delete_many(user_ids.iter().flat_map(|(id, username)| {
[ [
@@ -402,6 +411,8 @@ impl User {
user_ids: &[UserId], user_ids: &[UserId],
redis: &RedisPool, redis: &RedisPool,
) -> Result<(), DatabaseError> { ) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
redis redis
.delete_many( .delete_many(
user_ids user_ids

23
src/database/models/version_item.rs
View File

@@ -492,18 +492,27 @@ impl Version {
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres>, E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{ {
use futures::stream::TryStreamExt;
if version_ids.is_empty() { if version_ids.is_empty() {
return Ok(Vec::new()); return Ok(Vec::new());
} }
use futures::stream::TryStreamExt; let mut redis = redis.connect().await?;
let mut version_ids_parsed: Vec<i64> = version_ids.iter().map(|x| x.0).collect(); let mut version_ids_parsed: Vec<i64> = version_ids.iter().map(|x| x.0).collect();
let mut found_versions = Vec::new(); let mut found_versions = Vec::new();
let versions = redis let versions = redis
.multi_get::<String, _>(VERSIONS_NAMESPACE, version_ids_parsed.clone()) .multi_get::<String>(
VERSIONS_NAMESPACE,
version_ids_parsed
.clone()
.iter()
.map(|x| x.to_string())
.collect::<Vec<_>>(),
)
.await?; .await?;
for version in versions { for version in versions {
@@ -721,18 +730,20 @@ impl Version {
where where
E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy, E: sqlx::Executor<'a, Database = sqlx::Postgres> + Copy,
{ {
use futures::stream::TryStreamExt;
let mut redis = redis.connect().await?;
if hashes.is_empty() { if hashes.is_empty() {
return Ok(Vec::new()); return Ok(Vec::new());
} }
use futures::stream::TryStreamExt;
let mut file_ids_parsed = hashes.to_vec(); let mut file_ids_parsed = hashes.to_vec();
let mut found_files = Vec::new(); let mut found_files = Vec::new();
let files = redis let files = redis
.multi_get::<String, _>( .multi_get::<String>(
VERSION_FILES_NAMESPACE, VERSION_FILES_NAMESPACE,
file_ids_parsed file_ids_parsed
.iter() .iter()
@@ -829,6 +840,8 @@ impl Version {
version: &QueryVersion, version: &QueryVersion,
redis: &RedisPool, redis: &RedisPool,
) -> Result<(), DatabaseError> { ) -> Result<(), DatabaseError> {
let mut redis = redis.connect().await?;
redis redis
.delete_many( .delete_many(
iter::once((VERSIONS_NAMESPACE, Some(version.inner.id.0.to_string()))).chain( iter::once((VERSIONS_NAMESPACE, Some(version.inner.id.0.to_string()))).chain(

169
src/database/redis.rs
View File

@@ -1,6 +1,7 @@
use super::models::DatabaseError; use super::models::DatabaseError;
use deadpool_redis::{Config, Runtime}; use deadpool_redis::{Config, Runtime};
use redis::{cmd, FromRedisValue, ToRedisArgs}; use itertools::Itertools;
use redis::{cmd, Cmd};
use std::fmt::Display; use std::fmt::Display;
const DEFAULT_EXPIRY: i64 = 1800; // 30 minutes const DEFAULT_EXPIRY: i64 = 1800; // 30 minutes
@@ -11,6 +12,11 @@ pub struct RedisPool {
meta_namespace: String, meta_namespace: String,
} }
pub struct RedisConnection {
pub connection: deadpool_redis::Connection,
meta_namespace: String,
}
impl RedisPool { impl RedisPool {
// initiate a new redis pool // initiate a new redis pool
// testing pool uses a hashmap to mimic redis behaviour for very small data sizes (ie: tests) // testing pool uses a hashmap to mimic redis behaviour for very small data sizes (ie: tests)
@@ -35,32 +41,39 @@ impl RedisPool {
} }
} }
pub async fn set<T1, T2>( pub async fn connect(&self) -> Result<RedisConnection, DatabaseError> {
&self, Ok(RedisConnection {
connection: self.pool.get().await?,
meta_namespace: self.meta_namespace.clone(),
})
}
}
impl RedisConnection {
pub async fn set(
&mut self,
namespace: &str, namespace: &str,
id: T1, id: &str,
data: T2, data: &str,
expiry: Option<i64>, expiry: Option<i64>,
) -> Result<(), DatabaseError> ) -> Result<(), DatabaseError> {
where let mut cmd = cmd("SET");
T1: Display, redis_args(
T2: ToRedisArgs, &mut cmd,
{ vec![
let mut redis_connection = self.pool.get().await?; format!("{}_{}:{}", self.meta_namespace, namespace, id),
data.to_string(),
cmd("SET") "EX".to_string(),
.arg(format!("{}_{}:{}", self.meta_namespace, namespace, id)) expiry.unwrap_or(DEFAULT_EXPIRY).to_string(),
.arg(data) ]
.arg("EX") .as_slice(),
.arg(expiry.unwrap_or(DEFAULT_EXPIRY)) );
.query_async::<_, ()>(&mut redis_connection) redis_execute(&mut cmd, &mut self.connection).await?;
.await?;
Ok(()) Ok(())
} }
pub async fn set_serialized_to_json<Id, D>( pub async fn set_serialized_to_json<Id, D>(
&self, &mut self,
namespace: &str, namespace: &str,
id: Id, id: Id,
data: D, data: D,
@@ -70,92 +83,116 @@ impl RedisPool {
Id: Display, Id: Display,
D: serde::Serialize, D: serde::Serialize,
{ {
self.set(namespace, id, serde_json::to_string(&data)?, expiry) self.set(
.await namespace,
&id.to_string(),
&serde_json::to_string(&data)?,
expiry,
)
.await
} }
pub async fn get<R, Id>(&self, namespace: &str, id: Id) -> Result<Option<R>, DatabaseError> pub async fn get(
where &mut self,
Id: Display, namespace: &str,
R: FromRedisValue, id: &str,
{ ) -> Result<Option<String>, DatabaseError> {
let mut redis_connection = self.pool.get().await?; let mut cmd = cmd("GET");
redis_args(
let res = cmd("GET") &mut cmd,
.arg(format!("{}_{}:{}", self.meta_namespace, namespace, id)) vec![format!("{}_{}:{}", self.meta_namespace, namespace, id)].as_slice(),
.query_async::<_, Option<R>>(&mut redis_connection) );
.await?; let res = redis_execute(&mut cmd, &mut self.connection).await?;
Ok(res) Ok(res)
} }
pub async fn get_deserialized_from_json<R, Id>( pub async fn get_deserialized_from_json<R>(
&self, &mut self,
namespace: &str, namespace: &str,
id: Id, id: &str,
) -> Result<Option<R>, DatabaseError> ) -> Result<Option<R>, DatabaseError>
where where
Id: Display,
R: for<'a> serde::Deserialize<'a>, R: for<'a> serde::Deserialize<'a>,
{ {
Ok(self Ok(self
.get::<String, Id>(namespace, id) .get(namespace, id)
.await? .await?
.and_then(|x| serde_json::from_str(&x).ok())) .and_then(|x| serde_json::from_str(&x).ok()))
} }
pub async fn multi_get<R, T1>( pub async fn multi_get<R>(
&self, &mut self,
namespace: &str, namespace: &str,
ids: impl IntoIterator<Item = T1>, ids: impl IntoIterator<Item = impl Display>,
) -> Result<Vec<Option<R>>, DatabaseError> ) -> Result<Vec<Option<R>>, DatabaseError>
where where
T1: Display, R: for<'a> serde::Deserialize<'a>,
R: FromRedisValue,
{ {
let mut redis_connection = self.pool.get().await?; let mut cmd = cmd("MGET");
let res = cmd("MGET")
.arg( redis_args(
ids.into_iter() &mut cmd,
.map(|x| format!("{}_{}:{}", self.meta_namespace, namespace, x)) &ids.into_iter()
.collect::<Vec<_>>(), .map(|x| format!("{}_{}:{}", self.meta_namespace, namespace, x))
) .collect_vec(),
.query_async::<_, Vec<Option<R>>>(&mut redis_connection) );
.await?; let res: Vec<Option<String>> = redis_execute(&mut cmd, &mut self.connection).await?;
Ok(res) Ok(res
.into_iter()
.map(|x| x.and_then(|x| serde_json::from_str(&x).ok()))
.collect())
} }
pub async fn delete<T1>(&self, namespace: &str, id: T1) -> Result<(), DatabaseError> pub async fn delete<T1>(&mut self, namespace: &str, id: T1) -> Result<(), DatabaseError>
where where
T1: Display, T1: Display,
{ {
let mut redis_connection = self.pool.get().await?; let mut cmd = cmd("DEL");
redis_args(
cmd("DEL") &mut cmd,
.arg(format!("{}_{}:{}", self.meta_namespace, namespace, id)) vec![format!("{}_{}:{}", self.meta_namespace, namespace, id)].as_slice(),
.query_async::<_, ()>(&mut redis_connection) );
.await?; redis_execute(&mut cmd, &mut self.connection).await?;
Ok(()) Ok(())
} }
pub async fn delete_many( pub async fn delete_many(
&self, &mut self,
iter: impl IntoIterator<Item = (&str, Option<String>)>, iter: impl IntoIterator<Item = (&str, Option<String>)>,
) -> Result<(), DatabaseError> { ) -> Result<(), DatabaseError> {
let mut cmd = cmd("DEL"); let mut cmd = cmd("DEL");
let mut any = false; let mut any = false;
for (namespace, id) in iter { for (namespace, id) in iter {
if let Some(id) = id { if let Some(id) = id {
cmd.arg(format!("{}_{}:{}", self.meta_namespace, namespace, id)); redis_args(
&mut cmd,
[format!("{}_{}:{}", self.meta_namespace, namespace, id)].as_slice(),
);
any = true; any = true;
} }
} }
if any { if any {
let mut redis_connection = self.pool.get().await?; redis_execute(&mut cmd, &mut self.connection).await?;
cmd.query_async::<_, ()>(&mut redis_connection).await?;
} }
Ok(()) Ok(())
} }
} }
pub fn redis_args(cmd: &mut Cmd, args: &[String]) {
for arg in args {
cmd.arg(arg);
}
}
pub async fn redis_execute<T>(
cmd: &mut Cmd,
redis: &mut deadpool_redis::Connection,
) -> Result<T, deadpool_redis::PoolError>
where
T: redis::FromRedisValue,
{
let res = cmd.query_async::<_, T>(redis).await?;
Ok(res)
}

1
src/util/mod.rs
View File

@@ -7,6 +7,7 @@ pub mod env;
pub mod ext; pub mod ext;
pub mod guards; pub mod guards;
pub mod img; pub mod img;
pub mod redis;
pub mod routes; pub mod routes;
pub mod validate; pub mod validate;
pub mod webhook; pub mod webhook;

18
src/util/redis.rs Normal file
View File

@@ -0,0 +1,18 @@
use redis::Cmd;
pub fn redis_args(cmd: &mut Cmd, args: &[String]) {
for arg in args {
cmd.arg(arg);
}
}
pub async fn redis_execute<T>(
cmd: &mut Cmd,
redis: &mut deadpool_redis::Connection,
) -> Result<T, deadpool_redis::PoolError>
where
T: redis::FromRedisValue,
{
let res = cmd.query_async::<_, T>(redis).await?;
Ok(res)
}

38
tests/project.rs
View File

@@ -40,20 +40,21 @@ async fn test_get_project() {
assert_eq!(versions[0], json!(alpha_version_id)); assert_eq!(versions[0], json!(alpha_version_id));
// Confirm that the request was cached // Confirm that the request was cached
let mut redis_pool = test_env.db.redis_pool.connect().await.unwrap();
assert_eq!( assert_eq!(
test_env redis_pool
.db .get(PROJECTS_SLUGS_NAMESPACE, alpha_project_slug)
.redis_pool
.get::<i64, _>(PROJECTS_SLUGS_NAMESPACE, alpha_project_slug)
.await .await
.unwrap(), .unwrap()
.and_then(|x| x.parse::<i64>().ok()),
Some(parse_base62(alpha_project_id).unwrap() as i64) Some(parse_base62(alpha_project_id).unwrap() as i64)
); );
let cached_project = test_env let cached_project = redis_pool
.db .get(
.redis_pool PROJECTS_NAMESPACE,
.get::<String, _>(PROJECTS_NAMESPACE, parse_base62(alpha_project_id).unwrap()) &parse_base62(alpha_project_id).unwrap().to_string(),
)
.await .await
.unwrap() .unwrap()
.unwrap(); .unwrap();
@@ -249,22 +250,21 @@ async fn test_add_remove_project() {
assert_eq!(resp.status(), 204); assert_eq!(resp.status(), 204);
// Confirm that the project is gone from the cache // Confirm that the project is gone from the cache
let mut redis_pool = test_env.db.redis_pool.connect().await.unwrap();
assert_eq!( assert_eq!(
test_env redis_pool
.db .get(PROJECTS_SLUGS_NAMESPACE, "demo")
.redis_pool
.get::<i64, _>(PROJECTS_SLUGS_NAMESPACE, "demo")
.await .await
.unwrap(), .unwrap()
.and_then(|x| x.parse::<i64>().ok()),
None None
); );
assert_eq!( assert_eq!(
test_env redis_pool
.db .get(PROJECTS_SLUGS_NAMESPACE, &id)
.redis_pool
.get::<i64, _>(PROJECTS_SLUGS_NAMESPACE, id)
.await .await
.unwrap(), .unwrap()
.and_then(|x| x.parse::<i64>().ok()),
None None
); );

2
tests/search.rs
View File

@@ -20,7 +20,7 @@ mod common;
#[actix_rt::test] #[actix_rt::test]
async fn search_projects() { async fn search_projects() {
// Test setup and dummy data // Test setup and dummy data
let test_env = TestEnvironment::build(Some(8)).await; let test_env = TestEnvironment::build(Some(10)).await;
let api = &test_env.v3; let api = &test_env.v3;
let test_name = test_env.db.database_name.clone(); let test_name = test_env.db.database_name.clone();

19
tests/v2/project.rs
View File

@@ -221,22 +221,21 @@ async fn test_add_remove_project() {
assert_eq!(resp.status(), 204); assert_eq!(resp.status(), 204);
// Confirm that the project is gone from the cache // Confirm that the project is gone from the cache
let mut redis_conn = test_env.db.redis_pool.connect().await.unwrap();
assert_eq!( assert_eq!(
test_env redis_conn
.db .get(PROJECTS_SLUGS_NAMESPACE, "demo")
.redis_pool
.get::<i64, _>(PROJECTS_SLUGS_NAMESPACE, "demo")
.await .await
.unwrap(), .unwrap()
.map(|x| x.parse::<i64>().unwrap()),
None None
); );
assert_eq!( assert_eq!(
test_env redis_conn
.db .get(PROJECTS_SLUGS_NAMESPACE, &id)
.redis_pool
.get::<i64, _>(PROJECTS_SLUGS_NAMESPACE, id)
.await .await
.unwrap(), .unwrap()
.map(|x| x.parse::<i64>().unwrap()),
None None
); );

2
tests/v2/search.rs
View File

@@ -17,7 +17,7 @@ async fn search_projects() {
// It should drastically simplify this function // It should drastically simplify this function
// Test setup and dummy data // Test setup and dummy data
let test_env = TestEnvironment::build(Some(8)).await; let test_env = TestEnvironment::build(Some(10)).await;
let api = &test_env.v2; let api = &test_env.v2;
let test_name = test_env.db.database_name.clone(); let test_name = test_env.db.database_name.clone();

10
tests/version.rs
View File

@@ -33,10 +33,12 @@ async fn test_get_version() {
assert_eq!(&version.project_id.to_string(), alpha_project_id); assert_eq!(&version.project_id.to_string(), alpha_project_id);
assert_eq!(&version.id.to_string(), alpha_version_id); assert_eq!(&version.id.to_string(), alpha_version_id);
let cached_project = test_env let mut redis_conn = test_env.db.redis_pool.connect().await.unwrap();
.db let cached_project = redis_conn
.redis_pool .get(
.get::<String, _>(VERSIONS_NAMESPACE, parse_base62(alpha_version_id).unwrap()) VERSIONS_NAMESPACE,
&parse_base62(alpha_version_id).unwrap().to_string(),
)
.await .await
.unwrap() .unwrap()
.unwrap(); .unwrap();