From 9c5d817a8a36d9785f3b443e301375654416027d Mon Sep 17 00:00:00 2001 From: Jai Agrawal <18202329+Geometrically@users.noreply.github.com> Date: Fri, 7 Mar 2025 17:16:45 -0800 Subject: [PATCH 01/36] Add prom metrics for database, use redis max conn var (#3359) --- Cargo.lock | 8 ++++ apps/labrinth/Cargo.toml | 1 + apps/labrinth/src/database/mod.rs | 1 + .../src/database/postgres_database.rs | 28 ++++++++++++ apps/labrinth/src/database/redis.rs | 45 ++++++++++++++++++- apps/labrinth/src/main.rs | 8 ++++ 6 files changed, 90 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 6aff08232..ff38ac482 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4288,6 +4288,7 @@ dependencies = [ "maxminddb", "meilisearch-sdk", "murmur2", + "prometheus", "rand 0.8.5", "rand_chacha 0.3.1", "redis", @@ -6133,9 +6134,16 @@ dependencies = [ "memchr", "parking_lot 0.12.3", "procfs", + "protobuf", "thiserror 1.0.64", ] +[[package]] +name = "protobuf" +version = "2.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "106dd99e98437432fed6519dedecfade6a06a73bb7b2a1e019fdd2bee5778d94" + [[package]] name = "psm" version = "0.1.23" diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 856cd8f2a..ce023ba3a 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -17,6 +17,7 @@ actix-multipart = "0.6.1" actix-cors = "0.7.0" actix-ws = "0.3.0" actix-files = "0.6.5" +prometheus = "0.13.4" actix-web-prom = { version = "0.9.0", features = ["process"] } governor = "0.6.3" diff --git a/apps/labrinth/src/database/mod.rs b/apps/labrinth/src/database/mod.rs index 2bba7dca6..a1c51f2a5 100644 --- a/apps/labrinth/src/database/mod.rs +++ b/apps/labrinth/src/database/mod.rs @@ -6,3 +6,4 @@ pub use models::Project; pub use models::Version; pub use postgres_database::check_for_migrations; pub use postgres_database::connect; +pub use postgres_database::register_and_set_metrics; diff --git a/apps/labrinth/src/database/postgres_database.rs b/apps/labrinth/src/database/postgres_database.rs index 65601bde9..c90e37f90 100644 --- a/apps/labrinth/src/database/postgres_database.rs +++ b/apps/labrinth/src/database/postgres_database.rs @@ -1,4 +1,5 @@ use log::info; +use prometheus::{IntGauge, Registry}; use sqlx::migrate::MigrateDatabase; use sqlx::postgres::{PgPool, PgPoolOptions}; use sqlx::{Connection, PgConnection, Postgres}; @@ -45,3 +46,30 @@ pub async fn check_for_migrations() -> Result<(), sqlx::Error> { Ok(()) } + +pub async fn register_and_set_metrics( + pool: &PgPool, + registry: &Registry, +) -> Result<(), prometheus::Error> { + let pg_pool_size = + IntGauge::new("labrinth_pg_pool_size", "Size of Postgres pool")?; + let pg_pool_idle = IntGauge::new( + "labrinth_pg_pool_idle", + "Number of idle Postgres connections", + )?; + + registry.register(Box::new(pg_pool_size.clone()))?; + registry.register(Box::new(pg_pool_idle.clone()))?; + + let pool_ref = pool.clone(); + tokio::spawn(async move { + loop { + pg_pool_size.set(pool_ref.size() as i64); + pg_pool_idle.set(pool_ref.num_idle() as i64); + + tokio::time::sleep(Duration::from_secs(5)).await; + } + }); + + Ok(()) +} diff --git a/apps/labrinth/src/database/redis.rs b/apps/labrinth/src/database/redis.rs index fa77707ef..3fe92a53d 100644 --- a/apps/labrinth/src/database/redis.rs +++ b/apps/labrinth/src/database/redis.rs @@ -3,6 +3,7 @@ use ariadne::ids::base62_impl::{parse_base62, to_base62}; use chrono::{TimeZone, Utc}; use dashmap::DashMap; use deadpool_redis::{Config, Runtime}; +use prometheus::{IntGauge, Registry}; use redis::{cmd, Cmd, ExistenceCheck, SetExpiry, SetOptions}; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; @@ -38,7 +39,7 @@ impl RedisPool { .builder() .expect("Error building Redis pool") .max_size( - dotenvy::var("DATABASE_MAX_CONNECTIONS") + dotenvy::var("REDIS_MAX_CONNECTIONS") .ok() .and_then(|x| x.parse().ok()) .unwrap_or(10000), @@ -53,6 +54,48 @@ impl RedisPool { } } + pub async fn register_and_set_metrics( + &self, + registry: &Registry, + ) -> Result<(), prometheus::Error> { + let redis_max_size = IntGauge::new( + "labrinth_redis_pool_max_size", + "Maximum size of Redis pool", + )?; + let redis_size = IntGauge::new( + "labrinth_redis_pool_size", + "Current size of Redis pool", + )?; + let redis_available = IntGauge::new( + "labrinth_redis_pool_available", + "Available connections in Redis pool", + )?; + let redis_waiting = IntGauge::new( + "labrinth_redis_pool_waiting", + "Number of futures waiting for a Redis connection", + )?; + + registry.register(Box::new(redis_max_size.clone()))?; + registry.register(Box::new(redis_size.clone()))?; + registry.register(Box::new(redis_available.clone()))?; + registry.register(Box::new(redis_waiting.clone()))?; + + let redis_pool_ref = self.pool.clone(); + tokio::spawn(async move { + loop { + let status = redis_pool_ref.status(); + redis_max_size.set(status.max_size as i64); + redis_size.set(status.size as i64); + redis_available.set(status.available as i64); + redis_waiting.set(status.waiting as i64); + + tokio::time::sleep(Duration::from_secs(5)).await; + } + }); + + Ok(()) + } + pub async fn connect(&self) -> Result { Ok(RedisConnection { connection: self.pool.get().await?, diff --git a/apps/labrinth/src/main.rs b/apps/labrinth/src/main.rs index 9310a0886..d9b0b7b12 100644 --- a/apps/labrinth/src/main.rs +++ b/apps/labrinth/src/main.rs @@ -99,6 +99,14 @@ async fn main() -> std::io::Result<()> { .build() .expect("Failed to create prometheus metrics middleware"); + database::register_and_set_metrics(&pool, &prometheus.registry) + .await + .expect("Failed to register database metrics"); + redis_pool + .register_and_set_metrics(&prometheus.registry) + .await + .expect("Failed to register redis metrics"); + let search_config = search::SearchConfig::new(None); let labrinth_config = labrinth::app_setup( From 5c8ed9a8ca1b39f49f25bb8eb0d6ed701cc733d1 Mon Sep 17 00:00:00 2001 From: Jai Agrawal <18202329+Geometrically@users.noreply.github.com> Date: Sun, 9 Mar 2025 13:01:24 -0700 Subject: [PATCH 02/36] Tracing support (#3372) * Tracing support * Add console subscriber * Add console subscriber --- .cargo/config.toml | 3 + Cargo.lock | 312 ++++++++++++++++-- apps/labrinth/Cargo.toml | 7 +- .../src/database/postgres_database.rs | 2 +- apps/labrinth/src/lib.rs | 2 +- apps/labrinth/src/main.rs | 11 +- apps/labrinth/src/queue/maxmind.rs | 2 +- apps/labrinth/src/routes/internal/admin.rs | 2 +- apps/labrinth/src/routes/internal/billing.rs | 2 +- apps/labrinth/src/routes/internal/statuses.rs | 7 +- .../src/routes/v3/version_creation.rs | 2 +- apps/labrinth/src/scheduler.rs | 2 +- .../src/search/indexing/local_import.rs | 2 +- apps/labrinth/src/search/indexing/mod.rs | 2 +- 14 files changed, 306 insertions(+), 52 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index ca333857a..a52d5a685 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,6 @@ # Windows has stack overflows when calling from Tauri, so we increase compiler size [target.'cfg(windows)'] rustflags = ["-C", "link-args=/STACK:16777220"] + +[build] +rustflags = ["--cfg", "tokio_unstable"] \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index ff38ac482..ed6c485b2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -631,6 +631,28 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "async-stream" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.90", +] + [[package]] name = "async-stripe" version = "0.39.1" @@ -793,6 +815,53 @@ dependencies = [ "thiserror 1.0.64", ] +[[package]] +name = "axum" +version = "0.7.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f" +dependencies = [ + "async-trait", + "axum-core", + "bytes 1.7.2", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "itoa 1.0.11", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper 1.0.1", + "tower 0.5.2", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199" +dependencies = [ + "async-trait", + "bytes 1.7.2", + "futures-util", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 1.0.1", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -1450,6 +1519,45 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "console-api" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8030735ecb0d128428b64cd379809817e620a40e5001c54465b99ec5feec2857" +dependencies = [ + "futures-core", + "prost", + "prost-types", + "tonic", + "tracing-core", +] + +[[package]] +name = "console-subscriber" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6539aa9c6a4cd31f4b1c040f860a1eac9aa80e7df6b05d506a6e7179936d6a01" +dependencies = [ + "console-api", + "crossbeam-channel", + "crossbeam-utils 0.8.20", + "futures-task", + "hdrhistogram", + "humantime", + "hyper-util", + "prost", + "prost-types", + "serde", + "serde_json", + "thread_local", + "tokio 1.42.0", + "tokio-stream", + "tonic", + "tracing", + "tracing-core", + "tracing-subscriber", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -2441,19 +2549,6 @@ dependencies = [ "syn 2.0.90", ] -[[package]] -name = "env_logger" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd405aab171cb85d6735e5c8d9db038c17d3ca007a4d2c25f337935c3d90580" -dependencies = [ - "humantime", - "is-terminal", - "log", - "regex", - "termcolor", -] - [[package]] name = "equivalent" version = "1.0.1" @@ -3301,6 +3396,19 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "hdrhistogram" +version = "7.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d" +dependencies = [ + "base64 0.21.7", + "byteorder", + "flate2", + "nom", + "num-traits", +] + [[package]] name = "heck" version = "0.3.3" @@ -3540,6 +3648,7 @@ dependencies = [ "http 1.1.0", "http-body 1.0.1", "httparse", + "httpdate", "itoa 1.0.11", "pin-project-lite", "smallvec 1.13.2", @@ -3582,6 +3691,19 @@ dependencies = [ "webpki-roots 0.26.6", ] +[[package]] +name = "hyper-timeout" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0" +dependencies = [ + "hyper 1.4.1", + "hyper-util", + "pin-project-lite", + "tokio 1.42.0", + "tower-service", +] + [[package]] name = "hyper-tls" version = "0.5.0" @@ -3613,9 +3735,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41296eb09f183ac68eec06e03cdbea2e759633d4067b2f6552fc2e009bcad08b" +checksum = "df2dcfbe0677734ab2f3ffa7fa7bfd4706bfdc1ef393f2ee30184aed67e631b4" dependencies = [ "bytes 1.7.2", "futures-channel", @@ -4263,12 +4385,12 @@ dependencies = [ "chrono", "clickhouse", "color-thief", + "console-subscriber", "dashmap 5.5.3", "deadpool-redis", "derive-new", "dotenvy", "either", - "env_logger", "flate2", "futures 0.3.30", "futures-timer", @@ -4284,7 +4406,6 @@ dependencies = [ "json-patch", "lazy_static", "lettre", - "log", "maxminddb", "meilisearch-sdk", "murmur2", @@ -4314,6 +4435,9 @@ dependencies = [ "tokio 1.42.0", "tokio-stream", "totp-rs", + "tracing", + "tracing-actix-web", + "tracing-subscriber", "url", "urlencoding", "uuid 1.12.0", @@ -4604,6 +4728,12 @@ version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5" +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "maxminddb" version = "0.24.0" @@ -4862,6 +4992,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb585ade2549a017db2e35978b77c319214fa4b37cede841e27954dd6e8f3ca8" +[[package]] +name = "mutually_exclusive_features" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e94e1e6445d314f972ff7395df2de295fe51b71821694f0b0e1e79c4f12c8577" + [[package]] name = "native-dialog" version = "0.7.0" @@ -6138,6 +6274,38 @@ dependencies = [ "thiserror 1.0.64", ] +[[package]] +name = "prost" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" +dependencies = [ + "bytes 1.7.2", + "prost-derive", +] + +[[package]] +name = "prost-derive" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" +dependencies = [ + "anyhow", + "itertools 0.13.0", + "proc-macro2", + "quote", + "syn 2.0.90", +] + +[[package]] +name = "prost-types" +version = "0.13.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16" +dependencies = [ + "prost", +] + [[package]] name = "protobuf" version = "2.28.0" @@ -8911,15 +9079,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "termcolor" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" -dependencies = [ - "winapi-util", -] - [[package]] name = "theseus" version = "0.9.3" @@ -9502,6 +9661,36 @@ dependencies = [ "winnow 0.6.20", ] +[[package]] +name = "tonic" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64 0.22.1", + "bytes 1.7.2", + "h2 0.4.6", + "http 1.1.0", + "http-body 1.0.1", + "http-body-util", + "hyper 1.4.1", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "socket2", + "tokio 1.42.0", + "tokio-stream", + "tower 0.4.13", + "tower-layer", + "tower-service", + "tracing", +] + [[package]] name = "totp-rs" version = "5.6.0" @@ -9516,6 +9705,46 @@ dependencies = [ "sha2 0.10.8", ] +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand 0.8.5", + "slab", + "tokio 1.42.0", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" +dependencies = [ + "futures-core", + "futures-util", + "pin-project-lite", + "sync_wrapper 1.0.1", + "tower-layer", + "tower-service", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + [[package]] name = "tower-service" version = "0.3.3" @@ -9524,9 +9753,9 @@ checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" [[package]] name = "tracing" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0" dependencies = [ "log", "pin-project-lite", @@ -9535,10 +9764,23 @@ dependencies = [ ] [[package]] -name = "tracing-attributes" -version = "0.1.27" +name = "tracing-actix-web" +version = "0.7.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +checksum = "332bbdf3bd208d1fe6446f8ffb4e8c2ae66e25da0fb38e0b69545e640ecee6a6" +dependencies = [ + "actix-web", + "mutually_exclusive_features", + "pin-project", + "tracing", + "uuid 1.12.0", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", @@ -9547,9 +9789,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.32" +version = "0.1.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" +checksum = "e672c95779cf947c5311f83787af4fa8fffd12fb27e4993211a84bdfd9610f9c" dependencies = [ "once_cell", "valuable", @@ -9590,9 +9832,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" +checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "chrono", "matchers", diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index ce023ba3a..928af6aa3 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -21,6 +21,11 @@ prometheus = "0.13.4" actix-web-prom = { version = "0.9.0", features = ["process"] } governor = "0.6.3" +tracing = "0.1.41" +tracing-subscriber = "0.3.19" +tracing-actix-web = "0.7.16" +console-subscriber = "0.4.1" + tokio = { version = "1.35.1", features = ["sync"] } tokio-stream = "0.1.14" @@ -74,8 +79,6 @@ censor = "0.3.0" spdx = { version = "0.10.3", features = ["text"] } dotenvy = "0.15.7" -log = "0.4.20" -env_logger = "0.10.1" thiserror = "1.0.56" either = "1.13" diff --git a/apps/labrinth/src/database/postgres_database.rs b/apps/labrinth/src/database/postgres_database.rs index c90e37f90..3ace44dfe 100644 --- a/apps/labrinth/src/database/postgres_database.rs +++ b/apps/labrinth/src/database/postgres_database.rs @@ -1,9 +1,9 @@ -use log::info; use prometheus::{IntGauge, Registry}; use sqlx::migrate::MigrateDatabase; use sqlx::postgres::{PgPool, PgPoolOptions}; use sqlx::{Connection, PgConnection, Postgres}; use std::time::Duration; +use tracing::info; pub async fn connect() -> Result { info!("Initializing database connection"); diff --git a/apps/labrinth/src/lib.rs b/apps/labrinth/src/lib.rs index 5d946fbdc..f520bd157 100644 --- a/apps/labrinth/src/lib.rs +++ b/apps/labrinth/src/lib.rs @@ -4,12 +4,12 @@ use std::time::Duration; use actix_web::web; use database::redis::RedisPool; -use log::{info, warn}; use queue::{ analytics::AnalyticsQueue, payouts::PayoutsQueue, session::AuthQueue, socket::ActiveSockets, }; use sqlx::Postgres; +use tracing::{info, warn}; extern crate clickhouse as clickhouse_crate; use clickhouse_crate::Client; diff --git a/apps/labrinth/src/main.rs b/apps/labrinth/src/main.rs index d9b0b7b12..2878ef125 100644 --- a/apps/labrinth/src/main.rs +++ b/apps/labrinth/src/main.rs @@ -1,13 +1,16 @@ use actix_web::{App, HttpServer}; use actix_web_prom::PrometheusMetricsBuilder; -use env_logger::Env; use labrinth::database::redis::RedisPool; use labrinth::file_hosting::S3Host; use labrinth::search; use labrinth::util::ratelimit::RateLimit; use labrinth::{check_env_vars, clickhouse, database, file_hosting, queue}; -use log::{error, info}; use std::sync::Arc; +use tracing::{error, info}; +use tracing_actix_web::TracingLogger; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::util::SubscriberInitExt; +use tracing_subscriber::{fmt, EnvFilter}; #[cfg(feature = "jemalloc")] #[global_allocator] @@ -21,8 +24,7 @@ pub struct Pepper { #[actix_rt::main] async fn main() -> std::io::Result<()> { dotenvy::dotenv().ok(); - env_logger::Builder::from_env(Env::default().default_filter_or("info")) - .init(); + console_subscriber::init(); if check_env_vars() { error!("Some environment variables are missing!"); @@ -123,6 +125,7 @@ async fn main() -> std::io::Result<()> { // Init App HttpServer::new(move || { App::new() + .wrap(TracingLogger::default()) .wrap(prometheus.clone()) .wrap(RateLimit(Arc::clone(&labrinth_config.rate_limiter))) .wrap(actix_web::middleware::Compress::default()) diff --git a/apps/labrinth/src/queue/maxmind.rs b/apps/labrinth/src/queue/maxmind.rs index e551a8ca4..30a83d1ec 100644 --- a/apps/labrinth/src/queue/maxmind.rs +++ b/apps/labrinth/src/queue/maxmind.rs @@ -1,10 +1,10 @@ use flate2::read::GzDecoder; -use log::warn; use maxminddb::geoip2::Country; use std::io::{Cursor, Read}; use std::net::Ipv6Addr; use tar::Archive; use tokio::sync::RwLock; +use tracing::warn; pub struct MaxMindIndexer { pub reader: RwLock>>>, diff --git a/apps/labrinth/src/routes/internal/admin.rs b/apps/labrinth/src/routes/internal/admin.rs index cf7f8a10c..a6db8dcdc 100644 --- a/apps/labrinth/src/routes/internal/admin.rs +++ b/apps/labrinth/src/routes/internal/admin.rs @@ -15,12 +15,12 @@ use crate::search::SearchConfig; use crate::util::date::get_current_tenths_of_ms; use crate::util::guards::admin_key_guard; use actix_web::{get, patch, post, web, HttpRequest, HttpResponse}; -use log::info; use serde::Deserialize; use sqlx::PgPool; use std::collections::HashMap; use std::net::Ipv4Addr; use std::sync::Arc; +use tracing::info; pub fn config(cfg: &mut web::ServiceConfig) { cfg.service( diff --git a/apps/labrinth/src/routes/internal/billing.rs b/apps/labrinth/src/routes/internal/billing.rs index 28c9954c1..8488af0da 100644 --- a/apps/labrinth/src/routes/internal/billing.rs +++ b/apps/labrinth/src/routes/internal/billing.rs @@ -17,7 +17,6 @@ use crate::routes::ApiError; use actix_web::{delete, get, patch, post, web, HttpRequest, HttpResponse}; use ariadne::ids::base62_impl::{parse_base62, to_base62}; use chrono::Utc; -use log::{info, warn}; use rust_decimal::prelude::ToPrimitive; use rust_decimal::Decimal; use serde::Serialize; @@ -34,6 +33,7 @@ use stripe::{ PaymentIntentSetupFutureUsage, PaymentMethodId, SetupIntent, UpdateCustomer, Webhook, }; +use tracing::{info, warn}; pub fn config(cfg: &mut web::ServiceConfig) { cfg.service( diff --git a/apps/labrinth/src/routes/internal/statuses.rs b/apps/labrinth/src/routes/internal/statuses.rs index 5054ad678..0c5c08ad6 100644 --- a/apps/labrinth/src/routes/internal/statuses.rs +++ b/apps/labrinth/src/routes/internal/statuses.rs @@ -114,7 +114,7 @@ pub async fn ws_init( .insert(socket_id); #[cfg(debug_assertions)] - log::info!("Connection {socket_id} opened by {}", user.id); + tracing::info!("Connection {socket_id} opened by {}", user.id); broadcast_friends( user.id, @@ -161,7 +161,10 @@ pub async fn ws_init( #[cfg(debug_assertions)] if !message.is_binary() { - log::info!("Received message from {socket_id}: {:?}", message); + tracing::info!( + "Received message from {socket_id}: {:?}", + message + ); } match message { diff --git a/apps/labrinth/src/routes/v3/version_creation.rs b/apps/labrinth/src/routes/v3/version_creation.rs index cd1c52308..0b91cd0e4 100644 --- a/apps/labrinth/src/routes/v3/version_creation.rs +++ b/apps/labrinth/src/routes/v3/version_creation.rs @@ -31,11 +31,11 @@ use actix_web::{web, HttpRequest, HttpResponse}; use chrono::Utc; use futures::stream::StreamExt; use itertools::Itertools; -use log::error; use serde::{Deserialize, Serialize}; use sqlx::postgres::PgPool; use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use tracing::error; use validator::Validate; fn default_requested_status() -> VersionStatus { diff --git a/apps/labrinth/src/scheduler.rs b/apps/labrinth/src/scheduler.rs index 7bc5e5195..f94540392 100644 --- a/apps/labrinth/src/scheduler.rs +++ b/apps/labrinth/src/scheduler.rs @@ -36,7 +36,7 @@ impl Drop for Scheduler { } } -use log::{info, warn}; +use tracing::{info, warn}; pub fn schedule_versions( scheduler: &mut Scheduler, diff --git a/apps/labrinth/src/search/indexing/local_import.rs b/apps/labrinth/src/search/indexing/local_import.rs index 4306888f3..c9b2eeaea 100644 --- a/apps/labrinth/src/search/indexing/local_import.rs +++ b/apps/labrinth/src/search/indexing/local_import.rs @@ -2,8 +2,8 @@ use chrono::{DateTime, Utc}; use dashmap::DashMap; use futures::TryStreamExt; use itertools::Itertools; -use log::info; use std::collections::HashMap; +use tracing::info; use super::IndexingError; use crate::database::models::loader_fields::{ diff --git a/apps/labrinth/src/search/indexing/mod.rs b/apps/labrinth/src/search/indexing/mod.rs index 28de1c12a..3f4dcdee9 100644 --- a/apps/labrinth/src/search/indexing/mod.rs +++ b/apps/labrinth/src/search/indexing/mod.rs @@ -5,12 +5,12 @@ use crate::database::redis::RedisPool; use crate::search::{SearchConfig, UploadSearchProject}; use ariadne::ids::base62_impl::to_base62; use local_import::index_local; -use log::info; use meilisearch_sdk::client::{Client, SwapIndexes}; use meilisearch_sdk::indexes::Index; use meilisearch_sdk::settings::{PaginationSetting, Settings}; use sqlx::postgres::PgPool; use thiserror::Error; +use tracing::info; #[derive(Error, Debug)] pub enum IndexingError { #[error("Error while connecting to the MeiliSearch database")] From d3427375b049156b9f31e4642b689022df25ecfc Mon Sep 17 00:00:00 2001 From: Jai A Date: Sun, 9 Mar 2025 13:21:15 -0700 Subject: [PATCH 03/36] fix lint --- apps/labrinth/src/main.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/apps/labrinth/src/main.rs b/apps/labrinth/src/main.rs index 2878ef125..fd3739c3d 100644 --- a/apps/labrinth/src/main.rs +++ b/apps/labrinth/src/main.rs @@ -8,9 +8,6 @@ use labrinth::{check_env_vars, clickhouse, database, file_hosting, queue}; use std::sync::Arc; use tracing::{error, info}; use tracing_actix_web::TracingLogger; -use tracing_subscriber::layer::SubscriberExt; -use tracing_subscriber::util::SubscriberInitExt; -use tracing_subscriber::{fmt, EnvFilter}; #[cfg(feature = "jemalloc")] #[global_allocator] From 1009695a150fc501ea86791da836f89b5931bb21 Mon Sep 17 00:00:00 2001 From: Jai Agrawal <18202329+Geometrically@users.noreply.github.com> Date: Sun, 9 Mar 2025 14:41:19 -0700 Subject: [PATCH 04/36] Support jemalloc profiling and heap dumps (#3373) --- Cargo.lock | 295 ++++++++++++++++++++++++-- apps/labrinth/Cargo.toml | 7 +- apps/labrinth/src/lib.rs | 1 + apps/labrinth/src/main.rs | 11 +- apps/labrinth/src/routes/debug/mod.rs | 87 ++++++++ apps/labrinth/src/routes/mod.rs | 2 + 6 files changed, 382 insertions(+), 21 deletions(-) create mode 100644 apps/labrinth/src/routes/debug/mod.rs diff --git a/Cargo.lock b/Cargo.lock index ed6c485b2..55f13a32d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -393,6 +393,55 @@ dependencies = [ "libc", ] +[[package]] +name = "anstream" +version = "0.6.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" + +[[package]] +name = "anstyle-parse" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +dependencies = [ + "windows-sys 0.59.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2109dbce0e72be3ec00bed26e6a7479ca384ad226efdd66db8fa2e3a38c83125" +dependencies = [ + "anstyle", + "windows-sys 0.59.0", +] + [[package]] name = "anyhow" version = "1.0.89" @@ -1354,6 +1403,46 @@ dependencies = [ "inout", ] +[[package]] +name = "clap" +version = "4.5.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "027bb0d98429ae334a8698531da7077bdf906419543a35a55c2cb1b66437d767" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5589e0cba072e0f3d23791efac0fd8627b49c829c196a492e88168e6a669d863" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim 0.11.1", +] + +[[package]] +name = "clap_derive" +version = "4.5.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4ced95c6f4a675af3da73304b9ac4ed991640c36374e4b46795c49e17cf1ed" +dependencies = [ + "heck 0.5.0", + "proc-macro2", + "quote", + "syn 2.0.90", +] + +[[package]] +name = "clap_lex" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" + [[package]] name = "clickhouse" version = "0.11.6" @@ -1483,6 +1572,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "colorchoice" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" + [[package]] name = "combine" version = "4.6.7" @@ -2549,6 +2644,25 @@ dependencies = [ "syn 2.0.90", ] +[[package]] +name = "env_filter" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +dependencies = [ + "log", +] + +[[package]] +name = "env_logger" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dcaee3d8e3cfc3fd92428d477bc97fc29ec8716d180c0d74c643bb26166660e0" +dependencies = [ + "env_filter", + "log", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -4032,6 +4146,28 @@ dependencies = [ "cfb", ] +[[package]] +name = "inferno" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "692eda1cc790750b9f5a5e3921ef9c117fd5498b97cfacbc910693e5b29002dc" +dependencies = [ + "ahash 0.8.11", + "clap", + "crossbeam-channel", + "crossbeam-utils 0.8.20", + "dashmap 6.1.0", + "env_logger", + "indexmap 2.5.0", + "itoa 1.0.11", + "log", + "num-format", + "once_cell", + "quick-xml 0.37.2", + "rgb", + "str_stack", +] + [[package]] name = "inotify" version = "0.9.6" @@ -4124,6 +4260,12 @@ dependencies = [ "once_cell", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" + [[package]] name = "iso8601" version = "0.6.1" @@ -4205,23 +4347,20 @@ dependencies = [ ] [[package]] -name = "jemalloc-sys" -version = "0.5.4+5.3.0-patched" +name = "jemalloc_pprof" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac6c1946e1cea1788cbfde01c993b52a10e2da07f4bac608228d1bed20bfebf2" +checksum = "5622af6d21ff86ed7797ef98e11b8f302da25ec69a7db9f6cde8e2e1c8df9992" dependencies = [ - "cc", - "libc", -] - -[[package]] -name = "jemallocator" -version = "0.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0de374a9f8e63150e6f5e8a60cc14c668226d7a347d8aee1a45766e3c4dd3bc" -dependencies = [ - "jemalloc-sys", + "anyhow", "libc", + "mappings", + "once_cell", + "pprof_util", + "tempfile", + "tikv-jemalloc-ctl", + "tokio 1.42.0", + "tracing", ] [[package]] @@ -4402,7 +4541,7 @@ dependencies = [ "hyper-tls 0.5.0", "image 0.24.9", "itertools 0.12.1", - "jemallocator", + "jemalloc_pprof", "json-patch", "lazy_static", "lettre", @@ -4432,6 +4571,8 @@ dependencies = [ "sqlx", "tar", "thiserror 1.0.64", + "tikv-jemalloc-ctl", + "tikv-jemallocator", "tokio 1.42.0", "tokio-stream", "totp-rs", @@ -4699,6 +4840,19 @@ dependencies = [ "libc", ] +[[package]] +name = "mappings" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e434981a332777c2b3062652d16a55f8e74fa78e6b1882633f0d77399c84fc2a" +dependencies = [ + "anyhow", + "libc", + "once_cell", + "pprof_util", + "tracing", +] + [[package]] name = "markup5ever" version = "0.11.0" @@ -5183,6 +5337,20 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + [[package]] name = "num-bigint" version = "0.4.6" @@ -5210,12 +5378,31 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-format" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a652d9771a63711fd3c3deb670acfbe5c30a4072e664d7a3bf5a9e1056ac72c3" +dependencies = [ + "arrayvec", + "itoa 1.0.11", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -5236,6 +5423,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -6129,6 +6327,21 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +[[package]] +name = "pprof_util" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa015c78eed2130951e22c58d2095849391e73817ab2e74f71b0b9f63dd8416" +dependencies = [ + "anyhow", + "backtrace", + "flate2", + "inferno", + "num", + "paste", + "prost", +] + [[package]] name = "ppv-lite86" version = "0.2.20" @@ -6408,6 +6621,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "quick-xml" +version = "0.37.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "165859e9e55f79d67b96c5d96f4e88b6f2695a1972849c15a6a3f5c59fc2c003" +dependencies = [ + "memchr", +] + [[package]] name = "quinn" version = "0.11.5" @@ -8319,6 +8541,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "str_stack" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091b6114800a5f2141aee1d1b9d6ca3592ac062dc5decb3764ec5895a47b4eb" + [[package]] name = "strfmt" version = "0.2.4" @@ -9258,6 +9486,37 @@ dependencies = [ "weezl", ] +[[package]] +name = "tikv-jemalloc-ctl" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f21f216790c8df74ce3ab25b534e0718da5a1916719771d3fec23315c99e468b" +dependencies = [ + "libc", + "paste", + "tikv-jemalloc-sys", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.6.0+5.3.0-1-ge13ca993e8ccb9ba9847cc330696e02839f328f7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3c60906412afa9c2b5b5a48ca6a5abe5736aec9eb48ad05037a677e52e4e2d" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4cec5ff18518d81584f477e9bfdf957f5bb0979b0bac3af4ca30b5b3ae2d2865" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "time" version = "0.3.36" @@ -10104,6 +10363,12 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "0.8.2" diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 928af6aa3..d7adce63a 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -125,7 +125,9 @@ lettre = "0.11.3" derive-new = "0.6.0" rust_iso3166 = "0.1.11" -jemallocator = { version = "0.5.4", optional = true } +tikv-jemallocator = { version = "0.6.0", features = ["profiling", "unprefixed_malloc_on_supported_platforms"] } +tikv-jemalloc-ctl = { version = "0.6.0", features = ["stats"] } +jemalloc_pprof = { version = "0.7.0", features = ["flamegraph"] } async-stripe = { version = "0.39.1", features = ["runtime-tokio-hyper-rustls"] } rusty-money = "0.4.1" @@ -135,6 +137,3 @@ ariadne = { path = "../../packages/ariadne" } [dev-dependencies] actix-http = "3.4.0" - -[features] -jemalloc = ["jemallocator"] diff --git a/apps/labrinth/src/lib.rs b/apps/labrinth/src/lib.rs index f520bd157..8df3ddd49 100644 --- a/apps/labrinth/src/lib.rs +++ b/apps/labrinth/src/lib.rs @@ -352,6 +352,7 @@ pub fn app_config( .app_data(labrinth_config.active_sockets.clone()) .app_data(labrinth_config.automated_moderation_queue.clone()) .app_data(web::Data::new(labrinth_config.stripe_client.clone())) + .configure(routes::debug::config) .configure(routes::v2::config) .configure(routes::v3::config) .configure(routes::internal::config) diff --git a/apps/labrinth/src/main.rs b/apps/labrinth/src/main.rs index fd3739c3d..df39ea10b 100644 --- a/apps/labrinth/src/main.rs +++ b/apps/labrinth/src/main.rs @@ -9,9 +9,14 @@ use std::sync::Arc; use tracing::{error, info}; use tracing_actix_web::TracingLogger; -#[cfg(feature = "jemalloc")] +#[cfg(not(target_env = "msvc"))] #[global_allocator] -static ALLOC: jemallocator::Jemalloc = jemallocator::Jemalloc; +static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +#[allow(non_upper_case_globals)] +#[export_name = "malloc_conf"] +pub static malloc_conf: &[u8] = + b"prof:true,prof_active:true,lg_prof_sample:19\0"; #[derive(Clone)] pub struct Pepper { @@ -105,6 +110,8 @@ async fn main() -> std::io::Result<()> { .register_and_set_metrics(&prometheus.registry) .await .expect("Failed to register redis metrics"); + labrinth::routes::debug::jemalloc_mmeory_stats(&prometheus.registry) + .expect("Failed to register jemalloc metrics"); let search_config = search::SearchConfig::new(None); diff --git a/apps/labrinth/src/routes/debug/mod.rs b/apps/labrinth/src/routes/debug/mod.rs new file mode 100644 index 000000000..163909f05 --- /dev/null +++ b/apps/labrinth/src/routes/debug/mod.rs @@ -0,0 +1,87 @@ +use crate::routes::ApiError; +use crate::util::cors::default_cors; +use crate::util::guards::admin_key_guard; +use actix_web::{get, HttpResponse}; +use prometheus::{IntGauge, Registry}; +use std::time::Duration; + +pub fn config(cfg: &mut actix_web::web::ServiceConfig) { + cfg.service( + actix_web::web::scope("/debug") + .wrap(default_cors()) + .service(heap) + .service(flame_graph), + ); +} + +#[get("pprof/heap", guard = "admin_key_guard")] +pub async fn heap() -> Result { + let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; + require_profiling_activated(&prof_ctl)?; + let pprof = prof_ctl + .dump_pprof() + .map_err(|err| ApiError::InvalidInput(err.to_string()))?; + + Ok(HttpResponse::Ok() + .content_type("application/octet-stream") + .body(pprof)) +} + +#[get("pprof/heap/flamegraph", guard = "admin_key_guard")] +pub async fn flame_graph() -> Result { + let mut prof_ctl = jemalloc_pprof::PROF_CTL.as_ref().unwrap().lock().await; + require_profiling_activated(&prof_ctl)?; + let svg = prof_ctl + .dump_flamegraph() + .map_err(|err| ApiError::InvalidInput(err.to_string()))?; + + Ok(HttpResponse::Ok().content_type("image/svg+xml").body(svg)) +} + +fn require_profiling_activated( + prof_ctl: &jemalloc_pprof::JemallocProfCtl, +) -> Result<(), ApiError> { + if prof_ctl.activated() { + Ok(()) + } else { + Err(ApiError::InvalidInput( + "Profiling is not activated".to_string(), + )) + } +} + +pub fn jemalloc_mmeory_stats( + registry: &Registry, +) -> Result<(), prometheus::Error> { + let allocated_mem = IntGauge::new( + "labrinth_memory_allocated", + "Labrinth allocated memory", + )?; + let resident_mem = + IntGauge::new("labrinth_resident_memory", "Labrinth resident memory")?; + + registry.register(Box::new(allocated_mem.clone()))?; + registry.register(Box::new(resident_mem.clone()))?; + + tokio::spawn(async move { + let e = tikv_jemalloc_ctl::epoch::mib().unwrap(); + let allocated = tikv_jemalloc_ctl::stats::allocated::mib().unwrap(); + let resident = tikv_jemalloc_ctl::stats::resident::mib().unwrap(); + + loop { + e.advance().unwrap(); + + if let Ok(allocated) = allocated.read() { + allocated_mem.set(allocated as i64); + } + + if let Ok(resident) = resident.read() { + resident_mem.set(resident as i64); + } + + tokio::time::sleep(Duration::from_secs(5)).await; + } + }); + + Ok(()) +} diff --git a/apps/labrinth/src/routes/mod.rs b/apps/labrinth/src/routes/mod.rs index c2a990fab..b5afc015f 100644 --- a/apps/labrinth/src/routes/mod.rs +++ b/apps/labrinth/src/routes/mod.rs @@ -12,6 +12,8 @@ pub mod internal; pub mod v2; pub mod v3; +pub mod debug; + pub mod v2_reroute; mod analytics; From 4dd33a2f9e4150d447dafd9de8af78d2369e976f Mon Sep 17 00:00:00 2001 From: Jai Agrawal <18202329+Geometrically@users.noreply.github.com> Date: Sun, 9 Mar 2025 17:15:07 -0700 Subject: [PATCH 05/36] Fix issues being excluded by delphi (#3374) --- apps/labrinth/src/routes/internal/admin.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/apps/labrinth/src/routes/internal/admin.rs b/apps/labrinth/src/routes/internal/admin.rs index a6db8dcdc..70c8c88f6 100644 --- a/apps/labrinth/src/routes/internal/admin.rs +++ b/apps/labrinth/src/routes/internal/admin.rs @@ -247,6 +247,10 @@ pub async fn delphi_result_ingest( path )); } + + if trace.is_empty() { + thread_header.push_str(&format!("\n\n- issue {issue} found",)); + } } let mut transaction = pool.begin().await?; From d6c8af7ed5cdd20a51d8725cfb24126441902404 Mon Sep 17 00:00:00 2001 From: Josiah Glosson Date: Mon, 10 Mar 2025 16:45:36 -0500 Subject: [PATCH 06/36] Fix Labrinth not compiling on Windows due to jemalloc dependency (#3378) --- apps/labrinth/Cargo.toml | 9 +++++---- apps/labrinth/src/lib.rs | 8 +++++++- apps/labrinth/src/main.rs | 2 ++ apps/labrinth/src/routes/mod.rs | 1 + 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index d7adce63a..584b8b935 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -125,15 +125,16 @@ lettre = "0.11.3" derive-new = "0.6.0" rust_iso3166 = "0.1.11" -tikv-jemallocator = { version = "0.6.0", features = ["profiling", "unprefixed_malloc_on_supported_platforms"] } -tikv-jemalloc-ctl = { version = "0.6.0", features = ["stats"] } -jemalloc_pprof = { version = "0.7.0", features = ["flamegraph"] } - async-stripe = { version = "0.39.1", features = ["runtime-tokio-hyper-rustls"] } rusty-money = "0.4.1" json-patch = "*" ariadne = { path = "../../packages/ariadne" } +[target.'cfg(not(target_env = "msvc"))'.dependencies] +tikv-jemallocator = { version = "0.6.0", features = ["profiling", "unprefixed_malloc_on_supported_platforms"] } +tikv-jemalloc-ctl = { version = "0.6.0", features = ["stats"] } +jemalloc_pprof = { version = "0.7.0", features = ["flamegraph"] } + [dev-dependencies] actix-http = "3.4.0" diff --git a/apps/labrinth/src/lib.rs b/apps/labrinth/src/lib.rs index 8df3ddd49..4ad4510de 100644 --- a/apps/labrinth/src/lib.rs +++ b/apps/labrinth/src/lib.rs @@ -352,7 +352,13 @@ pub fn app_config( .app_data(labrinth_config.active_sockets.clone()) .app_data(labrinth_config.automated_moderation_queue.clone()) .app_data(web::Data::new(labrinth_config.stripe_client.clone())) - .configure(routes::debug::config) + .configure( + #[allow(unused_variables)] + |cfg| { + #[cfg(not(target_env = "msvc"))] + routes::debug::config(cfg) + }, + ) .configure(routes::v2::config) .configure(routes::v3::config) .configure(routes::internal::config) diff --git a/apps/labrinth/src/main.rs b/apps/labrinth/src/main.rs index df39ea10b..e6058eed6 100644 --- a/apps/labrinth/src/main.rs +++ b/apps/labrinth/src/main.rs @@ -110,6 +110,8 @@ async fn main() -> std::io::Result<()> { .register_and_set_metrics(&prometheus.registry) .await .expect("Failed to register redis metrics"); + + #[cfg(not(target_env = "msvc"))] labrinth::routes::debug::jemalloc_mmeory_stats(&prometheus.registry) .expect("Failed to register jemalloc metrics"); diff --git a/apps/labrinth/src/routes/mod.rs b/apps/labrinth/src/routes/mod.rs index b5afc015f..39337d5f3 100644 --- a/apps/labrinth/src/routes/mod.rs +++ b/apps/labrinth/src/routes/mod.rs @@ -12,6 +12,7 @@ pub mod internal; pub mod v2; pub mod v3; +#[cfg(not(target_env = "msvc"))] pub mod debug; pub mod v2_reroute; From 366f528853dc32701e9670fd8d9c51fa3d136441 Mon Sep 17 00:00:00 2001 From: felix <60808107+ItsFelix5@users.noreply.github.com> Date: Tue, 11 Mar 2025 20:46:31 +0100 Subject: [PATCH 07/36] Fix but better (#3376) Signed-off-by: felix <60808107+ItsFelix5@users.noreply.github.com> --- packages/ui/src/components/base/Avatar.vue | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ui/src/components/base/Avatar.vue b/packages/ui/src/components/base/Avatar.vue index 091e2d810..00887e1fa 100644 --- a/packages/ui/src/components/base/Avatar.vue +++ b/packages/ui/src/components/base/Avatar.vue @@ -8,7 +8,7 @@ circle: circle, 'no-shadow': noShadow, raised: raised, - pixelated: raised, + pixelated: pixelated, }" :src="src" :alt="alt" @@ -96,7 +96,7 @@ const LEGACY_PRESETS = { const cssSize = computed(() => LEGACY_PRESETS[props.size] ?? props.size) function updatePixelated() { - if (img.value && img.value.naturalWidth && img.value.naturalWidth <= 96) { + if (img.value && img.value.naturalWidth && img.value.naturalWidth < 32) { pixelated.value = true } else { pixelated.value = false From 1ea196051f26612713ca394560d04bf6612d3070 Mon Sep 17 00:00:00 2001 From: Emma Alexia Date: Tue, 11 Mar 2025 20:00:30 -0400 Subject: [PATCH 08/36] Update CCPA notice with updated information for Servers (#3384) --- apps/frontend/src/pages/legal/ccpa.vue | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/frontend/src/pages/legal/ccpa.vue b/apps/frontend/src/pages/legal/ccpa.vue index d1487ec3d..885f67d1c 100644 --- a/apps/frontend/src/pages/legal/ccpa.vue +++ b/apps/frontend/src/pages/legal/ccpa.vue @@ -2,7 +2,7 @@

Privacy Notice for California Residents

Effective Date: August 5th, 2023

-

Last reviewed on: August 5th, 2023

+

Last reviewed on: March 11th, 2025

This Privacy Notice for California Residents supplements the information contained in the Privacy Policy of Rinth, Inc. (the "Company," "we," @@ -49,7 +49,7 @@ information, medical information, or health insurance information.

Some personal information included in this category may overlap with other categories. - NO + YES C. Protected classification characteristics. @@ -68,7 +68,7 @@ Records of personal property, products or services purchased, obtained, or considered, or other purchasing or consuming histories or tendencies. - NO + YES E. Biometric information. From 887e437d352a5c79e9198752f50d24519ca76299 Mon Sep 17 00:00:00 2001 From: Jai Agrawal <18202329+Geometrically@users.noreply.github.com> Date: Tue, 11 Mar 2025 23:27:49 -0700 Subject: [PATCH 09/36] Move archon to env var (#3386) --- apps/labrinth/.env | 4 +- apps/labrinth/src/lib.rs | 2 + apps/labrinth/src/routes/internal/billing.rs | 40 ++++++++++++-------- 3 files changed, 30 insertions(+), 16 deletions(-) diff --git a/apps/labrinth/.env b/apps/labrinth/.env index 3a73d5053..1a3359185 100644 --- a/apps/labrinth/.env +++ b/apps/labrinth/.env @@ -116,4 +116,6 @@ BREX_API_URL=https://platform.brexapis.com/v2/ BREX_API_KEY=none DELPHI_URL=none -DELPHI_SLACK_WEBHOOK=none \ No newline at end of file +DELPHI_SLACK_WEBHOOK=none + +ARCHON_URL=none \ No newline at end of file diff --git a/apps/labrinth/src/lib.rs b/apps/labrinth/src/lib.rs index 4ad4510de..bfaf95d81 100644 --- a/apps/labrinth/src/lib.rs +++ b/apps/labrinth/src/lib.rs @@ -502,5 +502,7 @@ pub fn check_env_vars() -> bool { failed |= check_var::("DELPHI_URL"); + failed |= check_var::("ARCHON_URL"); + failed } diff --git a/apps/labrinth/src/routes/internal/billing.rs b/apps/labrinth/src/routes/internal/billing.rs index 8488af0da..036c3c3db 100644 --- a/apps/labrinth/src/routes/internal/billing.rs +++ b/apps/labrinth/src/routes/internal/billing.rs @@ -1749,7 +1749,8 @@ pub async fn stripe_webhook( { client .post(format!( - "https://archon.pyro.host/modrinth/v0/servers/{}/unsuspend", + "{}/modrinth/v0/servers/{}/unsuspend", + dotenvy::var("ARCHON_URL")?, id )) .header("X-Master-Key", dotenvy::var("PYRO_API_KEY")?) @@ -1757,20 +1758,25 @@ pub async fn stripe_webhook( .await? .error_for_status()?; - client.post(format!( - "https://archon.pyro.host/modrinth/v0/servers/{}/reallocate", + client + .post(format!( + "{}/modrinth/v0/servers/{}/reallocate", + dotenvy::var("ARCHON_URL")?, id )) - .header("X-Master-Key", dotenvy::var("PYRO_API_KEY")?) - .json(&serde_json::json!({ - "memory_mb": ram, - "cpu": cpu, - "swap_mb": swap, - "storage_mb": storage, - })) - .send() - .await? - .error_for_status()?; + .header( + "X-Master-Key", + dotenvy::var("PYRO_API_KEY")?, + ) + .json(&serde_json::json!({ + "memory_mb": ram, + "cpu": cpu, + "swap_mb": swap, + "storage_mb": storage, + })) + .send() + .await? + .error_for_status()?; } else { let (server_name, source) = if let Some( PaymentRequestMetadata::Pyro { @@ -1814,7 +1820,10 @@ pub async fn stripe_webhook( } let res = client - .post("https://archon.pyro.host/modrinth/v0/servers/create") + .post(format!( + "{}/modrinth/v0/servers/create", + dotenvy::var("ARCHON_URL")?, + )) .header("X-Master-Key", dotenvy::var("PYRO_API_KEY")?) .json(&serde_json::json!({ "user_id": to_base62(metadata.user_item.id.0 as u64), @@ -2201,7 +2210,8 @@ pub async fn subscription_task(pool: PgPool, redis: RedisPool) { { let res = reqwest::Client::new() .post(format!( - "https://archon.pyro.host/modrinth/v0/servers/{}/suspend", + "{}/modrinth/v0/servers/{}/suspend", + dotenvy::var("ARCHON_URL")?, id )) .header("X-Master-Key", dotenvy::var("PYRO_API_KEY")?) From 621ed5fb02f0fd1f81d3f922c010d93eebb128f8 Mon Sep 17 00:00:00 2001 From: Prospector <6166773+Prospector@users.noreply.github.com> Date: Tue, 11 Mar 2025 23:28:17 -0700 Subject: [PATCH 10/36] Fix wording on CMP info page (#3385) --- apps/frontend/src/pages/legal/cmp-info.vue | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/frontend/src/pages/legal/cmp-info.vue b/apps/frontend/src/pages/legal/cmp-info.vue index 0a10b3be8..ddf4ec842 100644 --- a/apps/frontend/src/pages/legal/cmp-info.vue +++ b/apps/frontend/src/pages/legal/cmp-info.vue @@ -125,9 +125,9 @@ revenue distribution system. We also have an - API route that allows users - to query exact daily revenue for the site - so far, Modrinth has generated - {{ formatMoney(platformRevenue) }} in revenue. + API route + to query the exact daily advertising revenue for the site - so far, creators on Modrinth have + earned a total of {{ formatMoney(platformRevenue) }} in ad revenue.

@@ -148,7 +148,7 @@
Modrinth's total revenue in the previous 5 days, for the entire dataset, use the + >Modrinth's total ad revenue in the previous 5 days, for the entire dataset, use the aforementioned API route. From 31a001bbc11035fc8fe6f3d5f1e043e7a821b44d Mon Sep 17 00:00:00 2001 From: Emma Alexia Date: Wed, 12 Mar 2025 12:53:11 -0400 Subject: [PATCH 11/36] Fix moderation review page (#3389) --- apps/frontend/src/pages/moderation/review.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/frontend/src/pages/moderation/review.vue b/apps/frontend/src/pages/moderation/review.vue index 6348fbe02..7d2372cff 100644 --- a/apps/frontend/src/pages/moderation/review.vue +++ b/apps/frontend/src/pages/moderation/review.vue @@ -164,7 +164,7 @@ const projectTypes = computed(() => { return [...set]; }); -function segmentData(data, segmentSize = 900) { +function segmentData(data, segmentSize = 800) { return data.reduce((acc, curr, index) => { const segment = Math.floor(index / segmentSize); From 801c03981a54ef98a25b06e6663005483ca31a29 Mon Sep 17 00:00:00 2001 From: Erb3 <49862976+Erb3@users.noreply.github.com> Date: Wed, 12 Mar 2025 17:53:44 +0100 Subject: [PATCH 12/36] refactor(web): properly create table in cmp-info according to spec (#3362) Stops the vue compiler from nagging you, and improves consistency with other tables. --- apps/frontend/src/pages/legal/cmp-info.vue | 56 ++++++++++++---------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/apps/frontend/src/pages/legal/cmp-info.vue b/apps/frontend/src/pages/legal/cmp-info.vue index ddf4ec842..d08bec903 100644 --- a/apps/frontend/src/pages/legal/cmp-info.vue +++ b/apps/frontend/src/pages/legal/cmp-info.vue @@ -92,32 +92,38 @@

- - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + +
TimelineDate
Revenue earned on - - -
End of the month{{ formatDate(endOfMonthDate) }}
NET 60 policy applied+ 60 days
Available for withdrawal{{ formatDate(withdrawalDate) }}
TimelineDate
Revenue earned on + + +
End of the month{{ formatDate(endOfMonthDate) }}
NET 60 policy applied+ 60 days
Available for withdrawal{{ formatDate(withdrawalDate) }}
+

How do I know Modrinth is being transparent about revenue?

We aim to be as transparent as possible with creator revenue. All of our code is open source, From 827e3ec0a0a7149709df4d292add222c490e8318 Mon Sep 17 00:00:00 2001 From: Tiziano <69322987+T1xx1@users.noreply.github.com> Date: Wed, 12 Mar 2025 17:53:55 +0100 Subject: [PATCH 13/36] fix(frontend): mobile navbar covers legal disclaimer in the bottom of the footer (#3366) --- apps/frontend/src/layouts/default.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/frontend/src/layouts/default.vue b/apps/frontend/src/layouts/default.vue index 8a70ab6bb..1e2297b69 100644 --- a/apps/frontend/src/layouts/default.vue +++ b/apps/frontend/src/layouts/default.vue @@ -540,7 +540,7 @@

-
+
From eccd8524266b3aaa5e747a45a70911a69713f58d Mon Sep 17 00:00:00 2001 From: Josiah Glosson Date: Thu, 13 Mar 2025 15:04:44 -0500 Subject: [PATCH 14/36] Add log config parsing support to daedelus (#3395) --- packages/daedalus/src/minecraft.rs | 41 ++++++++++++++++++++++++++++++ packages/daedalus/src/modded.rs | 1 + 2 files changed, 42 insertions(+) diff --git a/packages/daedalus/src/minecraft.rs b/packages/daedalus/src/minecraft.rs index e117052ad..24e94a67e 100644 --- a/packages/daedalus/src/minecraft.rs +++ b/packages/daedalus/src/minecraft.rs @@ -402,6 +402,44 @@ pub enum ArgumentType { Jvm, } +#[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Hash)] +#[serde(rename_all = "snake_case")] +/// The physical side of the logging configuration +pub enum LoggingSide { + /// Client logging configuration + Client, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +/// File download information for a logging configuration +pub struct LogConfigDownload { + /// The path that the logging configuration should be saved to + pub id: String, + /// The SHA1 hash of the logging configuration + pub sha1: String, + /// The size of the logging configuration + pub size: u32, + /// The URL where the logging configuration can be downloaded + pub url: String, +} + +#[derive(Serialize, Deserialize, Debug)] +#[serde( + tag = "type", + rename_all = "kebab-case", + rename_all_fields = "camelCase" +)] +/// Information about a version's logging configuration +pub enum LoggingConfiguration { + /// Use a log4j2 XML log config file + Log4j2Xml { + /// The JVM argument for passing the file to the Java process + argument: String, + /// The config file to download + file: LogConfigDownload, + }, +} + #[derive(Serialize, Deserialize, Debug)] #[serde(rename_all = "camelCase")] /// Information about a version @@ -422,6 +460,9 @@ pub struct VersionInfo { pub java_version: Option, /// Libraries that the version depends on pub libraries: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + /// The logging configuration data for the game + pub logging: Option>, /// The classpath to the main class to launch the game pub main_class: String, #[serde(skip_serializing_if = "Option::is_none")] diff --git a/packages/daedalus/src/modded.rs b/packages/daedalus/src/modded.rs index 4016870ff..b828cd6dc 100644 --- a/packages/daedalus/src/modded.rs +++ b/packages/daedalus/src/modded.rs @@ -164,6 +164,7 @@ pub fn merge_partial_version( x }) .collect::>(), + logging: merge.logging, main_class: if let Some(main_class) = partial.main_class { main_class } else { From b665c17be85fdcbf453b98a5fd4048097a0c506e Mon Sep 17 00:00:00 2001 From: Prospector <6166773+Prospector@users.noreply.github.com> Date: Thu, 13 Mar 2025 19:19:40 -0700 Subject: [PATCH 15/36] Update servers marketing page (#3399) Co-authored-by: Jai Agrawal <18202329+Geometrically@users.noreply.github.com> --- apps/frontend/src/pages/servers/index.vue | 20 +++++-------------- .../src/pages/servers/manage/[id].vue | 2 -- .../src/pages/servers/manage/index.vue | 2 -- 3 files changed, 5 insertions(+), 19 deletions(-) diff --git a/apps/frontend/src/pages/servers/index.vue b/apps/frontend/src/pages/servers/index.vue index e310e7000..ecea9e0cb 100644 --- a/apps/frontend/src/pages/servers/index.vue +++ b/apps/frontend/src/pages/servers/index.vue @@ -62,7 +62,6 @@
@@ -209,9 +208,7 @@ -

- Experience modern, reliable hosting powered by Pyro -

+

Experience modern, reliable hosting

Modrinth Servers are hosted on 2023 Ryzen 7/9 CPUs with DDR5 RAM, running on @@ -223,15 +220,8 @@

Consistently fast

- Under Pyro, infrastructure is never overloaded, meaning each Modrinth server always - runs at its full performance. - - See the infrastructure - + Our infrastructure is never overloaded, meaning each server hosted with Modrinth + always runs at its full performance.

@@ -443,8 +433,8 @@

Yes. All Modrinth Servers come with DDoS protection. Protection is powered by a - combination of in-house network filtering by Pyro as well as with our data center - provider. Your server is safe on Modrinth. + combination of in-house network filtering as well as with our data center provider. + Your server is safe on Modrinth.

diff --git a/apps/frontend/src/pages/servers/manage/[id].vue b/apps/frontend/src/pages/servers/manage/[id].vue index f7a3af7ea..0d1178738 100644 --- a/apps/frontend/src/pages/servers/manage/[id].vue +++ b/apps/frontend/src/pages/servers/manage/[id].vue @@ -381,8 +381,6 @@ @reinstall="onReinstall" /> - - diff --git a/apps/frontend/src/pages/servers/manage/index.vue b/apps/frontend/src/pages/servers/manage/index.vue index c3f0cf9db..32b69676f 100644 --- a/apps/frontend/src/pages/servers/manage/index.vue +++ b/apps/frontend/src/pages/servers/manage/index.vue @@ -109,8 +109,6 @@

No servers found.

- - From 09ae3515f7a7a659c1768db113a56e27e2a49364 Mon Sep 17 00:00:00 2001 From: Prospector Date: Wed, 12 Mar 2025 10:08:44 -0700 Subject: [PATCH 16/36] Update changelog --- packages/utils/changelog.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/packages/utils/changelog.ts b/packages/utils/changelog.ts index 5ab3a42a7..14172ba95 100644 --- a/packages/utils/changelog.ts +++ b/packages/utils/changelog.ts @@ -10,6 +10,16 @@ export type VersionEntry = { } const VERSIONS: VersionEntry[] = [ + { + date: `2025-03-12T10:15:00-08:00`, + product: 'web', + body: `### Improvements +- Fixed low-res icons being pixelated. +- Fixed mobile navbar hiding bottom of footer. +- Updated CMP info page to correct some incorrect information. +- Updated CCPA notice with updated information since Modrinth Servers and Modrinth+. +- Fixed review page failing under edge case.`, + }, { date: `2025-03-05T17:40:00-08:00`, product: 'web', From 84a9438a70ba358d6949d4d2fa50f0d3ae7a3130 Mon Sep 17 00:00:00 2001 From: Prospector Date: Thu, 13 Mar 2025 19:24:34 -0700 Subject: [PATCH 17/36] Update changelog --- packages/utils/changelog.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/packages/utils/changelog.ts b/packages/utils/changelog.ts index 14172ba95..1bb2ebcb7 100644 --- a/packages/utils/changelog.ts +++ b/packages/utils/changelog.ts @@ -10,6 +10,12 @@ export type VersionEntry = { } const VERSIONS: VersionEntry[] = [ + { + date: `2025-03-13T19:30:00-08:00`, + product: 'web', + body: `### Improvements +- Updated Modrinth Servers marketing page, removing Pyro branding.`, + }, { date: `2025-03-12T10:15:00-08:00`, product: 'web', From c998d2566ed491ea6680d07002b6f2e14c31c34e Mon Sep 17 00:00:00 2001 From: Josiah Glosson Date: Sat, 15 Mar 2025 09:28:20 -0500 Subject: [PATCH 18/36] Allow multiple labrinth instances (#3360) * Move a lot of scheduled tasks to be runnable from the command-line * Use pubsub to handle sockets connected to multiple Labrinths * Clippy fix * Fix build and merge some stuff * Fix build fmt : --------- Signed-off-by: Jai Agrawal <18202329+Geometrically@users.noreply.github.com> Co-authored-by: Jai A Co-authored-by: Jai Agrawal <18202329+Geometrically@users.noreply.github.com> --- Cargo.lock | 1 + ...179cc9c02b1c0364319e76454dff713abdd45.json | 14 + ...bd168aaa2f791305f30adbf3b002ba39da7fa.json | 14 + ...a6e45856841256d812ce9ae3c07f903c5cc62.json | 14 - ...2800a0a43dfef4a37a5725403d33ccb20d908.json | 15 - ...f9e8193eb240501d30d5ffb4129e2103efd3d.json | 14 - ...95ff0b6eb1d3c4350d8e025d39d927d4547fc.json | 15 + apps/labrinth/Cargo.toml | 2 + apps/labrinth/src/background_task.rs | 278 +++++++++ apps/labrinth/src/database/redis.rs | 31 +- apps/labrinth/src/lib.rs | 219 ++++--- apps/labrinth/src/main.rs | 43 +- apps/labrinth/src/routes/internal/billing.rs | 552 +++++++++--------- apps/labrinth/src/routes/internal/statuses.rs | 162 +++-- apps/labrinth/src/routes/mod.rs | 6 +- apps/labrinth/src/routes/v3/friends.rs | 26 +- apps/labrinth/src/scheduler.rs | 177 ------ apps/labrinth/src/sync/friends.rs | 87 +++ apps/labrinth/src/sync/mod.rs | 2 + apps/labrinth/src/sync/status.rs | 71 +++ apps/labrinth/tests/common/mod.rs | 5 + 21 files changed, 1056 insertions(+), 692 deletions(-) create mode 100644 apps/labrinth/.sqlx/query-41ec8301348dc912d0e5a16def1179cc9c02b1c0364319e76454dff713abdd45.json create mode 100644 apps/labrinth/.sqlx/query-4ce906f3bec42a2d4b9ed8b8481bd168aaa2f791305f30adbf3b002ba39da7fa.json delete mode 100644 apps/labrinth/.sqlx/query-b971cecafab7046c5952447fd78a6e45856841256d812ce9ae3c07f903c5cc62.json delete mode 100644 apps/labrinth/.sqlx/query-bd26a27ce80ca796ae19bc709c92800a0a43dfef4a37a5725403d33ccb20d908.json delete mode 100644 apps/labrinth/.sqlx/query-c8a27a122160a0896914c786deef9e8193eb240501d30d5ffb4129e2103efd3d.json create mode 100644 apps/labrinth/.sqlx/query-f2525e9be3b90fc0c42c8333ca795ff0b6eb1d3c4350d8e025d39d927d4547fc.json create mode 100644 apps/labrinth/src/background_task.rs create mode 100644 apps/labrinth/src/sync/friends.rs create mode 100644 apps/labrinth/src/sync/mod.rs create mode 100644 apps/labrinth/src/sync/status.rs diff --git a/Cargo.lock b/Cargo.lock index 55f13a32d..3396996df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4522,6 +4522,7 @@ dependencies = [ "bytes 1.7.2", "censor", "chrono", + "clap", "clickhouse", "color-thief", "console-subscriber", diff --git a/apps/labrinth/.sqlx/query-41ec8301348dc912d0e5a16def1179cc9c02b1c0364319e76454dff713abdd45.json b/apps/labrinth/.sqlx/query-41ec8301348dc912d0e5a16def1179cc9c02b1c0364319e76454dff713abdd45.json new file mode 100644 index 000000000..117ee6ed2 --- /dev/null +++ b/apps/labrinth/.sqlx/query-41ec8301348dc912d0e5a16def1179cc9c02b1c0364319e76454dff713abdd45.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE versions\n SET status = requested_status\n WHERE status = $1 AND date_published < CURRENT_DATE AND requested_status IS NOT NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "41ec8301348dc912d0e5a16def1179cc9c02b1c0364319e76454dff713abdd45" +} diff --git a/apps/labrinth/.sqlx/query-4ce906f3bec42a2d4b9ed8b8481bd168aaa2f791305f30adbf3b002ba39da7fa.json b/apps/labrinth/.sqlx/query-4ce906f3bec42a2d4b9ed8b8481bd168aaa2f791305f30adbf3b002ba39da7fa.json new file mode 100644 index 000000000..ec8e1a23b --- /dev/null +++ b/apps/labrinth/.sqlx/query-4ce906f3bec42a2d4b9ed8b8481bd168aaa2f791305f30adbf3b002ba39da7fa.json @@ -0,0 +1,14 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE mods\n SET status = requested_status\n WHERE status = $1 AND approved < CURRENT_DATE AND requested_status IS NOT NULL\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [] + }, + "hash": "4ce906f3bec42a2d4b9ed8b8481bd168aaa2f791305f30adbf3b002ba39da7fa" +} diff --git a/apps/labrinth/.sqlx/query-b971cecafab7046c5952447fd78a6e45856841256d812ce9ae3c07f903c5cc62.json b/apps/labrinth/.sqlx/query-b971cecafab7046c5952447fd78a6e45856841256d812ce9ae3c07f903c5cc62.json deleted file mode 100644 index be3795083..000000000 --- a/apps/labrinth/.sqlx/query-b971cecafab7046c5952447fd78a6e45856841256d812ce9ae3c07f903c5cc62.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE mods\n SET status = requested_status\n WHERE status = $1 AND approved < CURRENT_DATE AND requested_status IS NOT NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [] - }, - "hash": "b971cecafab7046c5952447fd78a6e45856841256d812ce9ae3c07f903c5cc62" -} diff --git a/apps/labrinth/.sqlx/query-bd26a27ce80ca796ae19bc709c92800a0a43dfef4a37a5725403d33ccb20d908.json b/apps/labrinth/.sqlx/query-bd26a27ce80ca796ae19bc709c92800a0a43dfef4a37a5725403d33ccb20d908.json deleted file mode 100644 index c0c2cbe97..000000000 --- a/apps/labrinth/.sqlx/query-bd26a27ce80ca796ae19bc709c92800a0a43dfef4a37a5725403d33ccb20d908.json +++ /dev/null @@ -1,15 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE users\n SET badges = $1\n WHERE (id = $2)\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Int8", - "Int8" - ] - }, - "nullable": [] - }, - "hash": "bd26a27ce80ca796ae19bc709c92800a0a43dfef4a37a5725403d33ccb20d908" -} diff --git a/apps/labrinth/.sqlx/query-c8a27a122160a0896914c786deef9e8193eb240501d30d5ffb4129e2103efd3d.json b/apps/labrinth/.sqlx/query-c8a27a122160a0896914c786deef9e8193eb240501d30d5ffb4129e2103efd3d.json deleted file mode 100644 index bebb6425a..000000000 --- a/apps/labrinth/.sqlx/query-c8a27a122160a0896914c786deef9e8193eb240501d30d5ffb4129e2103efd3d.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n UPDATE versions\n SET status = requested_status\n WHERE status = $1 AND date_published < CURRENT_DATE AND requested_status IS NOT NULL\n ", - "describe": { - "columns": [], - "parameters": { - "Left": [ - "Text" - ] - }, - "nullable": [] - }, - "hash": "c8a27a122160a0896914c786deef9e8193eb240501d30d5ffb4129e2103efd3d" -} diff --git a/apps/labrinth/.sqlx/query-f2525e9be3b90fc0c42c8333ca795ff0b6eb1d3c4350d8e025d39d927d4547fc.json b/apps/labrinth/.sqlx/query-f2525e9be3b90fc0c42c8333ca795ff0b6eb1d3c4350d8e025d39d927d4547fc.json new file mode 100644 index 000000000..7b1913392 --- /dev/null +++ b/apps/labrinth/.sqlx/query-f2525e9be3b90fc0c42c8333ca795ff0b6eb1d3c4350d8e025d39d927d4547fc.json @@ -0,0 +1,15 @@ +{ + "db_name": "PostgreSQL", + "query": "\n UPDATE users\n SET badges = $1\n WHERE (id = $2)\n ", + "describe": { + "columns": [], + "parameters": { + "Left": [ + "Int8", + "Int8" + ] + }, + "nullable": [] + }, + "hash": "f2525e9be3b90fc0c42c8333ca795ff0b6eb1d3c4350d8e025d39d927d4547fc" +} diff --git a/apps/labrinth/Cargo.toml b/apps/labrinth/Cargo.toml index 584b8b935..07bb1fbfe 100644 --- a/apps/labrinth/Cargo.toml +++ b/apps/labrinth/Cargo.toml @@ -131,6 +131,8 @@ json-patch = "*" ariadne = { path = "../../packages/ariadne" } +clap = { version = "4.5", features = ["derive"] } + [target.'cfg(not(target_env = "msvc"))'.dependencies] tikv-jemallocator = { version = "0.6.0", features = ["profiling", "unprefixed_malloc_on_supported_platforms"] } tikv-jemalloc-ctl = { version = "0.6.0", features = ["stats"] } diff --git a/apps/labrinth/src/background_task.rs b/apps/labrinth/src/background_task.rs new file mode 100644 index 000000000..c534a83a4 --- /dev/null +++ b/apps/labrinth/src/background_task.rs @@ -0,0 +1,278 @@ +use crate::database::redis::RedisPool; +use crate::queue::payouts::process_payout; +use crate::search; +use crate::search::indexing::index_projects; +use clap::ValueEnum; +use sqlx::Postgres; +use tracing::{info, warn}; + +#[derive(ValueEnum, Debug, Copy, Clone, PartialEq, Eq)] +#[clap(rename_all = "kebab_case")] +pub enum BackgroundTask { + IndexSearch, + ReleaseScheduled, + UpdateVersions, + Payouts, + IndexBilling, + IndexSubscriptions, +} + +impl BackgroundTask { + pub async fn run( + self, + pool: sqlx::Pool, + redis_pool: RedisPool, + search_config: search::SearchConfig, + clickhouse: clickhouse::Client, + stripe_client: stripe::Client, + ) { + use BackgroundTask::*; + match self { + IndexSearch => index_search(pool, redis_pool, search_config).await, + ReleaseScheduled => release_scheduled(pool).await, + UpdateVersions => update_versions(pool, redis_pool).await, + Payouts => payouts(pool, clickhouse).await, + IndexBilling => { + crate::routes::internal::billing::index_billing( + stripe_client, + pool, + redis_pool, + ) + .await + } + IndexSubscriptions => { + crate::routes::internal::billing::index_subscriptions( + pool, redis_pool, + ) + .await + } + } + } +} + +pub async fn index_search( + pool: sqlx::Pool, + redis_pool: RedisPool, + search_config: search::SearchConfig, +) { + info!("Indexing local database"); + let result = index_projects(pool, redis_pool, &search_config).await; + if let Err(e) = result { + warn!("Local project indexing failed: {:?}", e); + } + info!("Done indexing local database"); +} + +pub async fn release_scheduled(pool: sqlx::Pool) { + info!("Releasing scheduled versions/projects!"); + + let projects_results = sqlx::query!( + " + UPDATE mods + SET status = requested_status + WHERE status = $1 AND approved < CURRENT_DATE AND requested_status IS NOT NULL + ", + crate::models::projects::ProjectStatus::Scheduled.as_str(), + ) + .execute(&pool) + .await; + + if let Err(e) = projects_results { + warn!("Syncing scheduled releases for projects failed: {:?}", e); + } + + let versions_results = sqlx::query!( + " + UPDATE versions + SET status = requested_status + WHERE status = $1 AND date_published < CURRENT_DATE AND requested_status IS NOT NULL + ", + crate::models::projects::VersionStatus::Scheduled.as_str(), + ) + .execute(&pool) + .await; + + if let Err(e) = versions_results { + warn!("Syncing scheduled releases for versions failed: {:?}", e); + } + + info!("Finished releasing scheduled versions/projects"); +} + +pub async fn update_versions( + pool: sqlx::Pool, + redis_pool: RedisPool, +) { + info!("Indexing game versions list from Mojang"); + let result = version_updater::update_versions(&pool, &redis_pool).await; + if let Err(e) = result { + warn!("Version update failed: {}", e); + } + info!("Done indexing game versions"); +} + +pub async fn payouts( + pool: sqlx::Pool, + clickhouse: clickhouse::Client, +) { + info!("Started running payouts"); + let result = process_payout(&pool, &clickhouse).await; + if let Err(e) = result { + warn!("Payouts run failed: {:?}", e); + } + info!("Done running payouts"); +} + +mod version_updater { + use crate::database::models::legacy_loader_fields::MinecraftGameVersion; + use crate::database::redis::RedisPool; + use chrono::{DateTime, Utc}; + use serde::Deserialize; + use sqlx::Postgres; + use thiserror::Error; + use tracing::warn; + + #[derive(Deserialize)] + struct InputFormat<'a> { + // latest: LatestFormat, + versions: Vec>, + } + + #[derive(Deserialize)] + struct VersionFormat<'a> { + id: String, + #[serde(rename = "type")] + type_: std::borrow::Cow<'a, str>, + #[serde(rename = "releaseTime")] + release_time: DateTime, + } + + #[derive(Error, Debug)] + pub enum VersionIndexingError { + #[error("Network error while updating game versions list: {0}")] + NetworkError(#[from] reqwest::Error), + #[error("Database error while updating game versions list: {0}")] + DatabaseError(#[from] crate::database::models::DatabaseError), + } + + pub async fn update_versions( + pool: &sqlx::Pool, + redis: &RedisPool, + ) -> Result<(), VersionIndexingError> { + let input = reqwest::get( + "https://piston-meta.mojang.com/mc/game/version_manifest_v2.json", + ) + .await? + .json::() + .await?; + + let mut skipped_versions_count = 0u32; + + // A list of version names that contains spaces. + // Generated using the command + // ```sh + // curl https://launchermeta.mojang.com/mc/game/version_manifest.json \ + // | jq '[.versions[].id | select(contains(" "))]' + // ``` + const HALL_OF_SHAME: [(&str, &str); 12] = [ + ("1.14.2 Pre-Release 4", "1.14.2-pre4"), + ("1.14.2 Pre-Release 3", "1.14.2-pre3"), + ("1.14.2 Pre-Release 2", "1.14.2-pre2"), + ("1.14.2 Pre-Release 1", "1.14.2-pre1"), + ("1.14.1 Pre-Release 2", "1.14.1-pre2"), + ("1.14.1 Pre-Release 1", "1.14.1-pre1"), + ("1.14 Pre-Release 5", "1.14-pre5"), + ("1.14 Pre-Release 4", "1.14-pre4"), + ("1.14 Pre-Release 3", "1.14-pre3"), + ("1.14 Pre-Release 2", "1.14-pre2"), + ("1.14 Pre-Release 1", "1.14-pre1"), + ("3D Shareware v1.34", "3D-Shareware-v1.34"), + ]; + + lazy_static::lazy_static! { + /// Mojank for some reason has versions released at the same DateTime. This hardcodes them to fix this, + /// as most of our ordering logic is with DateTime + static ref HALL_OF_SHAME_2: [(&'static str, DateTime); 4] = [ + ( + "1.4.5", + chrono::DateTime::parse_from_rfc3339("2012-12-19T22:00:00+00:00") + .unwrap() + .into(), + ), + ( + "1.4.6", + chrono::DateTime::parse_from_rfc3339("2012-12-19T22:00:01+00:00") + .unwrap() + .into(), + ), + ( + "1.6.3", + chrono::DateTime::parse_from_rfc3339("2013-09-13T10:54:41+00:00") + .unwrap() + .into(), + ), + ( + "13w37b", + chrono::DateTime::parse_from_rfc3339("2013-09-13T10:54:42+00:00") + .unwrap() + .into(), + ), + ]; + } + + for version in input.versions.into_iter() { + let mut name = version.id; + if !name + .chars() + .all(|c| c.is_ascii_alphanumeric() || "-_.".contains(c)) + { + if let Some((_, alternate)) = + HALL_OF_SHAME.iter().find(|(version, _)| name == *version) + { + name = String::from(*alternate); + } else { + // We'll deal with these manually + skipped_versions_count += 1; + continue; + } + } + + let type_ = match &*version.type_ { + "release" => "release", + "snapshot" => "snapshot", + "old_alpha" => "alpha", + "old_beta" => "beta", + _ => "other", + }; + + MinecraftGameVersion::builder() + .version(&name)? + .version_type(type_)? + .created( + if let Some((_, alternate)) = HALL_OF_SHAME_2 + .iter() + .find(|(version, _)| name == *version) + { + alternate + } else { + &version.release_time + }, + ) + .insert(pool, redis) + .await?; + } + + if skipped_versions_count > 0 { + // This will currently always trigger due to 1.14 pre releases + // and the shareware april fools update. We could set a threshold + // that accounts for those versions and update it whenever we + // manually fix another version. + warn!( + "Skipped {} game versions; check for new versions and add them manually", + skipped_versions_count + ); + } + + Ok(()) + } +} diff --git a/apps/labrinth/src/database/redis.rs b/apps/labrinth/src/database/redis.rs index 3fe92a53d..9a0409071 100644 --- a/apps/labrinth/src/database/redis.rs +++ b/apps/labrinth/src/database/redis.rs @@ -19,6 +19,7 @@ const ACTUAL_EXPIRY: i64 = 60 * 30; // 30 minutes #[derive(Clone)] pub struct RedisPool { + pub url: String, pub pool: deadpool_redis::Pool, meta_namespace: String, } @@ -33,23 +34,23 @@ impl RedisPool { // testing pool uses a hashmap to mimic redis behaviour for very small data sizes (ie: tests) // PANICS: production pool will panic if redis url is not set pub fn new(meta_namespace: Option) -> Self { - let redis_pool = Config::from_url( - dotenvy::var("REDIS_URL").expect("Redis URL not set"), - ) - .builder() - .expect("Error building Redis pool") - .max_size( - dotenvy::var("REDIS_MAX_CONNECTIONS") - .ok() - .and_then(|x| x.parse().ok()) - .unwrap_or(10000), - ) - .runtime(Runtime::Tokio1) - .build() - .expect("Redis connection failed"); + let url = dotenvy::var("REDIS_URL").expect("Redis URL not set"); + let pool = Config::from_url(url.clone()) + .builder() + .expect("Error building Redis pool") + .max_size( + dotenvy::var("REDIS_MAX_CONNECTIONS") + .ok() + .and_then(|x| x.parse().ok()) + .unwrap_or(10000), + ) + .runtime(Runtime::Tokio1) + .build() + .expect("Redis connection failed"); RedisPool { - pool: redis_pool, + url, + pool, meta_namespace: meta_namespace.unwrap_or("".to_string()), } } diff --git a/apps/labrinth/src/lib.rs b/apps/labrinth/src/lib.rs index bfaf95d81..97052fb78 100644 --- a/apps/labrinth/src/lib.rs +++ b/apps/labrinth/src/lib.rs @@ -17,15 +17,14 @@ use governor::middleware::StateInformationMiddleware; use governor::{Quota, RateLimiter}; use util::cors::default_cors; +use crate::background_task::update_versions; use crate::queue::moderation::AutomatedModerationQueue; +use crate::util::env::{parse_strings_from_var, parse_var}; use crate::util::ratelimit::KeyedRateLimiter; -use crate::{ - queue::payouts::process_payout, - search::indexing::index_projects, - util::env::{parse_strings_from_var, parse_var}, -}; +use sync::friends::handle_pubsub; pub mod auth; +pub mod background_task; pub mod clickhouse; pub mod database; pub mod file_hosting; @@ -34,6 +33,7 @@ pub mod queue; pub mod routes; pub mod scheduler; pub mod search; +pub mod sync; pub mod util; pub mod validate; @@ -61,6 +61,7 @@ pub struct LabrinthConfig { pub stripe_client: stripe::Client, } +#[allow(clippy::too_many_arguments)] pub fn app_setup( pool: sqlx::Pool, redis_pool: RedisPool, @@ -68,6 +69,8 @@ pub fn app_setup( clickhouse: &mut Client, file_host: Arc, maxmind: Arc, + stripe_client: stripe::Client, + enable_background_tasks: bool, ) -> LabrinthConfig { info!( "Starting Labrinth on {}", @@ -109,88 +112,97 @@ pub fn app_setup( async move {} }); - // The interval in seconds at which the local database is indexed - // for searching. Defaults to 1 hour if unset. - let local_index_interval = std::time::Duration::from_secs( - parse_var("LOCAL_INDEX_INTERVAL").unwrap_or(3600), - ); - - let pool_ref = pool.clone(); - let search_config_ref = search_config.clone(); - let redis_pool_ref = redis_pool.clone(); - scheduler.run(local_index_interval, move || { - let pool_ref = pool_ref.clone(); - let redis_pool_ref = redis_pool_ref.clone(); - let search_config_ref = search_config_ref.clone(); - async move { - info!("Indexing local database"); - let result = index_projects( - pool_ref, - redis_pool_ref.clone(), - &search_config_ref, - ) - .await; - if let Err(e) = result { - warn!("Local project indexing failed: {:?}", e); + if enable_background_tasks { + // The interval in seconds at which the local database is indexed + // for searching. Defaults to 1 hour if unset. + let local_index_interval = Duration::from_secs( + parse_var("LOCAL_INDEX_INTERVAL").unwrap_or(3600), + ); + let pool_ref = pool.clone(); + let search_config_ref = search_config.clone(); + let redis_pool_ref = redis_pool.clone(); + scheduler.run(local_index_interval, move || { + let pool_ref = pool_ref.clone(); + let redis_pool_ref = redis_pool_ref.clone(); + let search_config_ref = search_config_ref.clone(); + async move { + background_task::index_search( + pool_ref, + redis_pool_ref, + search_config_ref, + ) + .await; } - info!("Done indexing local database"); - } - }); + }); - // Changes statuses of scheduled projects/versions - let pool_ref = pool.clone(); - // TODO: Clear cache when these are run - scheduler.run(std::time::Duration::from_secs(60 * 5), move || { - let pool_ref = pool_ref.clone(); - info!("Releasing scheduled versions/projects!"); - - async move { - let projects_results = sqlx::query!( - " - UPDATE mods - SET status = requested_status - WHERE status = $1 AND approved < CURRENT_DATE AND requested_status IS NOT NULL - ", - crate::models::projects::ProjectStatus::Scheduled.as_str(), - ) - .execute(&pool_ref) - .await; - - if let Err(e) = projects_results { - warn!("Syncing scheduled releases for projects failed: {:?}", e); + // Changes statuses of scheduled projects/versions + let pool_ref = pool.clone(); + // TODO: Clear cache when these are run + scheduler.run(Duration::from_secs(60 * 5), move || { + let pool_ref = pool_ref.clone(); + async move { + background_task::release_scheduled(pool_ref).await; } + }); - let versions_results = sqlx::query!( - " - UPDATE versions - SET status = requested_status - WHERE status = $1 AND date_published < CURRENT_DATE AND requested_status IS NOT NULL - ", - crate::models::projects::VersionStatus::Scheduled.as_str(), - ) - .execute(&pool_ref) - .await; - - if let Err(e) = versions_results { - warn!("Syncing scheduled releases for versions failed: {:?}", e); + let version_index_interval = Duration::from_secs( + parse_var("VERSION_INDEX_INTERVAL").unwrap_or(1800), + ); + let pool_ref = pool.clone(); + let redis_pool_ref = redis_pool.clone(); + scheduler.run(version_index_interval, move || { + let pool_ref = pool_ref.clone(); + let redis = redis_pool_ref.clone(); + async move { + update_versions(pool_ref, redis).await; } + }); - info!("Finished releasing scheduled versions/projects"); - } - }); + let pool_ref = pool.clone(); + let client_ref = clickhouse.clone(); + scheduler.run(Duration::from_secs(60 * 60 * 6), move || { + let pool_ref = pool_ref.clone(); + let client_ref = client_ref.clone(); + async move { + background_task::payouts(pool_ref, client_ref).await; + } + }); - scheduler::schedule_versions( - &mut scheduler, - pool.clone(), - redis_pool.clone(), - ); + let pool_ref = pool.clone(); + let redis_ref = redis_pool.clone(); + let stripe_client_ref = stripe_client.clone(); + actix_rt::spawn(async move { + loop { + routes::internal::billing::index_billing( + stripe_client_ref.clone(), + pool_ref.clone(), + redis_ref.clone(), + ) + .await; + tokio::time::sleep(Duration::from_secs(60 * 5)).await; + } + }); + + let pool_ref = pool.clone(); + let redis_ref = redis_pool.clone(); + actix_rt::spawn(async move { + loop { + routes::internal::billing::index_subscriptions( + pool_ref.clone(), + redis_ref.clone(), + ) + .await; + tokio::time::sleep(Duration::from_secs(60 * 5)).await; + } + }); + } let session_queue = web::Data::new(AuthQueue::new()); let pool_ref = pool.clone(); let redis_ref = redis_pool.clone(); let session_queue_ref = session_queue.clone(); - scheduler.run(std::time::Duration::from_secs(60 * 30), move || { + scheduler.run(Duration::from_secs(60 * 30), move || { let pool_ref = pool_ref.clone(); let redis_ref = redis_ref.clone(); let session_queue_ref = session_queue_ref.clone(); @@ -208,7 +220,7 @@ pub fn app_setup( let reader = maxmind.clone(); { let reader_ref = reader; - scheduler.run(std::time::Duration::from_secs(60 * 60 * 24), move || { + scheduler.run(Duration::from_secs(60 * 60 * 24), move || { let reader_ref = reader_ref.clone(); async move { @@ -232,7 +244,7 @@ pub fn app_setup( let analytics_queue_ref = analytics_queue.clone(); let pool_ref = pool.clone(); let redis_ref = redis_pool.clone(); - scheduler.run(std::time::Duration::from_secs(15), move || { + scheduler.run(Duration::from_secs(15), move || { let client_ref = client_ref.clone(); let analytics_queue_ref = analytics_queue_ref.clone(); let pool_ref = pool_ref.clone(); @@ -251,51 +263,6 @@ pub fn app_setup( }); } - { - let pool_ref = pool.clone(); - let client_ref = clickhouse.clone(); - scheduler.run(std::time::Duration::from_secs(60 * 60 * 6), move || { - let pool_ref = pool_ref.clone(); - let client_ref = client_ref.clone(); - - async move { - info!("Started running payouts"); - let result = process_payout(&pool_ref, &client_ref).await; - if let Err(e) = result { - warn!("Payouts run failed: {:?}", e); - } - info!("Done running payouts"); - } - }); - } - - let stripe_client = - stripe::Client::new(dotenvy::var("STRIPE_API_KEY").unwrap()); - { - let pool_ref = pool.clone(); - let redis_ref = redis_pool.clone(); - let stripe_client_ref = stripe_client.clone(); - - actix_rt::spawn(async move { - routes::internal::billing::task( - stripe_client_ref, - pool_ref, - redis_ref, - ) - .await; - }); - } - - { - let pool_ref = pool.clone(); - let redis_ref = redis_pool.clone(); - - actix_rt::spawn(async move { - routes::internal::billing::subscription_task(pool_ref, redis_ref) - .await; - }); - } - let ip_salt = Pepper { pepper: ariadne::ids::Base62Id(ariadne::ids::random_base62(11)) .to_string(), @@ -304,6 +271,16 @@ pub fn app_setup( let payouts_queue = web::Data::new(PayoutsQueue::new()); let active_sockets = web::Data::new(ActiveSockets::default()); + { + let pool = pool.clone(); + let redis_client = redis::Client::open(redis_pool.url.clone()).unwrap(); + let sockets = active_sockets.clone(); + actix_rt::spawn(async move { + let pubsub = redis_client.get_async_pubsub().await.unwrap(); + handle_pubsub(pubsub, pool, sockets).await; + }); + } + LabrinthConfig { pool, redis_pool, diff --git a/apps/labrinth/src/main.rs b/apps/labrinth/src/main.rs index e6058eed6..f45b3609c 100644 --- a/apps/labrinth/src/main.rs +++ b/apps/labrinth/src/main.rs @@ -1,5 +1,7 @@ use actix_web::{App, HttpServer}; use actix_web_prom::PrometheusMetricsBuilder; +use clap::Parser; +use labrinth::background_task::BackgroundTask; use labrinth::database::redis::RedisPool; use labrinth::file_hosting::S3Host; use labrinth::search; @@ -23,8 +25,23 @@ pub struct Pepper { pub pepper: String, } +#[derive(Parser)] +#[command(version)] +struct Args { + /// Don't run regularly scheduled background tasks. This means the tasks should be run + /// manually with --run-background-task. + #[arg(long)] + no_background_tasks: bool, + + /// Run a single background task and then exit. Perfect for cron jobs. + #[arg(long, value_enum, id = "task")] + run_background_task: Option, +} + #[actix_rt::main] async fn main() -> std::io::Result<()> { + let args = Args::parse(); + dotenvy::dotenv().ok(); console_subscriber::init(); @@ -44,10 +61,12 @@ async fn main() -> std::io::Result<()> { std::env::set_var("RUST_BACKTRACE", "1"); } - info!( - "Starting Labrinth on {}", - dotenvy::var("BIND_ADDR").unwrap() - ); + if args.run_background_task.is_none() { + info!( + "Starting Labrinth on {}", + dotenvy::var("BIND_ADDR").unwrap() + ); + } database::check_for_migrations() .await @@ -91,6 +110,18 @@ async fn main() -> std::io::Result<()> { info!("Initializing clickhouse connection"); let mut clickhouse = clickhouse::init_client().await.unwrap(); + let search_config = search::SearchConfig::new(None); + + let stripe_client = + stripe::Client::new(dotenvy::var("STRIPE_API_KEY").unwrap()); + + if let Some(task) = args.run_background_task { + info!("Running task {task:?} and exiting"); + task.run(pool, redis_pool, search_config, clickhouse, stripe_client) + .await; + return Ok(()); + } + let maxmind_reader = Arc::new(queue::maxmind::MaxMindIndexer::new().await.unwrap()); @@ -115,8 +146,6 @@ async fn main() -> std::io::Result<()> { labrinth::routes::debug::jemalloc_mmeory_stats(&prometheus.registry) .expect("Failed to register jemalloc metrics"); - let search_config = search::SearchConfig::new(None); - let labrinth_config = labrinth::app_setup( pool.clone(), redis_pool.clone(), @@ -124,6 +153,8 @@ async fn main() -> std::io::Result<()> { &mut clickhouse, file_host.clone(), maxmind_reader.clone(), + stripe_client, + !args.no_background_tasks, ); info!("Starting Actix HTTP server!"); diff --git a/apps/labrinth/src/routes/internal/billing.rs b/apps/labrinth/src/routes/internal/billing.rs index 036c3c3db..469555946 100644 --- a/apps/labrinth/src/routes/internal/billing.rs +++ b/apps/labrinth/src/routes/internal/billing.rs @@ -2091,323 +2091,331 @@ async fn get_or_create_customer( } } -pub async fn subscription_task(pool: PgPool, redis: RedisPool) { - loop { - info!("Indexing subscriptions"); +pub async fn index_subscriptions(pool: PgPool, redis: RedisPool) { + info!("Indexing subscriptions"); - let res = async { - let mut transaction = pool.begin().await?; - let mut clear_cache_users = Vec::new(); + let res = async { + let mut transaction = pool.begin().await?; + let mut clear_cache_users = Vec::new(); - // If an active subscription has a canceled charge OR a failed charge more than two days ago, it should be cancelled - let all_charges = ChargeItem::get_unprovision(&pool).await?; + // If an active subscription has a canceled charge OR a failed charge more than two days ago, it should be cancelled + let all_charges = ChargeItem::get_unprovision(&pool).await?; - let mut all_subscriptions = - user_subscription_item::UserSubscriptionItem::get_many( - &all_charges - .iter() - .filter_map(|x| x.subscription_id) - .collect::>() - .into_iter() - .collect::>(), - &pool, - ) - .await?; - let subscription_prices = product_item::ProductPriceItem::get_many( - &all_subscriptions + let mut all_subscriptions = + user_subscription_item::UserSubscriptionItem::get_many( + &all_charges .iter() - .map(|x| x.price_id) + .filter_map(|x| x.subscription_id) .collect::>() .into_iter() .collect::>(), &pool, ) .await?; - let subscription_products = product_item::ProductItem::get_many( - &subscription_prices - .iter() - .map(|x| x.product_id) - .collect::>() - .into_iter() - .collect::>(), - &pool, - ) - .await?; - let users = crate::database::models::User::get_many_ids( - &all_subscriptions - .iter() - .map(|x| x.user_id) - .collect::>() - .into_iter() - .collect::>(), - &pool, - &redis, - ) - .await?; + let subscription_prices = product_item::ProductPriceItem::get_many( + &all_subscriptions + .iter() + .map(|x| x.price_id) + .collect::>() + .into_iter() + .collect::>(), + &pool, + ) + .await?; + let subscription_products = product_item::ProductItem::get_many( + &subscription_prices + .iter() + .map(|x| x.product_id) + .collect::>() + .into_iter() + .collect::>(), + &pool, + ) + .await?; + let users = crate::database::models::User::get_many_ids( + &all_subscriptions + .iter() + .map(|x| x.user_id) + .collect::>() + .into_iter() + .collect::>(), + &pool, + &redis, + ) + .await?; - for charge in all_charges { - let subscription = if let Some(subscription) = all_subscriptions - .iter_mut() - .find(|x| Some(x.id) == charge.subscription_id) - { - subscription - } else { - continue; - }; + for charge in all_charges { + let subscription = if let Some(subscription) = all_subscriptions + .iter_mut() + .find(|x| Some(x.id) == charge.subscription_id) + { + subscription + } else { + continue; + }; - if subscription.status == SubscriptionStatus::Unprovisioned { - continue; + if subscription.status == SubscriptionStatus::Unprovisioned { + continue; + } + + let product_price = if let Some(product_price) = subscription_prices + .iter() + .find(|x| x.id == subscription.price_id) + { + product_price + } else { + continue; + }; + + let product = if let Some(product) = subscription_products + .iter() + .find(|x| x.id == product_price.product_id) + { + product + } else { + continue; + }; + + let user = if let Some(user) = + users.iter().find(|x| x.id == subscription.user_id) + { + user + } else { + continue; + }; + + let unprovisioned = match product.metadata { + ProductMetadata::Midas => { + let badges = user.badges - Badges::MIDAS; + + sqlx::query!( + " + UPDATE users + SET badges = $1 + WHERE (id = $2) + ", + badges.bits() as i64, + user.id as crate::database::models::ids::UserId, + ) + .execute(&mut *transaction) + .await?; + + true } + ProductMetadata::Pyro { .. } => { + if let Some(SubscriptionMetadata::Pyro { id }) = + &subscription.metadata + { + let res = reqwest::Client::new() + .post(format!( + "{}/modrinth/v0/servers/{}/suspend", + dotenvy::var("ARCHON_URL")?, + id + )) + .header("X-Master-Key", dotenvy::var("PYRO_API_KEY")?) + .json(&serde_json::json!({ + "reason": if charge.status == ChargeStatus::Cancelled { + "cancelled" + } else { + "paymentfailed" + } + })) + .send() + .await; - let product_price = if let Some(product_price) = - subscription_prices - .iter() - .find(|x| x.id == subscription.price_id) - { - product_price - } else { - continue; - }; - - let product = if let Some(product) = subscription_products - .iter() - .find(|x| x.id == product_price.product_id) - { - product - } else { - continue; - }; - - let user = if let Some(user) = - users.iter().find(|x| x.id == subscription.user_id) - { - user - } else { - continue; - }; - - let unprovisioned = match product.metadata { - ProductMetadata::Midas => { - let badges = user.badges - Badges::MIDAS; - - sqlx::query!( - " - UPDATE users - SET badges = $1 - WHERE (id = $2) - ", - badges.bits() as i64, - user.id as crate::database::models::ids::UserId, - ) - .execute(&mut *transaction) - .await?; - - true - } - ProductMetadata::Pyro { .. } => { - if let Some(SubscriptionMetadata::Pyro { id }) = - &subscription.metadata - { - let res = reqwest::Client::new() - .post(format!( - "{}/modrinth/v0/servers/{}/suspend", - dotenvy::var("ARCHON_URL")?, - id - )) - .header("X-Master-Key", dotenvy::var("PYRO_API_KEY")?) - .json(&serde_json::json!({ - "reason": if charge.status == ChargeStatus::Cancelled { - "cancelled" - } else { - "paymentfailed" - } - })) - .send() - .await; - - if let Err(e) = res { - warn!("Error suspending pyro server: {:?}", e); - false - } else { - true - } + if let Err(e) = res { + warn!("Error suspending pyro server: {:?}", e); + false } else { true } + } else { + true } - }; - - if unprovisioned { - subscription.status = SubscriptionStatus::Unprovisioned; - subscription.upsert(&mut transaction).await?; } + }; - clear_cache_users.push(user.id); + if unprovisioned { + subscription.status = SubscriptionStatus::Unprovisioned; + subscription.upsert(&mut transaction).await?; } - crate::database::models::User::clear_caches( - &clear_cache_users - .into_iter() - .map(|x| (x, None)) - .collect::>(), - &redis, - ) - .await?; - transaction.commit().await?; - - Ok::<(), ApiError>(()) - }; - - if let Err(e) = res.await { - warn!("Error indexing billing queue: {:?}", e); + clear_cache_users.push(user.id); } - info!("Done indexing billing queue"); + crate::database::models::User::clear_caches( + &clear_cache_users + .into_iter() + .map(|x| (x, None)) + .collect::>(), + &redis, + ) + .await?; + transaction.commit().await?; - tokio::time::sleep(std::time::Duration::from_secs(60 * 5)).await; + Ok::<(), ApiError>(()) + }; + + if let Err(e) = res.await { + warn!("Error indexing subscriptions: {:?}", e); } + + info!("Done indexing subscriptions"); } -pub async fn task( +pub async fn index_billing( stripe_client: stripe::Client, pool: PgPool, redis: RedisPool, ) { - loop { - info!("Indexing billing queue"); - let res = async { - // If a charge is open and due or has been attempted more than two days ago, it should be processed - let charges_to_do = - crate::database::models::charge_item::ChargeItem::get_chargeable(&pool).await?; - - let prices = product_item::ProductPriceItem::get_many( - &charges_to_do - .iter() - .map(|x| x.price_id) - .collect::>() - .into_iter() - .collect::>(), + info!("Indexing billing queue"); + let res = async { + // If a charge is open and due or has been attempted more than two days ago, it should be processed + let charges_to_do = + crate::database::models::charge_item::ChargeItem::get_chargeable( &pool, ) .await?; - let users = crate::database::models::User::get_many_ids( - &charges_to_do - .iter() - .map(|x| x.user_id) - .collect::>() - .into_iter() - .collect::>(), - &pool, - &redis, - ) - .await?; + let prices = product_item::ProductPriceItem::get_many( + &charges_to_do + .iter() + .map(|x| x.price_id) + .collect::>() + .into_iter() + .collect::>(), + &pool, + ) + .await?; - let mut transaction = pool.begin().await?; + let users = crate::database::models::User::get_many_ids( + &charges_to_do + .iter() + .map(|x| x.user_id) + .collect::>() + .into_iter() + .collect::>(), + &pool, + &redis, + ) + .await?; - for mut charge in charges_to_do { - let product_price = - if let Some(price) = prices.iter().find(|x| x.id == charge.price_id) { - price + let mut transaction = pool.begin().await?; + + for mut charge in charges_to_do { + let product_price = if let Some(price) = + prices.iter().find(|x| x.id == charge.price_id) + { + price + } else { + continue; + }; + + let user = if let Some(user) = + users.iter().find(|x| x.id == charge.user_id) + { + user + } else { + continue; + }; + + let price = match &product_price.prices { + Price::OneTime { price } => Some(price), + Price::Recurring { intervals } => { + if let Some(ref interval) = charge.subscription_interval { + intervals.get(interval) } else { + warn!( + "Could not find subscription for charge {:?}", + charge.id + ); continue; - }; - - let user = if let Some(user) = users.iter().find(|x| x.id == charge.user_id) { - user - } else { - continue; - }; - - let price = match &product_price.prices { - Price::OneTime { price } => Some(price), - Price::Recurring { intervals } => { - if let Some(ref interval) = charge.subscription_interval { - intervals.get(interval) - } else { - warn!("Could not find subscription for charge {:?}", charge.id); - continue; - } } - }; - - if let Some(price) = price { - let customer_id = get_or_create_customer( - user.id.into(), - user.stripe_customer_id.as_deref(), - user.email.as_deref(), - &stripe_client, - &pool, - &redis, - ) - .await?; - - let customer = - stripe::Customer::retrieve(&stripe_client, &customer_id, &[]).await?; - - let currency = - match Currency::from_str(&product_price.currency_code.to_lowercase()) { - Ok(x) => x, - Err(_) => { - warn!( - "Could not find currency for {}", - product_price.currency_code - ); - continue; - } - }; - - let mut intent = CreatePaymentIntent::new(*price as i64, currency); - - let mut metadata = HashMap::new(); - metadata.insert( - "modrinth_user_id".to_string(), - to_base62(charge.user_id.0 as u64), - ); - metadata.insert( - "modrinth_charge_id".to_string(), - to_base62(charge.id.0 as u64), - ); - metadata.insert( - "modrinth_charge_type".to_string(), - charge.type_.as_str().to_string(), - ); - - intent.metadata = Some(metadata); - intent.customer = Some(customer.id); - - if let Some(payment_method) = customer - .invoice_settings - .and_then(|x| x.default_payment_method.map(|x| x.id())) - { - intent.payment_method = Some(payment_method); - intent.confirm = Some(true); - intent.off_session = Some(PaymentIntentOffSession::Exists(true)); - - charge.status = ChargeStatus::Processing; - - stripe::PaymentIntent::create(&stripe_client, intent).await?; - } else { - charge.status = ChargeStatus::Failed; - charge.last_attempt = Some(Utc::now()); - } - - charge.upsert(&mut transaction).await?; } + }; + + if let Some(price) = price { + let customer_id = get_or_create_customer( + user.id.into(), + user.stripe_customer_id.as_deref(), + user.email.as_deref(), + &stripe_client, + &pool, + &redis, + ) + .await?; + + let customer = stripe::Customer::retrieve( + &stripe_client, + &customer_id, + &[], + ) + .await?; + + let currency = match Currency::from_str( + &product_price.currency_code.to_lowercase(), + ) { + Ok(x) => x, + Err(_) => { + warn!( + "Could not find currency for {}", + product_price.currency_code + ); + continue; + } + }; + + let mut intent = + CreatePaymentIntent::new(*price as i64, currency); + + let mut metadata = HashMap::new(); + metadata.insert( + "modrinth_user_id".to_string(), + to_base62(charge.user_id.0 as u64), + ); + metadata.insert( + "modrinth_charge_id".to_string(), + to_base62(charge.id.0 as u64), + ); + metadata.insert( + "modrinth_charge_type".to_string(), + charge.type_.as_str().to_string(), + ); + + intent.metadata = Some(metadata); + intent.customer = Some(customer.id); + + if let Some(payment_method) = customer + .invoice_settings + .and_then(|x| x.default_payment_method.map(|x| x.id())) + { + intent.payment_method = Some(payment_method); + intent.confirm = Some(true); + intent.off_session = + Some(PaymentIntentOffSession::Exists(true)); + + charge.status = ChargeStatus::Processing; + + stripe::PaymentIntent::create(&stripe_client, intent) + .await?; + } else { + charge.status = ChargeStatus::Failed; + charge.last_attempt = Some(Utc::now()); + } + + charge.upsert(&mut transaction).await?; } - - transaction.commit().await?; - - Ok::<(), ApiError>(()) - } - .await; - - if let Err(e) = res { - warn!("Error indexing billing queue: {:?}", e); } - info!("Done indexing billing queue"); + transaction.commit().await?; - tokio::time::sleep(std::time::Duration::from_secs(60 * 5)).await; + Ok::<(), ApiError>(()) } + .await; + + if let Err(e) = res { + warn!("Error indexing billing queue: {:?}", e); + } + + info!("Done indexing billing queue"); } diff --git a/apps/labrinth/src/routes/internal/statuses.rs b/apps/labrinth/src/routes/internal/statuses.rs index 0c5c08ad6..ac5721088 100644 --- a/apps/labrinth/src/routes/internal/statuses.rs +++ b/apps/labrinth/src/routes/internal/statuses.rs @@ -9,6 +9,10 @@ use crate::queue::socket::{ ActiveSocket, ActiveSockets, SocketId, TunnelSocketType, }; use crate::routes::ApiError; +use crate::sync::friends::{RedisFriendsMessage, FRIENDS_CHANNEL_NAME}; +use crate::sync::status::{ + get_user_status, push_back_user_expiry, replace_user_status, +}; use actix_web::web::{Data, Payload}; use actix_web::{get, web, HttpRequest, HttpResponse}; use actix_ws::Message; @@ -19,10 +23,15 @@ use ariadne::networking::message::{ use ariadne::users::UserStatus; use chrono::Utc; use either::Either; +use futures_util::future::select; use futures_util::{StreamExt, TryStreamExt}; +use redis::AsyncCommands; use serde::Deserialize; use sqlx::PgPool; +use std::pin::pin; use std::sync::atomic::Ordering; +use tokio::sync::oneshot::error::TryRecvError; +use tokio::time::{sleep, Duration}; pub fn config(cfg: &mut web::ServiceConfig) { cfg.service(ws_init); @@ -62,6 +71,7 @@ pub async fn ws_init( } let user = User::from_full(db_user); + let user_id = user.id; let (res, mut session, msg_stream) = match actix_ws::handle(&req, body) { Ok(x) => x, @@ -79,19 +89,32 @@ pub async fn ws_init( .await?; let friend_statuses = if !friends.is_empty() { - friends - .iter() - .filter_map(|x| { - db.get_status( - if x.user_id == user.id.into() { - x.friend_id - } else { - x.user_id + let db = db.clone(); + let redis = redis.clone(); + tokio_stream::iter(friends.iter()) + .map(|x| { + let db = db.clone(); + let redis = redis.clone(); + async move { + async move { + get_user_status( + if x.user_id == user_id.into() { + x.friend_id + } else { + x.user_id + } + .into(), + &db, + &redis, + ) + .await } - .into(), - ) + } }) + .buffer_unordered(16) + .filter_map(|x| x) .collect::>() + .await } else { Vec::new() }; @@ -116,20 +139,42 @@ pub async fn ws_init( #[cfg(debug_assertions)] tracing::info!("Connection {socket_id} opened by {}", user.id); - broadcast_friends( - user.id, - ServerToClientMessage::StatusUpdate { status }, - &pool, - &db, - Some(friends), + replace_user_status(None, Some(&status), &redis).await?; + broadcast_friends_message( + &redis, + RedisFriendsMessage::StatusUpdate { status }, ) .await?; + let (shutdown_sender, mut shutdown_receiver) = + tokio::sync::oneshot::channel::<()>(); + + { + let db = db.clone(); + let redis = redis.clone(); + actix_web::rt::spawn(async move { + while shutdown_receiver.try_recv() == Err(TryRecvError::Empty) { + sleep(Duration::from_secs(30)).await; + if let Some(socket) = db.sockets.get(&socket_id) { + let _ = socket.socket.clone().ping(&[]).await; + } + let _ = push_back_user_expiry(user_id, &redis).await; + } + }); + } + let mut stream = msg_stream.into_stream(); actix_web::rt::spawn(async move { - // receive messages from websocket - while let Some(msg) = stream.next().await { + loop { + let next = pin!(stream.next()); + let timeout = pin!(sleep(Duration::from_secs(30))); + let futures_util::future::Either::Left((Some(msg), _)) = + select(next, timeout).await + else { + break; + }; + let message = match msg { Ok(Message::Text(text)) => { ClientToServerMessage::deserialize(Either::Left(&text)) @@ -139,10 +184,7 @@ pub async fn ws_init( ClientToServerMessage::deserialize(Either::Right(&bytes)) } - Ok(Message::Close(_)) => { - let _ = close_socket(socket_id, &pool, &db).await; - continue; - } + Ok(Message::Close(_)) => break, Ok(Message::Ping(msg)) => { if let Some(socket) = db.sockets.get(&socket_id) { @@ -162,8 +204,7 @@ pub async fn ws_init( #[cfg(debug_assertions)] if !message.is_binary() { tracing::info!( - "Received message from {socket_id}: {:?}", - message + "Received message from {socket_id}: {message:?}" ); } @@ -172,6 +213,8 @@ pub async fn ws_init( if let Some(mut pair) = db.sockets.get_mut(&socket_id) { let ActiveSocket { status, .. } = pair.value_mut(); + let old_status = status.clone(); + if status .profile_name .as_ref() @@ -188,14 +231,17 @@ pub async fn ws_init( // We drop the pair to avoid holding the lock for too long drop(pair); - let _ = broadcast_friends( - user.id, - ServerToClientMessage::StatusUpdate { + let _ = replace_user_status( + Some(&old_status), + Some(&user_status), + &redis, + ) + .await; + let _ = broadcast_friends_message( + &redis, + RedisFriendsMessage::StatusUpdate { status: user_status, }, - &pool, - &db, - None, ) .await; } @@ -247,12 +293,11 @@ pub async fn ws_init( }; match tunnel_socket.socket_type { TunnelSocketType::Listening => { - let _ = broadcast_friends( + let _ = broadcast_to_local_friends( user.id, ServerToClientMessage::FriendSocketStoppedListening { user: user.id }, &pool, &db, - None, ) .await; } @@ -308,25 +353,48 @@ pub async fn ws_init( } } - let _ = close_socket(socket_id, &pool, &db).await; + let _ = shutdown_sender.send(()); + let _ = close_socket(socket_id, &pool, &db, &redis).await; }); Ok(res) } -pub async fn broadcast_friends( +pub async fn broadcast_friends_message( + redis: &RedisPool, + message: RedisFriendsMessage, +) -> Result<(), crate::database::models::DatabaseError> { + let _: () = redis + .pool + .get() + .await? + .publish(FRIENDS_CHANNEL_NAME, message) + .await?; + Ok(()) +} + +pub async fn broadcast_to_local_friends( user_id: UserId, message: ServerToClientMessage, pool: &PgPool, sockets: &ActiveSockets, - friends: Option>, +) -> Result<(), crate::database::models::DatabaseError> { + broadcast_to_known_local_friends( + user_id, + message, + sockets, + FriendItem::get_user_friends(user_id.into(), Some(true), pool).await?, + ) + .await +} + +async fn broadcast_to_known_local_friends( + user_id: UserId, + message: ServerToClientMessage, + sockets: &ActiveSockets, + friends: Vec, ) -> Result<(), crate::database::models::DatabaseError> { // FIXME Probably shouldn't be using database errors for this. Maybe ApiError? - let friends = if let Some(friends) = friends { - friends - } else { - FriendItem::get_user_friends(user_id.into(), Some(true), pool).await? - }; for friend in friends { let friend_id = if friend.user_id == user_id.into() { @@ -387,6 +455,7 @@ pub async fn close_socket( id: SocketId, pool: &PgPool, db: &ActiveSockets, + redis: &RedisPool, ) -> Result<(), crate::database::models::DatabaseError> { if let Some((_, socket)) = db.sockets.remove(&id) { let user_id = socket.status.user_id; @@ -397,12 +466,10 @@ pub async fn close_socket( let _ = socket.socket.close(None).await; - broadcast_friends( - user_id, - ServerToClientMessage::UserOffline { id: user_id }, - pool, - db, - None, + replace_user_status(Some(&socket.status), None, redis).await?; + broadcast_friends_message( + redis, + RedisFriendsMessage::UserOffline { user: user_id }, ) .await?; @@ -414,14 +481,13 @@ pub async fn close_socket( }; match tunnel_socket.socket_type { TunnelSocketType::Listening => { - let _ = broadcast_friends( + let _ = broadcast_to_local_friends( user_id, ServerToClientMessage::SocketClosed { socket: owned_socket, }, pool, db, - None, ) .await; } diff --git a/apps/labrinth/src/routes/mod.rs b/apps/labrinth/src/routes/mod.rs index 39337d5f3..131bddabc 100644 --- a/apps/labrinth/src/routes/mod.rs +++ b/apps/labrinth/src/routes/mod.rs @@ -95,6 +95,8 @@ pub enum ApiError { Database(#[from] crate::database::models::DatabaseError), #[error("Database Error: {0}")] SqlxDatabase(#[from] sqlx::Error), + #[error("Database Error: {0}")] + RedisDatabase(#[from] redis::RedisError), #[error("Clickhouse Error: {0}")] Clickhouse(#[from] clickhouse::error::Error), #[error("Internal server error: {0}")] @@ -148,8 +150,9 @@ impl ApiError { crate::models::error::ApiError { error: match self { ApiError::Env(..) => "environment_error", - ApiError::SqlxDatabase(..) => "database_error", ApiError::Database(..) => "database_error", + ApiError::SqlxDatabase(..) => "database_error", + ApiError::RedisDatabase(..) => "database_error", ApiError::Authentication(..) => "unauthorized", ApiError::CustomAuthentication(..) => "unauthorized", ApiError::Xml(..) => "xml_error", @@ -186,6 +189,7 @@ impl actix_web::ResponseError for ApiError { ApiError::Env(..) => StatusCode::INTERNAL_SERVER_ERROR, ApiError::Database(..) => StatusCode::INTERNAL_SERVER_ERROR, ApiError::SqlxDatabase(..) => StatusCode::INTERNAL_SERVER_ERROR, + ApiError::RedisDatabase(..) => StatusCode::INTERNAL_SERVER_ERROR, ApiError::Clickhouse(..) => StatusCode::INTERNAL_SERVER_ERROR, ApiError::Authentication(..) => StatusCode::UNAUTHORIZED, ApiError::CustomAuthentication(..) => StatusCode::UNAUTHORIZED, diff --git a/apps/labrinth/src/routes/v3/friends.rs b/apps/labrinth/src/routes/v3/friends.rs index 1f7453236..56d70d568 100644 --- a/apps/labrinth/src/routes/v3/friends.rs +++ b/apps/labrinth/src/routes/v3/friends.rs @@ -5,8 +5,12 @@ use crate::models::pats::Scopes; use crate::models::users::UserFriend; use crate::queue::session::AuthQueue; use crate::queue::socket::ActiveSockets; -use crate::routes::internal::statuses::send_message_to_user; +use crate::routes::internal::statuses::{ + broadcast_friends_message, send_message_to_user, +}; use crate::routes::ApiError; +use crate::sync::friends::RedisFriendsMessage; +use crate::sync::status::get_user_status; use actix_web::{delete, get, post, web, HttpRequest, HttpResponse}; use ariadne::networking::message::ServerToClientMessage; use chrono::Utc; @@ -76,14 +80,16 @@ pub async fn add_friend( user_id: UserId, friend_id: UserId, sockets: &ActiveSockets, + redis: &RedisPool, ) -> Result<(), ApiError> { - if let Some(friend_status) = sockets.get_status(user_id.into()) + if let Some(friend_status) = + get_user_status(user_id.into(), sockets, redis).await { - send_message_to_user( - sockets, - friend_id.into(), - &ServerToClientMessage::StatusUpdate { - status: friend_status.clone(), + broadcast_friends_message( + redis, + RedisFriendsMessage::DirectStatusUpdate { + to_user: friend_id.into(), + status: friend_status, }, ) .await?; @@ -92,8 +98,10 @@ pub async fn add_friend( Ok(()) } - send_friend_status(friend.user_id, friend.friend_id, &db).await?; - send_friend_status(friend.friend_id, friend.user_id, &db).await?; + send_friend_status(friend.user_id, friend.friend_id, &db, &redis) + .await?; + send_friend_status(friend.friend_id, friend.user_id, &db, &redis) + .await?; } else { if friend.id == user.id.into() { return Err(ApiError::InvalidInput( diff --git a/apps/labrinth/src/scheduler.rs b/apps/labrinth/src/scheduler.rs index f94540392..82213effd 100644 --- a/apps/labrinth/src/scheduler.rs +++ b/apps/labrinth/src/scheduler.rs @@ -36,181 +36,4 @@ impl Drop for Scheduler { } } -use tracing::{info, warn}; - -pub fn schedule_versions( - scheduler: &mut Scheduler, - pool: sqlx::Pool, - redis: RedisPool, -) { - let version_index_interval = std::time::Duration::from_secs( - parse_var("VERSION_INDEX_INTERVAL").unwrap_or(1800), - ); - - scheduler.run(version_index_interval, move || { - let pool_ref = pool.clone(); - let redis = redis.clone(); - async move { - info!("Indexing game versions list from Mojang"); - let result = update_versions(&pool_ref, &redis).await; - if let Err(e) = result { - warn!("Version update failed: {}", e); - } - info!("Done indexing game versions"); - } - }); -} - -use thiserror::Error; - -#[derive(Error, Debug)] -pub enum VersionIndexingError { - #[error("Network error while updating game versions list: {0}")] - NetworkError(#[from] reqwest::Error), - #[error("Database error while updating game versions list: {0}")] - DatabaseError(#[from] crate::database::models::DatabaseError), -} - -use crate::{ - database::{ - models::legacy_loader_fields::MinecraftGameVersion, redis::RedisPool, - }, - util::env::parse_var, -}; -use chrono::{DateTime, Utc}; -use serde::Deserialize; use tokio_stream::wrappers::IntervalStream; - -#[derive(Deserialize)] -struct InputFormat<'a> { - // latest: LatestFormat, - versions: Vec>, -} -#[derive(Deserialize)] -struct VersionFormat<'a> { - id: String, - #[serde(rename = "type")] - type_: std::borrow::Cow<'a, str>, - #[serde(rename = "releaseTime")] - release_time: DateTime, -} - -async fn update_versions( - pool: &sqlx::Pool, - redis: &RedisPool, -) -> Result<(), VersionIndexingError> { - let input = reqwest::get( - "https://piston-meta.mojang.com/mc/game/version_manifest_v2.json", - ) - .await? - .json::() - .await?; - - let mut skipped_versions_count = 0u32; - - // A list of version names that contains spaces. - // Generated using the command - // ```sh - // curl https://launchermeta.mojang.com/mc/game/version_manifest.json \ - // | jq '[.versions[].id | select(contains(" "))]' - // ``` - const HALL_OF_SHAME: [(&str, &str); 12] = [ - ("1.14.2 Pre-Release 4", "1.14.2-pre4"), - ("1.14.2 Pre-Release 3", "1.14.2-pre3"), - ("1.14.2 Pre-Release 2", "1.14.2-pre2"), - ("1.14.2 Pre-Release 1", "1.14.2-pre1"), - ("1.14.1 Pre-Release 2", "1.14.1-pre2"), - ("1.14.1 Pre-Release 1", "1.14.1-pre1"), - ("1.14 Pre-Release 5", "1.14-pre5"), - ("1.14 Pre-Release 4", "1.14-pre4"), - ("1.14 Pre-Release 3", "1.14-pre3"), - ("1.14 Pre-Release 2", "1.14-pre2"), - ("1.14 Pre-Release 1", "1.14-pre1"), - ("3D Shareware v1.34", "3D-Shareware-v1.34"), - ]; - - lazy_static::lazy_static! { - /// Mojank for some reason has versions released at the same DateTime. This hardcodes them to fix this, - /// as most of our ordering logic is with DateTime - static ref HALL_OF_SHAME_2: [(&'static str, chrono::DateTime); 4] = [ - ( - "1.4.5", - chrono::DateTime::parse_from_rfc3339("2012-12-19T22:00:00+00:00") - .unwrap() - .into(), - ), - ( - "1.4.6", - chrono::DateTime::parse_from_rfc3339("2012-12-19T22:00:01+00:00") - .unwrap() - .into(), - ), - ( - "1.6.3", - chrono::DateTime::parse_from_rfc3339("2013-09-13T10:54:41+00:00") - .unwrap() - .into(), - ), - ( - "13w37b", - chrono::DateTime::parse_from_rfc3339("2013-09-13T10:54:42+00:00") - .unwrap() - .into(), - ), - ]; - } - - for version in input.versions.into_iter() { - let mut name = version.id; - if !name - .chars() - .all(|c| c.is_ascii_alphanumeric() || "-_.".contains(c)) - { - if let Some((_, alternate)) = - HALL_OF_SHAME.iter().find(|(version, _)| name == *version) - { - name = String::from(*alternate); - } else { - // We'll deal with these manually - skipped_versions_count += 1; - continue; - } - } - - let type_ = match &*version.type_ { - "release" => "release", - "snapshot" => "snapshot", - "old_alpha" => "alpha", - "old_beta" => "beta", - _ => "other", - }; - - MinecraftGameVersion::builder() - .version(&name)? - .version_type(type_)? - .created( - if let Some((_, alternate)) = - HALL_OF_SHAME_2.iter().find(|(version, _)| name == *version) - { - alternate - } else { - &version.release_time - }, - ) - .insert(pool, redis) - .await?; - } - - if skipped_versions_count > 0 { - // This will currently always trigger due to 1.14 pre releases - // and the shareware april fools update. We could set a threshold - // that accounts for those versions and update it whenever we - // manually fix another version. - warn!( - "Skipped {} game versions; check for new versions and add them manually", - skipped_versions_count - ); - } - - Ok(()) -} diff --git a/apps/labrinth/src/sync/friends.rs b/apps/labrinth/src/sync/friends.rs new file mode 100644 index 000000000..77f407da3 --- /dev/null +++ b/apps/labrinth/src/sync/friends.rs @@ -0,0 +1,87 @@ +use crate::queue::socket::ActiveSockets; +use crate::routes::internal::statuses::{ + broadcast_to_local_friends, send_message_to_user, +}; +use actix_web::web::Data; +use ariadne::ids::UserId; +use ariadne::networking::message::ServerToClientMessage; +use ariadne::users::UserStatus; +use redis::aio::PubSub; +use redis::{RedisWrite, ToRedisArgs}; +use serde::{Deserialize, Serialize}; +use sqlx::PgPool; +use tokio_stream::StreamExt; + +pub const FRIENDS_CHANNEL_NAME: &str = "friends"; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +pub enum RedisFriendsMessage { + StatusUpdate { status: UserStatus }, + UserOffline { user: UserId }, + DirectStatusUpdate { to_user: UserId, status: UserStatus }, +} + +impl ToRedisArgs for RedisFriendsMessage { + fn write_redis_args(&self, out: &mut W) + where + W: ?Sized + RedisWrite, + { + out.write_arg(&serde_json::to_vec(&self).unwrap()) + } +} + +pub async fn handle_pubsub( + mut pubsub: PubSub, + pool: PgPool, + sockets: Data, +) { + pubsub.subscribe(FRIENDS_CHANNEL_NAME).await.unwrap(); + let mut stream = pubsub.into_on_message(); + while let Some(message) = stream.next().await { + if message.get_channel_name() != FRIENDS_CHANNEL_NAME { + continue; + } + let payload = serde_json::from_slice(message.get_payload_bytes()); + + let pool = pool.clone(); + let sockets = sockets.clone(); + actix_rt::spawn(async move { + match payload { + Ok(RedisFriendsMessage::StatusUpdate { status }) => { + let _ = broadcast_to_local_friends( + status.user_id, + ServerToClientMessage::StatusUpdate { status }, + &pool, + &sockets, + ) + .await; + } + + Ok(RedisFriendsMessage::UserOffline { user }) => { + let _ = broadcast_to_local_friends( + user, + ServerToClientMessage::UserOffline { id: user }, + &pool, + &sockets, + ) + .await; + } + + Ok(RedisFriendsMessage::DirectStatusUpdate { + to_user, + status, + }) => { + let _ = send_message_to_user( + &sockets, + to_user, + &ServerToClientMessage::StatusUpdate { status }, + ) + .await; + } + + Err(_) => {} + } + }); + } +} diff --git a/apps/labrinth/src/sync/mod.rs b/apps/labrinth/src/sync/mod.rs new file mode 100644 index 000000000..e7a3cc811 --- /dev/null +++ b/apps/labrinth/src/sync/mod.rs @@ -0,0 +1,2 @@ +pub mod friends; +pub mod status; diff --git a/apps/labrinth/src/sync/status.rs b/apps/labrinth/src/sync/status.rs new file mode 100644 index 000000000..b475e28b4 --- /dev/null +++ b/apps/labrinth/src/sync/status.rs @@ -0,0 +1,71 @@ +use crate::database::redis::RedisPool; +use crate::queue::socket::ActiveSockets; +use ariadne::ids::UserId; +use ariadne::users::UserStatus; +use redis::AsyncCommands; + +const EXPIRY_TIME_SECONDS: i64 = 60; + +pub async fn get_user_status( + user: UserId, + local_sockets: &ActiveSockets, + redis: &RedisPool, +) -> Option { + if let Some(friend_status) = local_sockets.get_status(user) { + return Some(friend_status); + } + + if let Ok(mut conn) = redis.pool.get().await { + if let Ok(mut statuses) = + conn.sscan::<_, String>(get_field_name(user)).await + { + if let Some(status_json) = statuses.next_item().await { + return serde_json::from_str::(&status_json).ok(); + } + } + } + + None +} + +pub async fn replace_user_status( + old_status: Option<&UserStatus>, + new_status: Option<&UserStatus>, + redis: &RedisPool, +) -> Result<(), redis::RedisError> { + let Some(user) = new_status.or(old_status).map(|x| x.user_id) else { + return Ok(()); + }; + + if let Ok(mut conn) = redis.pool.get().await { + let field_name = get_field_name(user); + let mut pipe = redis::pipe(); + pipe.atomic(); + if let Some(status) = old_status { + pipe.srem(&field_name, serde_json::to_string(&status).unwrap()) + .ignore(); + } + if let Some(status) = new_status { + pipe.sadd(&field_name, serde_json::to_string(&status).unwrap()) + .ignore(); + pipe.expire(&field_name, EXPIRY_TIME_SECONDS).ignore(); + } + return pipe.query_async(&mut conn).await; + } + + Ok(()) +} + +pub async fn push_back_user_expiry( + user: UserId, + redis: &RedisPool, +) -> Result<(), redis::RedisError> { + if let Ok(mut conn) = redis.pool.get().await { + return conn.expire(get_field_name(user), EXPIRY_TIME_SECONDS).await; + } + Ok(()) +} + +fn get_field_name(user: UserId) -> String { + format!("user_status:{}", user) +} diff --git a/apps/labrinth/tests/common/mod.rs b/apps/labrinth/tests/common/mod.rs index 840bad667..75d0ea7d7 100644 --- a/apps/labrinth/tests/common/mod.rs +++ b/apps/labrinth/tests/common/mod.rs @@ -35,6 +35,9 @@ pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig { let maxmind_reader = Arc::new(queue::maxmind::MaxMindIndexer::new().await.unwrap()); + let stripe_client = + stripe::Client::new(dotenvy::var("STRIPE_API_KEY").unwrap()); + labrinth::app_setup( pool.clone(), redis_pool.clone(), @@ -42,6 +45,8 @@ pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig { &mut clickhouse, file_host.clone(), maxmind_reader, + stripe_client, + false, ) } From d49cc87b8c000d0db8b0a0e8f02056df391931e8 Mon Sep 17 00:00:00 2001 From: Jai A Date: Sat, 15 Mar 2025 08:32:45 -0700 Subject: [PATCH 19/36] only run migrations on prod instances --- apps/labrinth/src/main.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/labrinth/src/main.rs b/apps/labrinth/src/main.rs index f45b3609c..d114002ba 100644 --- a/apps/labrinth/src/main.rs +++ b/apps/labrinth/src/main.rs @@ -66,11 +66,11 @@ async fn main() -> std::io::Result<()> { "Starting Labrinth on {}", dotenvy::var("BIND_ADDR").unwrap() ); - } - database::check_for_migrations() - .await - .expect("An error occurred while running migrations."); + database::check_for_migrations() + .await + .expect("An error occurred while running migrations."); + } // Database Connector let pool = database::connect() From 16893ec0e39d0322310d73626b15d2771bde9ac9 Mon Sep 17 00:00:00 2001 From: Prospector <6166773+Prospector@users.noreply.github.com> Date: Tue, 18 Mar 2025 18:24:14 -0700 Subject: [PATCH 20/36] Fix display of critical announcements in app (#3407) --- apps/app-frontend/src/App.vue | 10 ++++------ packages/ui/src/components/base/Admonition.vue | 8 ++++---- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/apps/app-frontend/src/App.vue b/apps/app-frontend/src/App.vue index 412ce1d3a..00c82c0cf 100644 --- a/apps/app-frontend/src/App.vue +++ b/apps/app-frontend/src/App.vue @@ -521,6 +521,10 @@ function handleAuxClick(e) { width: 'calc(100% - var(--right-bar-width))', }" > +
+

{{ criticalErrorMessage.header }}

+
+
-
-
-

{{ criticalErrorMessage.header }}

-
-
-
diff --git a/packages/ui/src/components/base/Admonition.vue b/packages/ui/src/components/base/Admonition.vue index fc9f691f6..538bb9d84 100644 --- a/packages/ui/src/components/base/Admonition.vue +++ b/packages/ui/src/components/base/Admonition.vue @@ -39,14 +39,14 @@ defineProps({ }) const typeClasses = { - info: 'border-blue bg-bg-blue', - warning: 'border-orange bg-bg-orange', + info: 'border-brand-blue bg-bg-blue', + warning: 'border-brand-orange bg-bg-orange', critical: 'border-brand-red bg-bg-red', } const iconClasses = { - info: 'text-blue', - warning: 'text-orange', + info: 'text-brand-blue', + warning: 'text-brand-orange', critical: 'text-brand-red', } From 77021d2af855c8f1126ccd0ca8f47c45851bd715 Mon Sep 17 00:00:00 2001 From: Prospector <6166773+Prospector@users.noreply.github.com> Date: Tue, 18 Mar 2025 18:26:57 -0700 Subject: [PATCH 21/36] Handle downtime errors, give more information on error pages. (#3402) Co-authored-by: Jai Agrawal <18202329+Geometrically@users.noreply.github.com> --- apps/frontend/nuxt.config.ts | 38 +- apps/frontend/src/assets/styles/layout.scss | 1 + apps/frontend/src/error.vue | 375 ++++++++++++++++++-- apps/frontend/src/layouts/default.vue | 34 +- apps/frontend/src/locales/en-US/index.json | 6 + apps/frontend/src/pages/[type]/[id].vue | 39 +- apps/frontend/src/pages/index.vue | 29 +- 7 files changed, 464 insertions(+), 58 deletions(-) diff --git a/apps/frontend/nuxt.config.ts b/apps/frontend/nuxt.config.ts index 6f9418832..354bff294 100644 --- a/apps/frontend/nuxt.config.ts +++ b/apps/frontend/nuxt.config.ts @@ -126,6 +126,7 @@ export default defineNuxtConfig({ homePageSearch?: any[]; homePageNotifs?: any[]; products?: any[]; + errors?: number[]; } = {}; try { @@ -157,6 +158,14 @@ export default defineNuxtConfig({ }, }; + const caughtErrorCodes = new Set(); + + function handleFetchError(err: any, defaultValue: any) { + console.error("Error generating state: ", err); + caughtErrorCodes.add(err.status); + return defaultValue; + } + const [ categories, loaders, @@ -168,15 +177,25 @@ export default defineNuxtConfig({ homePageNotifs, products, ] = await Promise.all([ - $fetch(`${API_URL}tag/category`, headers), - $fetch(`${API_URL}tag/loader`, headers), - $fetch(`${API_URL}tag/game_version`, headers), - $fetch(`${API_URL}tag/donation_platform`, headers), - $fetch(`${API_URL}tag/report_type`, headers), - $fetch(`${API_URL}projects_random?count=60`, headers), - $fetch(`${API_URL}search?limit=3&query=leave&index=relevance`, headers), - $fetch(`${API_URL}search?limit=3&query=&index=updated`, headers), - $fetch(`${API_URL.replace("/v2/", "/_internal/")}billing/products`, headers), + $fetch(`${API_URL}tag/category`, headers).catch((err) => handleFetchError(err, [])), + $fetch(`${API_URL}tag/loader`, headers).catch((err) => handleFetchError(err, [])), + $fetch(`${API_URL}tag/game_version`, headers).catch((err) => handleFetchError(err, [])), + $fetch(`${API_URL}tag/donation_platform`, headers).catch((err) => + handleFetchError(err, []), + ), + $fetch(`${API_URL}tag/report_type`, headers).catch((err) => handleFetchError(err, [])), + $fetch(`${API_URL}projects_random?count=60`, headers).catch((err) => + handleFetchError(err, []), + ), + $fetch(`${API_URL}search?limit=3&query=leave&index=relevance`, headers).catch((err) => + handleFetchError(err, {}), + ), + $fetch(`${API_URL}search?limit=3&query=&index=updated`, headers).catch((err) => + handleFetchError(err, {}), + ), + $fetch(`${API_URL.replace("/v2/", "/_internal/")}billing/products`, headers).catch((err) => + handleFetchError(err, []), + ), ]); state.categories = categories; @@ -188,6 +207,7 @@ export default defineNuxtConfig({ state.homePageSearch = homePageSearch; state.homePageNotifs = homePageNotifs; state.products = products; + state.errors = [...caughtErrorCodes]; await fs.writeFile("./src/generated/state.json", JSON.stringify(state)); diff --git a/apps/frontend/src/assets/styles/layout.scss b/apps/frontend/src/assets/styles/layout.scss index 41be6710e..60e05cfdd 100644 --- a/apps/frontend/src/assets/styles/layout.scss +++ b/apps/frontend/src/assets/styles/layout.scss @@ -126,6 +126,7 @@ max-width: 80rem; column-gap: 0.75rem; padding: 0 1.5rem; + padding-bottom: 1.5rem; grid-template: "header" diff --git a/apps/frontend/src/error.vue b/apps/frontend/src/error.vue index 20cca3e9a..be5cee6ea 100644 --- a/apps/frontend/src/error.vue +++ b/apps/frontend/src/error.vue @@ -1,21 +1,52 @@ diff --git a/apps/frontend/src/layouts/default.vue b/apps/frontend/src/layouts/default.vue index 1e2297b69..118074246 100644 --- a/apps/frontend/src/layouts/default.vue +++ b/apps/frontend/src/layouts/default.vue @@ -80,6 +80,23 @@ /> +
+
+
+
+ {{ + formatMessage(failedToBuildBannerMessages.description, { + errors: generatedStateErrors, + url: config.public.apiBaseUrl, + }) + }} +
+
@@ -538,7 +555,7 @@