Use new MaxMind env vars on Labrinth (#4573)

* Bring in modrinth-maxmind

* integrate modrinth-maxmind into labrinth

* Fix CI

apps/labrinth/.env.docker-compose

@@ -114,6 +114,7 @@ CLICKHOUSE_USER=default
 CLICKHOUSE_PASSWORD=default
 CLICKHOUSE_DATABASE=staging_ariadne
 
+MAXMIND_ACCOUNT_ID=none
 MAXMIND_LICENSE_KEY=none
 
 FLAME_ANVIL_URL=none

apps/labrinth/.env.local

@@ -115,6 +115,7 @@ CLICKHOUSE_USER=default
 CLICKHOUSE_PASSWORD=default
 CLICKHOUSE_DATABASE=staging_ariadne
 
+MAXMIND_ACCOUNT_ID=none
 MAXMIND_LICENSE_KEY=none
 
 FLAME_ANVIL_URL=none

apps/labrinth/Cargo.toml

@@ -42,7 +42,6 @@ deadpool-redis.workspace = true
 dotenvy = { workspace = true }
 either = { workspace = true }
 eyre = { workspace = true }
-flate2 = { workspace = true }
 futures = { workspace = true }
 futures-util = { workspace = true }
 hex = { workspace = true }
@@ -69,8 +68,8 @@ image = { workspace = true, features = [
 itertools = { workspace = true }
 json-patch = { workspace = true }
 lettre = { workspace = true }
-maxminddb = { workspace = true }
 meilisearch-sdk = { workspace = true, features = ["reqwest"] }
+modrinth-maxmind = { workspace = true }
 murmur2 = { workspace = true }
 paste = { workspace = true }
 path-util = { workspace = true }
@@ -111,7 +110,6 @@ sqlx = { workspace = true, features = [
   "rust_decimal",
   "tls-rustls-aws-lc-rs",
 ] }
-tar = { workspace = true }
 thiserror = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "sync"] }
 tokio-stream = { workspace = true }

apps/labrinth/src/lib.rs

@@ -3,6 +3,7 @@ use std::time::Duration;
 
 use actix_web::web;
 use database::redis::RedisPool;
+use modrinth_maxmind::MaxMind;
 use queue::{
     analytics::AnalyticsQueue, email::EmailQueue, payouts::PayoutsQueue,
     session::AuthQueue, socket::ActiveSockets,
@@ -49,7 +50,7 @@ pub struct LabrinthConfig {
     pub redis_pool: RedisPool,
     pub clickhouse: Client,
     pub file_host: Arc<dyn file_hosting::FileHost + Send + Sync>,
-    pub maxmind: Arc<queue::maxmind::MaxMindIndexer>,
+    pub maxmind: web::Data<MaxMind>,
     pub scheduler: Arc<scheduler::Scheduler>,
     pub ip_salt: Pepper,
     pub search_config: search::SearchConfig,
@@ -72,7 +73,7 @@ pub fn app_setup(
     search_config: search::SearchConfig,
     clickhouse: &mut Client,
     file_host: Arc<dyn file_hosting::FileHost + Send + Sync>,
-    maxmind: Arc<queue::maxmind::MaxMindIndexer>,
+    maxmind: MaxMind,
     stripe_client: stripe::Client,
     anrok_client: anrok::Client,
     email_queue: EmailQueue,
@@ -218,27 +219,6 @@ pub fn app_setup(
         }
     });
 
-    let reader = maxmind.clone();
-    {
-        let reader_ref = reader;
-        scheduler.run(Duration::from_secs(60 * 60 * 24), move || {
-            let reader_ref = reader_ref.clone();
-
-            async move {
-                info!("Downloading MaxMind GeoLite2 country database");
-                let result = reader_ref.index().await;
-                if let Err(e) = result {
-                    warn!(
-                        "Downloading MaxMind GeoLite2 country database failed: {:?}",
-                        e
-                    );
-                }
-                info!("Done downloading MaxMind GeoLite2 country database");
-            }
-        });
-    }
-    info!("Downloading MaxMind GeoLite2 country database");
-
     let analytics_queue = Arc::new(AnalyticsQueue::new());
     {
         let client_ref = clickhouse.clone();
@@ -287,7 +267,7 @@ pub fn app_setup(
         redis_pool,
         clickhouse: clickhouse.clone(),
         file_host,
-        maxmind,
+        maxmind: web::Data::new(maxmind),
         scheduler: Arc::new(scheduler),
         ip_salt,
         search_config,

apps/labrinth/src/main.rs

@@ -13,7 +13,7 @@ use labrinth::search;
 use labrinth::util::anrok;
 use labrinth::util::env::parse_var;
 use labrinth::util::ratelimit::rate_limit_middleware;
-use labrinth::{check_env_vars, clickhouse, database, file_hosting, queue};
+use labrinth::{check_env_vars, clickhouse, database, file_hosting};
 use std::ffi::CStr;
 use std::str::FromStr;
 use std::sync::Arc;
@@ -215,8 +215,7 @@ async fn main() -> std::io::Result<()> {
         return Ok(());
     }
 
-    let maxmind_reader =
-        Arc::new(queue::maxmind::MaxMindIndexer::new().await.unwrap());
+    let maxmind_reader = modrinth_maxmind::MaxMind::new().await;
 
     let prometheus = PrometheusMetricsBuilder::new("labrinth")
         .endpoint("/metrics")

apps/labrinth/src/queue/maxmind.rs

@@ -1,87 +1,23 @@
-use flate2::read::GzDecoder;
-use maxminddb::geoip2::Country;
-use std::io::{Cursor, Read};
+use modrinth_maxmind::{MaxMind, geoip2};
 use std::net::Ipv6Addr;
-use tar::Archive;
-use tokio::sync::RwLock;
-use tracing::warn;
 
 pub struct MaxMindIndexer {
-    pub reader: RwLock<Option<maxminddb::Reader<Vec<u8>>>>,
+    pub maxmind: MaxMind,
 }
 
 impl MaxMindIndexer {
-    pub async fn new() -> Result<Self, reqwest::Error> {
-        let reader = MaxMindIndexer::inner_index(false).await.ok().flatten();
-
-        Ok(MaxMindIndexer {
-            reader: RwLock::new(reader),
-        })
-    }
-
-    pub async fn index(&self) -> Result<(), reqwest::Error> {
-        let reader = MaxMindIndexer::inner_index(false).await?;
-
-        if let Some(reader) = reader {
-            let mut reader_new = self.reader.write().await;
-            *reader_new = Some(reader);
-        }
-
-        Ok(())
-    }
-
-    async fn inner_index(
-        should_panic: bool,
-    ) -> Result<Option<maxminddb::Reader<Vec<u8>>>, reqwest::Error> {
-        let response = reqwest::get(
-            format!(
-                "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country&license_key={}&suffix=tar.gz",
-                dotenvy::var("MAXMIND_LICENSE_KEY").unwrap()
-            )
-        ).await?.bytes().await.unwrap().to_vec();
-
-        let tarfile = GzDecoder::new(Cursor::new(response));
-        let mut archive = Archive::new(tarfile);
-
-        if let Ok(entries) = archive.entries() {
-            for mut file in entries.flatten() {
-                if let Ok(path) = file.header().path()
-                    && path.extension().and_then(|x| x.to_str()) == Some("mmdb")
-                {
-                    let mut buf = Vec::new();
-                    file.read_to_end(&mut buf).unwrap();
-
-                    let reader = maxminddb::Reader::from_source(buf).unwrap();
-
-                    return Ok(Some(reader));
-                }
-            }
-        }
-
-        if should_panic {
-            panic!(
-                "Unable to download maxmind database- did you get a license key?"
-            )
-        } else {
-            warn!("Unable to download maxmind database.");
-
-            Ok(None)
+    pub async fn new() -> Self {
+        Self {
+            maxmind: MaxMind::new().await,
         }
     }
 
     pub async fn query(&self, ip: Ipv6Addr) -> Option<String> {
-        let maxmind = self.reader.read().await;
-
-        if let Some(ref maxmind) = *maxmind {
-            maxmind
-                .lookup::<Country>(ip.into())
-                .ok()
-                .flatten()
-                .and_then(|x| {
-                    x.country.and_then(|x| x.iso_code.map(|x| x.to_string()))
-                })
-        } else {
-            None
-        }
+        let reader = self.maxmind.reader.as_ref()?;
+        reader
+            .lookup::<geoip2::Country>(ip.into())
+            .ok()?
+            .and_then(|c| c.country)
+            .and_then(|c| c.iso_code.map(|s| s.to_string()))
     }
 }

apps/labrinth/tests/common/mod.rs

@@ -1,7 +1,8 @@
 use labrinth::queue::email::EmailQueue;
 use labrinth::util::anrok;
-use labrinth::{LabrinthConfig, file_hosting, queue};
+use labrinth::{LabrinthConfig, file_hosting};
 use labrinth::{check_env_vars, clickhouse};
+use modrinth_maxmind::MaxMind;
 use std::sync::Arc;
 
 pub mod api_common;
@@ -37,8 +38,7 @@ pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig {
         Arc::new(file_hosting::MockHost::new());
     let mut clickhouse = clickhouse::init_client().await.unwrap();
 
-    let maxmind_reader =
-        Arc::new(queue::maxmind::MaxMindIndexer::new().await.unwrap());
+    let maxmind_reader = MaxMind::new().await;
 
     let stripe_client =
         stripe::Client::new(dotenvy::var("STRIPE_API_KEY").unwrap());