From 67e1743d6cdf686d7d7fc1fdf4edf7f64a8bb3f2 Mon Sep 17 00:00:00 2001 From: aecsocket Date: Fri, 29 May 2026 22:22:08 +0100 Subject: [PATCH] Flatten facets response format, remove detailed route (#6244) * Flatten facets response format * delete test file --- ...a1371fb63be744f1d87c68229342ffadbe998.json | 22 -- .../routes/v3/analytics_get/facets/dynamic.rs | 239 ------------------ .../routes/v3/analytics_get/facets/fixed.rs | 44 +--- .../src/routes/v3/analytics_get/facets/mod.rs | 60 ++--- .../routes/v3/analytics_get/metrics/mod.rs | 3 +- .../src/routes/v3/analytics_get/mod.rs | 1 + 6 files changed, 33 insertions(+), 336 deletions(-) delete mode 100644 apps/labrinth/.sqlx/query-12fa322e09465aab925ac33f8b2a1371fb63be744f1d87c68229342ffadbe998.json delete mode 100644 apps/labrinth/src/routes/v3/analytics_get/facets/dynamic.rs diff --git a/apps/labrinth/.sqlx/query-12fa322e09465aab925ac33f8b2a1371fb63be744f1d87c68229342ffadbe998.json b/apps/labrinth/.sqlx/query-12fa322e09465aab925ac33f8b2a1371fb63be744f1d87c68229342ffadbe998.json deleted file mode 100644 index 83a7b34b0..000000000 --- a/apps/labrinth/.sqlx/query-12fa322e09465aab925ac33f8b2a1371fb63be744f1d87c68229342ffadbe998.json +++ /dev/null @@ -1,22 +0,0 @@ -{ - "db_name": "PostgreSQL", - "query": "\n SELECT id\n FROM versions\n WHERE mod_id = ANY($1)\n ", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int8" - } - ], - "parameters": { - "Left": [ - "Int8Array" - ] - }, - "nullable": [ - false - ] - }, - "hash": "12fa322e09465aab925ac33f8b2a1371fb63be744f1d87c68229342ffadbe998" -} diff --git a/apps/labrinth/src/routes/v3/analytics_get/facets/dynamic.rs b/apps/labrinth/src/routes/v3/analytics_get/facets/dynamic.rs deleted file mode 100644 index 7454df931..000000000 --- a/apps/labrinth/src/routes/v3/analytics_get/facets/dynamic.rs +++ /dev/null @@ -1,239 +0,0 @@ -use std::collections::HashMap; - -use super::{ - AnalyticsFacets, FacetValue, ProjectDownloadsFacets, ProjectPlaytimeFacets, - ProjectViewsFacets, -}; -use crate::{ - database::{ - PgPool, - models::{DBProjectId, DBUser}, - redis::RedisPool, - }, - models::users::User, - routes::ApiError, -}; - -const FACET_LIMIT: u64 = 100; - -#[derive(Debug, clickhouse::Row, serde::Deserialize)] -struct StringFacetRow { - value: String, - count: u64, -} - -pub async fn fetch( - req: super::super::GetRequest, - user: &User, - pool: &PgPool, - redis: &RedisPool, - clickhouse: &clickhouse::Client, -) -> Result { - let project_ids = if req.project_ids.is_empty() { - DBUser::get_projects(user.id.into(), pool, redis).await? - } else { - req.project_ids - .iter() - .map(|id| DBProjectId::from(*id)) - .collect::>() - }; - let project_ids = super::super::filter_allowed_project_ids( - &project_ids, - user, - pool, - redis, - ) - .await?; - - Ok(AnalyticsFacets { - project_views: fetch_project_views_facets( - clickhouse, - &project_ids, - &req.time_range, - ) - .await?, - project_downloads: fetch_project_downloads_facets( - clickhouse, - &project_ids, - &req.time_range, - ) - .await?, - project_playtime: fetch_project_playtime_facets( - clickhouse, - &project_ids, - &req.time_range, - ) - .await?, - }) -} - -async fn fetch_project_views_facets( - clickhouse: &clickhouse::Client, - project_ids: &[DBProjectId], - time_range: &super::super::TimeRange, -) -> Result { - Ok(ProjectViewsFacets { - country: fetch_string_facet( - clickhouse, - "SELECT country AS value, COUNT(*) AS count FROM views WHERE recorded >= {time_range_start: Int64} AND recorded < {time_range_end: Int64} AND project_id IN {project_ids: Array(UInt64)} AND country != '' GROUP BY value ORDER BY count DESC, value LIMIT {facet_limit: UInt64}", - project_ids, - time_range, - ) - .await?, - ..Default::default() - }) -} - -async fn fetch_project_downloads_facets( - clickhouse: &clickhouse::Client, - project_ids: &[DBProjectId], - time_range: &super::super::TimeRange, -) -> Result { - let user_agents = fetch_string_facet( - clickhouse, - "SELECT user_agent AS value, COUNT(*) AS count FROM downloads WHERE recorded >= {time_range_start: Int64} AND recorded < {time_range_end: Int64} AND project_id IN {project_ids: Array(UInt64)} AND user_agent != '' GROUP BY value ORDER BY count DESC, value LIMIT {facet_limit: UInt64}", - project_ids, - time_range, - ) - .await?; - let user_agent = normalize_download_source_facets(&user_agents); - - Ok(ProjectDownloadsFacets { - user_agent, - country: fetch_string_facet( - clickhouse, - "SELECT country AS value, COUNT(*) AS count FROM downloads WHERE recorded >= {time_range_start: Int64} AND recorded < {time_range_end: Int64} AND project_id IN {project_ids: Array(UInt64)} AND country != '' GROUP BY value ORDER BY count DESC, value LIMIT {facet_limit: UInt64}", - project_ids, - time_range, - ) - .await?, - ..Default::default() - }) -} - -fn normalize_download_source_facets( - user_agents: &[FacetValue], -) -> Vec> { - let mut counts = HashMap::::new(); - for user_agent in user_agents { - if let Some(source) = - super::super::normalize_download_source(&user_agent.value) - { - *counts.entry(source).or_default() += user_agent.count; - } - } - - let mut sources = counts - .into_iter() - .map(|(value, count)| FacetValue { value, count }) - .collect::>(); - sources.sort_by(|a, b| { - download_source_sort_key(&a.value) - .cmp(download_source_sort_key(&b.value)) - }); - sources -} - -fn download_source_sort_key(source: &super::super::DownloadSource) -> &str { - match source { - super::super::DownloadSource::Named(name) => name, - super::super::DownloadSource::Website => "website", - super::super::DownloadSource::ModrinthApp => "modrinth_app", - super::super::DownloadSource::ModrinthHosting => "modrinth_hosting", - super::super::DownloadSource::ModrinthMaven => "modrinth_maven", - super::super::DownloadSource::Other => "other", - } -} - -async fn fetch_project_playtime_facets( - clickhouse: &clickhouse::Client, - project_ids: &[DBProjectId], - time_range: &super::super::TimeRange, -) -> Result { - Ok(ProjectPlaytimeFacets { - country: fetch_string_facet( - clickhouse, - "SELECT country AS value, COUNT(*) AS count FROM playtime WHERE recorded >= {time_range_start: Int64} AND recorded < {time_range_end: Int64} AND project_id IN {project_ids: Array(UInt64)} AND country != '' GROUP BY value ORDER BY count DESC, value LIMIT {facet_limit: UInt64}", - project_ids, - time_range, - ) - .await?, - ..Default::default() - }) -} - -async fn fetch_string_facet( - clickhouse: &clickhouse::Client, - query: &str, - project_ids: &[DBProjectId], - time_range: &super::super::TimeRange, -) -> Result>, ApiError> { - let mut rows = clickhouse - .query(query) - .param("time_range_start", time_range.start.timestamp()) - .param("time_range_end", time_range.end.timestamp()) - .param("project_ids", project_ids) - .param("facet_limit", FACET_LIMIT) - .fetch::()?; - let mut values = Vec::new(); - while let Some(row) = rows.next().await? { - values.push(FacetValue { - value: row.value, - count: row.count, - }); - } - Ok(values) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn user_agent_facets_use_normalized_sources() { - let user_agents = vec![ - FacetValue { - value: "MultiMC/5.0".to_string(), - count: 2, - }, - FacetValue { - value: "MultiMC/6.0".to_string(), - count: 3, - }, - FacetValue { - value: "PrismLauncher/6.1".to_string(), - count: 5, - }, - FacetValue { - value: "curl/8.7.1".to_string(), - count: 7, - }, - FacetValue { - value: "Mozilla/5.0 AppleWebKit/537.36".to_string(), - count: 11, - }, - ]; - - assert_eq!( - normalize_download_source_facets(&user_agents), - vec![ - FacetValue { - value: super::super::DownloadSource::Named( - "MultiMC".into() - ), - count: 5, - }, - FacetValue { - value: super::super::DownloadSource::Named( - "Prism Launcher".into() - ), - count: 5, - }, - FacetValue { - value: super::super::DownloadSource::Website, - count: 11, - }, - ], - ); - } -} diff --git a/apps/labrinth/src/routes/v3/analytics_get/facets/fixed.rs b/apps/labrinth/src/routes/v3/analytics_get/facets/fixed.rs index 6e9564834..ba60863fe 100644 --- a/apps/labrinth/src/routes/v3/analytics_get/facets/fixed.rs +++ b/apps/labrinth/src/routes/v3/analytics_get/facets/fixed.rs @@ -1,18 +1,16 @@ use super::super::all_download_sources; use super::{ - AnalyticsFacets, FacetValue, ProjectDownloadsFacets, ProjectPlaytimeFacets, + AnalyticsFacets, ProjectDownloadsFacets, ProjectPlaytimeFacets, ProjectViewsFacets, }; use crate::{ database::{PgPool, redis::RedisPool}, - models::{users::User, v3::analytics::DownloadReason}, + models::v3::analytics::DownloadReason, routes::ApiError, util::tags::valid_download_tags, }; pub async fn fetch( - _req: &super::super::GetRequest, - _user: &User, pool: &PgPool, redis: &RedisPool, ) -> Result { @@ -23,8 +21,8 @@ pub async fn fetch( tags.game_versions.iter().cloned().collect::>(); game_versions.sort(); - let loader_facets = string_facets(loaders); - let game_version_facets = string_facets(game_versions); + let loader_facets = loaders; + let game_version_facets = game_versions; let country_facets = country_facets(); Ok(AnalyticsFacets { @@ -53,20 +51,11 @@ pub async fn fetch( }) } -fn bool_facets() -> Vec> { - vec![ - FacetValue { - value: false, - count: 0, - }, - FacetValue { - value: true, - count: 0, - }, - ] +fn bool_facets() -> Vec { + vec![false, true] } -fn download_reason_facets() -> Vec> { +fn download_reason_facets() -> Vec { [ DownloadReason::Standalone, DownloadReason::Dependency, @@ -74,32 +63,19 @@ fn download_reason_facets() -> Vec> { DownloadReason::Update, ] .into_iter() - .map(|value| FacetValue { value, count: 0 }) .collect() } -fn download_source_facets() -> Vec> { +fn download_source_facets() -> Vec { all_download_sources() - .into_iter() - .map(|value| FacetValue { value, count: 0 }) - .collect() } -fn country_facets() -> Vec> { +fn country_facets() -> Vec { let mut countries = rust_iso3166::ALL_ALPHA2 .iter() .map(|country| country.to_string()) .collect::>(); countries.push("XX".to_string()); countries.sort(); - string_facets(countries) -} - -fn string_facets( - values: impl IntoIterator, -) -> Vec> { - values - .into_iter() - .map(|value| FacetValue { value, count: 0 }) - .collect() + countries } diff --git a/apps/labrinth/src/routes/v3/analytics_get/facets/mod.rs b/apps/labrinth/src/routes/v3/analytics_get/facets/mod.rs index 998ef86c8..145384e89 100644 --- a/apps/labrinth/src/routes/v3/analytics_get/facets/mod.rs +++ b/apps/labrinth/src/routes/v3/analytics_get/facets/mod.rs @@ -1,8 +1,7 @@ -mod dynamic; mod fixed; use actix_web::{HttpRequest, post, web}; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use super::DownloadSource; use crate::models::{ @@ -33,42 +32,30 @@ pub struct AnalyticsFacets { #[derive(Debug, Default, Serialize, utoipa::ToSchema)] pub struct ProjectViewsFacets { - pub domain: Vec>, - pub site_path: Vec>, - pub monetized: Vec>, - pub country: Vec>, + pub domain: Vec, + pub site_path: Vec, + pub monetized: Vec, + pub country: Vec, } #[derive(Debug, Default, Serialize, utoipa::ToSchema)] pub struct ProjectDownloadsFacets { - pub domain: Vec>, - pub user_agent: Vec>, - pub version_id: Vec>, - pub monetized: Vec>, - pub country: Vec>, - pub reason: Vec>, - pub game_version: Vec>, - pub loader: Vec>, + pub domain: Vec, + pub user_agent: Vec, + pub version_id: Vec, + pub monetized: Vec, + pub country: Vec, + pub reason: Vec, + pub game_version: Vec, + pub loader: Vec, } #[derive(Debug, Default, Serialize, utoipa::ToSchema)] pub struct ProjectPlaytimeFacets { - pub version_id: Vec>, - pub loader: Vec>, - pub game_version: Vec>, - pub country: Vec>, -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, utoipa::ToSchema)] -pub struct FacetValue { - pub value: T, - pub count: u64, -} - -#[derive(Debug, Deserialize)] -struct FacetsQuery { - #[serde(default)] - detailed: bool, + pub version_id: Vec, + pub loader: Vec, + pub game_version: Vec, + pub country: Vec, } #[utoipa::path( @@ -77,14 +64,12 @@ struct FacetsQuery { #[post("/facets")] pub async fn fetch_facets( http_req: HttpRequest, - query: web::Query, - req: web::Json, + _req: web::Json, pool: web::Data, redis: web::Data, session_queue: web::Data, - clickhouse: web::Data, ) -> Result, ApiError> { - let (_, user) = get_user_from_headers( + get_user_from_headers( &http_req, &**pool, &redis, @@ -93,12 +78,7 @@ pub async fn fetch_facets( ) .await?; - let facets = if query.detailed { - dynamic::fetch(req.into_inner(), &user, &pool, &redis, &clickhouse) - .await? - } else { - fixed::fetch(&req, &user, &pool, &redis).await? - }; + let facets = fixed::fetch(&pool, &redis).await?; Ok(web::Json(FacetsResponse { facets })) } diff --git a/apps/labrinth/src/routes/v3/analytics_get/metrics/mod.rs b/apps/labrinth/src/routes/v3/analytics_get/metrics/mod.rs index 6ce607e39..91f590033 100644 --- a/apps/labrinth/src/routes/v3/analytics_get/metrics/mod.rs +++ b/apps/labrinth/src/routes/v3/analytics_get/metrics/mod.rs @@ -24,13 +24,14 @@ pub use affiliate_code_revenue::{ AffiliateCodeRevenue, AffiliateCodeRevenueField, AffiliateCodeRevenueFilters, }; +#[cfg(test)] +pub(crate) use project_downloads::normalize_download_source; pub use project_downloads::{ DownloadSource, ProjectDownloads, ProjectDownloadsField, ProjectDownloadsFilters, }; pub(crate) use project_downloads::{ all_download_sources, fetch as fetch_project_downloads, - normalize_download_source, }; pub(crate) use project_playtime::fetch as fetch_project_playtime; pub use project_playtime::{ diff --git a/apps/labrinth/src/routes/v3/analytics_get/mod.rs b/apps/labrinth/src/routes/v3/analytics_get/mod.rs index 2fef3e232..6f32868cf 100644 --- a/apps/labrinth/src/routes/v3/analytics_get/mod.rs +++ b/apps/labrinth/src/routes/v3/analytics_get/mod.rs @@ -48,6 +48,7 @@ use crate::{ routes::ApiError, }; +#[cfg(test)] pub(crate) use metrics::normalize_download_source; pub use metrics::*;