didirus/AstralRinth
5
10
Fork 33
Code Issues 2 Pull Requests Packages Projects Releases 12 Wiki Activity

Flatten facets response format, remove detailed route (#6244)

Browse Source 
* Flatten facets response format

* delete test file
This commit is contained in:
aecsocket
2026-05-29 22:22:08 +01:00
committed by GitHub
parent cef20abceb
commit 67e1743d6c
6 changed files with 33 additions and 336 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/labrinth/.sqlx/query-12fa322e09465aab925ac33f8b2a1371fb63be744f1d87c68229342ffadbe998.json
Generated
-22
View File
@@ -1,22 +0,0 @@
{
"db_name": "PostgreSQL",
"query": "\n SELECT id\n FROM versions\n WHERE mod_id = ANY($1)\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int8Array"
]
},
"nullable": [
false
]
},
"hash": "12fa322e09465aab925ac33f8b2a1371fb63be744f1d87c68229342ffadbe998"
}
apps/labrinth/src/routes/v3/analytics_get/facets/dynamic.rs
-239
View File
@@ -1,239 +0,0 @@
use std::collections::HashMap;
use super::{
AnalyticsFacets, FacetValue, ProjectDownloadsFacets, ProjectPlaytimeFacets,
ProjectViewsFacets,
};
use crate::{
database::{
PgPool,
models::{DBProjectId, DBUser},
redis::RedisPool,
},
models::users::User,
routes::ApiError,
};
const FACET_LIMIT: u64 = 100;
#[derive(Debug, clickhouse::Row, serde::Deserialize)]
struct StringFacetRow {
value: String,
count: u64,
}
pub async fn fetch(
req: super::super::GetRequest,
user: &User,
pool: &PgPool,
redis: &RedisPool,
clickhouse: &clickhouse::Client,
) -> Result<AnalyticsFacets, ApiError> {
let project_ids = if req.project_ids.is_empty() {
DBUser::get_projects(user.id.into(), pool, redis).await?
} else {
req.project_ids
.iter()
.map(|id| DBProjectId::from(*id))
.collect::<Vec<_>>()
};
let project_ids = super::super::filter_allowed_project_ids(
&project_ids,
user,
pool,
redis,
)
.await?;
Ok(AnalyticsFacets {
project_views: fetch_project_views_facets(
clickhouse,
&project_ids,
&req.time_range,
)
.await?,
project_downloads: fetch_project_downloads_facets(
clickhouse,
&project_ids,
&req.time_range,
)
.await?,
project_playtime: fetch_project_playtime_facets(
clickhouse,
&project_ids,
&req.time_range,
)
.await?,
})
}
async fn fetch_project_views_facets(
clickhouse: &clickhouse::Client,
project_ids: &[DBProjectId],
time_range: &super::super::TimeRange,
) -> Result<ProjectViewsFacets, ApiError> {
Ok(ProjectViewsFacets {
country: fetch_string_facet(
clickhouse,
"SELECT country AS value, COUNT(*) AS count FROM views WHERE recorded >= {time_range_start: Int64} AND recorded < {time_range_end: Int64} AND project_id IN {project_ids: Array(UInt64)} AND country != '' GROUP BY value ORDER BY count DESC, value LIMIT {facet_limit: UInt64}",
project_ids,
time_range,
)
.await?,
..Default::default()
})
}
async fn fetch_project_downloads_facets(
clickhouse: &clickhouse::Client,
project_ids: &[DBProjectId],
time_range: &super::super::TimeRange,
) -> Result<ProjectDownloadsFacets, ApiError> {
let user_agents = fetch_string_facet(
clickhouse,
"SELECT user_agent AS value, COUNT(*) AS count FROM downloads WHERE recorded >= {time_range_start: Int64} AND recorded < {time_range_end: Int64} AND project_id IN {project_ids: Array(UInt64)} AND user_agent != '' GROUP BY value ORDER BY count DESC, value LIMIT {facet_limit: UInt64}",
project_ids,
time_range,
)
.await?;
let user_agent = normalize_download_source_facets(&user_agents);
Ok(ProjectDownloadsFacets {
user_agent,
country: fetch_string_facet(
clickhouse,
"SELECT country AS value, COUNT(*) AS count FROM downloads WHERE recorded >= {time_range_start: Int64} AND recorded < {time_range_end: Int64} AND project_id IN {project_ids: Array(UInt64)} AND country != '' GROUP BY value ORDER BY count DESC, value LIMIT {facet_limit: UInt64}",
project_ids,
time_range,
)
.await?,
..Default::default()
})
}
fn normalize_download_source_facets(
user_agents: &[FacetValue<String>],
) -> Vec<FacetValue<super::super::DownloadSource>> {
let mut counts = HashMap::<super::super::DownloadSource, u64>::new();
for user_agent in user_agents {
if let Some(source) =
super::super::normalize_download_source(&user_agent.value)
{
*counts.entry(source).or_default() += user_agent.count;
}
}
let mut sources = counts
.into_iter()
.map(|(value, count)| FacetValue { value, count })
.collect::<Vec<_>>();
sources.sort_by(|a, b| {
download_source_sort_key(&a.value)
.cmp(download_source_sort_key(&b.value))
});
sources
}
fn download_source_sort_key(source: &super::super::DownloadSource) -> &str {
match source {
super::super::DownloadSource::Named(name) => name,
super::super::DownloadSource::Website => "website",
super::super::DownloadSource::ModrinthApp => "modrinth_app",
super::super::DownloadSource::ModrinthHosting => "modrinth_hosting",
super::super::DownloadSource::ModrinthMaven => "modrinth_maven",
super::super::DownloadSource::Other => "other",
}
}
async fn fetch_project_playtime_facets(
clickhouse: &clickhouse::Client,
project_ids: &[DBProjectId],
time_range: &super::super::TimeRange,
) -> Result<ProjectPlaytimeFacets, ApiError> {
Ok(ProjectPlaytimeFacets {
country: fetch_string_facet(
clickhouse,
"SELECT country AS value, COUNT(*) AS count FROM playtime WHERE recorded >= {time_range_start: Int64} AND recorded < {time_range_end: Int64} AND project_id IN {project_ids: Array(UInt64)} AND country != '' GROUP BY value ORDER BY count DESC, value LIMIT {facet_limit: UInt64}",
project_ids,
time_range,
)
.await?,
..Default::default()
})
}
async fn fetch_string_facet(
clickhouse: &clickhouse::Client,
query: &str,
project_ids: &[DBProjectId],
time_range: &super::super::TimeRange,
) -> Result<Vec<FacetValue<String>>, ApiError> {
let mut rows = clickhouse
.query(query)
.param("time_range_start", time_range.start.timestamp())
.param("time_range_end", time_range.end.timestamp())
.param("project_ids", project_ids)
.param("facet_limit", FACET_LIMIT)
.fetch::<StringFacetRow>()?;
let mut values = Vec::new();
while let Some(row) = rows.next().await? {
values.push(FacetValue {
value: row.value,
count: row.count,
});
}
Ok(values)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn user_agent_facets_use_normalized_sources() {
let user_agents = vec![
FacetValue {
value: "MultiMC/5.0".to_string(),
count: 2,
},
FacetValue {
value: "MultiMC/6.0".to_string(),
count: 3,
},
FacetValue {
value: "PrismLauncher/6.1".to_string(),
count: 5,
},
FacetValue {
value: "curl/8.7.1".to_string(),
count: 7,
},
FacetValue {
value: "Mozilla/5.0 AppleWebKit/537.36".to_string(),
count: 11,
},
];
assert_eq!(
normalize_download_source_facets(&user_agents),
vec![
FacetValue {
value: super::super::DownloadSource::Named(
"MultiMC".into()
),
count: 5,
},
FacetValue {
value: super::super::DownloadSource::Named(
"Prism Launcher".into()
),
count: 5,
},
FacetValue {
value: super::super::DownloadSource::Website,
count: 11,
},
],
);
}
}
apps/labrinth/src/routes/v3/analytics_get/facets/fixed.rs
+10 -34
View File
@@ -1,18 +1,16 @@
use super::super::all_download_sources;
use super::{
AnalyticsFacets, FacetValue, ProjectDownloadsFacets, ProjectPlaytimeFacets,
AnalyticsFacets, ProjectDownloadsFacets, ProjectPlaytimeFacets,
ProjectViewsFacets,
};
use crate::{
database::{PgPool, redis::RedisPool},
models::{users::User, v3::analytics::DownloadReason},
models::v3::analytics::DownloadReason,
routes::ApiError,
util::tags::valid_download_tags,
};
pub async fn fetch(
_req: &super::super::GetRequest,
_user: &User,
pool: &PgPool,
redis: &RedisPool,
) -> Result<AnalyticsFacets, ApiError> {
@@ -23,8 +21,8 @@ pub async fn fetch(
tags.game_versions.iter().cloned().collect::<Vec<_>>();
game_versions.sort();
let loader_facets = string_facets(loaders);
let game_version_facets = string_facets(game_versions);
let loader_facets = loaders;
let game_version_facets = game_versions;
let country_facets = country_facets();
Ok(AnalyticsFacets {
@@ -53,20 +51,11 @@ pub async fn fetch(
})
}
fn bool_facets() -> Vec<FacetValue<bool>> {
vec![
FacetValue {
value: false,
count: 0,
},
FacetValue {
value: true,
count: 0,
},
]
fn bool_facets() -> Vec<bool> {
vec![false, true]
}
fn download_reason_facets() -> Vec<FacetValue<DownloadReason>> {
fn download_reason_facets() -> Vec<DownloadReason> {
[
DownloadReason::Standalone,
DownloadReason::Dependency,
@@ -74,32 +63,19 @@ fn download_reason_facets() -> Vec<FacetValue<DownloadReason>> {
DownloadReason::Update,
]
.into_iter()
.map(|value| FacetValue { value, count: 0 })
.collect()
}
fn download_source_facets() -> Vec<FacetValue<super::super::DownloadSource>> {
fn download_source_facets() -> Vec<super::super::DownloadSource> {
all_download_sources()
.into_iter()
.map(|value| FacetValue { value, count: 0 })
.collect()
}
fn country_facets() -> Vec<FacetValue<String>> {
fn country_facets() -> Vec<String> {
let mut countries = rust_iso3166::ALL_ALPHA2
.iter()
.map(|country| country.to_string())
.collect::<Vec<_>>();
countries.push("XX".to_string());
countries.sort();
string_facets(countries)
}
fn string_facets(
values: impl IntoIterator<Item = String>,
) -> Vec<FacetValue<String>> {
values
.into_iter()
.map(|value| FacetValue { value, count: 0 })
.collect()
countries
}
apps/labrinth/src/routes/v3/analytics_get/facets/mod.rs
+20 -40
View File
@@ -1,8 +1,7 @@
mod dynamic;
mod fixed;
use actix_web::{HttpRequest, post, web};
use serde::{Deserialize, Serialize};
use serde::Serialize;
use super::DownloadSource;
use crate::models::{
@@ -33,42 +32,30 @@ pub struct AnalyticsFacets {
#[derive(Debug, Default, Serialize, utoipa::ToSchema)]
pub struct ProjectViewsFacets {
pub domain: Vec<FacetValue<String>>,
pub site_path: Vec<FacetValue<String>>,
pub monetized: Vec<FacetValue<bool>>,
pub country: Vec<FacetValue<String>>,
pub domain: Vec<String>,
pub site_path: Vec<String>,
pub monetized: Vec<bool>,
pub country: Vec<String>,
}
#[derive(Debug, Default, Serialize, utoipa::ToSchema)]
pub struct ProjectDownloadsFacets {
pub domain: Vec<FacetValue<String>>,
pub user_agent: Vec<FacetValue<DownloadSource>>,
pub version_id: Vec<FacetValue<VersionId>>,
pub monetized: Vec<FacetValue<bool>>,
pub country: Vec<FacetValue<String>>,
pub reason: Vec<FacetValue<DownloadReason>>,
pub game_version: Vec<FacetValue<String>>,
pub loader: Vec<FacetValue<String>>,
pub domain: Vec<String>,
pub user_agent: Vec<DownloadSource>,
pub version_id: Vec<VersionId>,
pub monetized: Vec<bool>,
pub country: Vec<String>,
pub reason: Vec<DownloadReason>,
pub game_version: Vec<String>,
pub loader: Vec<String>,
}
#[derive(Debug, Default, Serialize, utoipa::ToSchema)]
pub struct ProjectPlaytimeFacets {
pub version_id: Vec<FacetValue<VersionId>>,
pub loader: Vec<FacetValue<String>>,
pub game_version: Vec<FacetValue<String>>,
pub country: Vec<FacetValue<String>>,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, utoipa::ToSchema)]
pub struct FacetValue<T> {
pub value: T,
pub count: u64,
}
#[derive(Debug, Deserialize)]
struct FacetsQuery {
#[serde(default)]
detailed: bool,
pub version_id: Vec<VersionId>,
pub loader: Vec<String>,
pub game_version: Vec<String>,
pub country: Vec<String>,
}
#[utoipa::path(
@@ -77,14 +64,12 @@ struct FacetsQuery {
#[post("/facets")]
pub async fn fetch_facets(
http_req: HttpRequest,
query: web::Query<FacetsQuery>,
req: web::Json<super::GetRequest>,
_req: web::Json<super::GetRequest>,
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
clickhouse: web::Data<clickhouse::Client>,
) -> Result<web::Json<FacetsResponse>, ApiError> {
let (_, user) = get_user_from_headers(
get_user_from_headers(
&http_req,
&**pool,
&redis,
@@ -93,12 +78,7 @@ pub async fn fetch_facets(
)
.await?;
let facets = if query.detailed {
dynamic::fetch(req.into_inner(), &user, &pool, &redis, &clickhouse)
.await?
} else {
fixed::fetch(&req, &user, &pool, &redis).await?
};
let facets = fixed::fetch(&pool, &redis).await?;
Ok(web::Json(FacetsResponse { facets }))
}
apps/labrinth/src/routes/v3/analytics_get/metrics/mod.rs
+2 -1
View File
@@ -24,13 +24,14 @@ pub use affiliate_code_revenue::{
AffiliateCodeRevenue, AffiliateCodeRevenueField,
AffiliateCodeRevenueFilters,
};
#[cfg(test)]
pub(crate) use project_downloads::normalize_download_source;
pub use project_downloads::{
DownloadSource, ProjectDownloads, ProjectDownloadsField,
ProjectDownloadsFilters,
};
pub(crate) use project_downloads::{
all_download_sources, fetch as fetch_project_downloads,
normalize_download_source,
};
pub(crate) use project_playtime::fetch as fetch_project_playtime;
pub use project_playtime::{
apps/labrinth/src/routes/v3/analytics_get/mod.rs
+1
View File
@@ -48,6 +48,7 @@ use crate::{
routes::ApiError,
};
#[cfg(test)]
pub(crate) use metrics::normalize_download_source;
pub use metrics::*;