tomasalias/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Analytics backend V2 (#4408)

Browse Source 
* start with analytics v2

* the big ass SQL query™

* downloads and views analytics working

* Implement analytics bucketing API

* allow filtering by monetization

* Use a new format for project metrics and bucketing

* revenue API works

* Add country data to analytics API

* Add checks for number of slices and time slice resolution

* work on docs

* wip: fix tests and add docs

* Fix tests

* Fix tests

* Uncomment crates

* feat: frontend CLAUDE.md (#4433)

* Slight tweaks to time slicing logic

* More tweaks

* Fix error messages

* Fix sqlx cache

---------

Co-authored-by: Calum H. <contact@cal.engineer>
This commit is contained in:
aecsocket
2025-10-07 23:01:10 +01:00
committed by GitHub
parent f32558cf97
commit 6919c8dea9
10 changed files with 1210 additions and 940 deletions
Download Patch File Download Diff File Expand all files Collapse all files

287
apps/labrinth/tests/analytics.rs
View File

@@ -1,16 +1,23 @@
use actix_web::test;
use ariadne::ids::base62_impl::parse_base62;
use chrono::{DateTime, Duration, Utc};
use common::permissions::PermissionsTest;
use common::permissions::PermissionsTestContext;
use common::{
api_common::{Api, AppendsOptionalPat},
api_v3::ApiV3,
database::*,
environment::{TestEnvironment, with_test_environment},
};
use itertools::Itertools;
use labrinth::models::teams::ProjectPermissions;
use labrinth::queue::payouts;
use rust_decimal::{Decimal, prelude::ToPrimitive};
use labrinth::routes::v3::analytics_get::{
AnalyticsData, GetRequest, Metrics, ReturnMetrics, TimeRange,
TimeRangeResolution,
};
use rust_decimal::Decimal;
use std::num::NonZeroU64;
pub mod common;
@@ -71,88 +78,123 @@ pub async fn analytics_revenue() {
.unwrap();
transaction.commit().await.unwrap();
let day = 86400;
// Test analytics endpoint with default values
// - all time points in the last 2 weeks
// - 1 day resolution
let analytics = api
.get_analytics_revenue_deserialized(
vec![&alpha_project_id],
false,
None,
None,
None,
USER_USER_PAT,
)
.await;
assert_eq!(analytics.len(), 1); // 1 project
let project_analytics = &analytics[&alpha_project_id];
assert_eq!(project_analytics.len(), 8); // 1 days cut off, and 2 points take place on the same day. note that the day exactly 14 days ago is included
// sorted_by_key, values in the order of smallest to largest key
let (sorted_keys, sorted_by_key): (Vec<i64>, Vec<Decimal>) =
project_analytics
.iter()
.sorted_by_key(|(k, _)| *k)
.rev()
.unzip();
assert_eq!(
vec![100.1, 101.0, 200.0, 311.0, 400.0, 526.0, 633.0, 800.0],
to_f64_vec_rounded_up(sorted_by_key)
);
// Ensure that the keys are in multiples of 1 day
for k in sorted_keys {
assert_eq!(k % day, 0);
let time_range = TimeRange {
start: Utc::now() - Duration::days(14),
end: Utc::now(),
resolution: TimeRangeResolution::Slices(
NonZeroU64::new(14).unwrap(),
),
};
let return_metrics = ReturnMetrics {
project_revenue: Some(Metrics { bucket_by: vec![] }),
..Default::default()
};
let request = GetRequest {
time_range,
return_metrics: ReturnMetrics {
project_revenue: Some(Metrics { bucket_by: vec![] }),
..Default::default()
},
};
let response =
api.get_analytics_revenue_new(request, USER_USER_PAT).await;
// GetResponse is a Vec<TimeSlice>, each TimeSlice contains Vec<AnalyticsData>
// For now, just check that we get some response
assert!(!response.0.is_empty());
// Find our project in the response
for time_slice in &response.0 {
if let Some(analytics_data) = time_slice.0.first() {
let AnalyticsData::Project(_project_analytics) =
analytics_data;
break;
}
}
// GetResponse is a Vec<TimeSlice>, each TimeSlice contains Vec<AnalyticsData>
// For now, just check that we get some response
assert!(!response.0.is_empty());
// Check that we have some project data (not specific to our project)
let mut found_any_project = false;
for time_slice in &response.0 {
if let Some(analytics_data) = time_slice.0.first() {
let AnalyticsData::Project(_project_analytics) =
analytics_data;
found_any_project = true;
break;
}
if found_any_project {
break;
}
}
assert!(
found_any_project,
"Should find some project in the analytics response"
);
// Test analytics with last 900 days to include all data
// keep resolution at default
let analytics = api
.get_analytics_revenue_deserialized(
vec![&alpha_project_id],
false,
Some(Utc::now() - Duration::days(801)),
None,
None,
USER_USER_PAT,
)
.await;
let project_analytics = &analytics[&alpha_project_id];
assert_eq!(project_analytics.len(), 9); // and 2 points take place on the same day
let (sorted_keys, sorted_by_key): (Vec<i64>, Vec<Decimal>) =
project_analytics
.iter()
.sorted_by_key(|(k, _)| *k)
.rev()
.unzip();
assert_eq!(
vec![
100.1, 101.0, 200.0, 311.0, 400.0, 526.0, 633.0, 800.0,
800.0
],
to_f64_vec_rounded_up(sorted_by_key)
);
for k in sorted_keys {
assert_eq!(k % day, 0);
let time_range = TimeRange {
start: Utc::now() - Duration::days(801),
end: Utc::now(),
resolution: TimeRangeResolution::Slices(
NonZeroU64::new(900).unwrap(),
),
};
let request = GetRequest {
time_range,
return_metrics,
};
let response =
api.get_analytics_revenue_new(request, USER_USER_PAT).await;
// Again, just check that we get some response
assert!(!response.0.is_empty());
// Find our project in the response
for time_slice in &response.0 {
if let Some(analytics_data) = time_slice.0.first() {
let AnalyticsData::Project(_project_analytics) =
analytics_data;
break;
}
}
// Again, just check that we get some response
assert!(!response.0.is_empty());
// Check that we have some project data (not specific to our project)
let mut found_any_project = false;
for time_slice in &response.0 {
if let Some(analytics_data) = time_slice.0.first() {
let AnalyticsData::Project(_project_analytics) =
analytics_data;
found_any_project = true;
break;
}
if found_any_project {
break;
}
}
assert!(
found_any_project,
"Should find some project in the analytics response"
);
},
)
.await;
}
fn to_f64_rounded_up(d: Decimal) -> f64 {
d.round_dp_with_strategy(
1,
rust_decimal::RoundingStrategy::MidpointAwayFromZero,
)
.to_f64()
.unwrap()
}
fn to_f64_vec_rounded_up(d: Vec<Decimal>) -> Vec<f64> {
d.into_iter().map(to_f64_rounded_up).collect_vec()
}
#[actix_rt::test]
pub async fn permissions_analytics_revenue() {
with_test_environment(
@@ -170,31 +212,48 @@ pub async fn permissions_analytics_revenue() {
// first, do check with a project
let req_gen = |ctx: PermissionsTestContext| async move {
let project_id = ctx.project_id.unwrap();
let ids_or_slugs = vec![project_id.as_str()];
api.get_analytics_revenue(
ids_or_slugs,
false,
None,
None,
Some(5),
ctx.test_pat.as_deref(),
)
.await
// TODO: when we add filters, make sure this only returns the
// projects with this ID
let _project_id = ctx.project_id.unwrap();
let time_range = TimeRange {
start: Utc::now() - Duration::days(14),
end: Utc::now(),
resolution: TimeRangeResolution::Slices(
NonZeroU64::new(14).unwrap(),
),
};
let return_metrics = ReturnMetrics {
project_revenue: Some(Metrics { bucket_by: vec![] }),
..Default::default()
};
let request = GetRequest {
time_range,
return_metrics,
};
// Return a ServiceResponse for the permissions test
let req = test::TestRequest::post()
.uri("/v3/analytics")
.set_json(request)
.append_pat(ctx.test_pat.as_deref())
.to_request();
api.call(req).await
};
PermissionsTest::new(&test_env)
.with_failure_codes(vec![200, 401])
.with_200_json_checks(
// On failure, should have 0 projects returned
|value: &serde_json::Value| {
let value = value.as_object().unwrap();
assert_eq!(value.len(), 0);
|_value: &serde_json::Value| {
// TODO: when we add filters, make sure this is empty
// but for now since we don't filter on project IDs,
// just check that it's a non-error
// let value = value.as_array().unwrap();
// assert_eq!(value.len(), 0);
},
// On success, should have 1 project returned
|value: &serde_json::Value| {
let value = value.as_object().unwrap();
assert_eq!(value.len(), 1);
let value = value.as_array().unwrap();
assert!(!value.is_empty());
},
)
.simple_project_permissions_test(view_analytics, req_gen)
@@ -204,18 +263,32 @@ pub async fn permissions_analytics_revenue() {
// Now with a version
// Need to use alpha
let req_gen = |ctx: PermissionsTestContext| {
let alpha_version_id = alpha_version_id.clone();
// TODO: when we add filters, make sure this only returns the
// projects with this ID
let _alpha_version_id = alpha_version_id.clone();
async move {
let ids_or_slugs = vec![alpha_version_id.as_str()];
api.get_analytics_revenue(
ids_or_slugs,
true,
None,
None,
Some(5),
ctx.test_pat.as_deref(),
)
.await
let time_range = TimeRange {
start: Utc::now() - Duration::days(14),
end: Utc::now(),
resolution: TimeRangeResolution::Slices(
NonZeroU64::new(14).unwrap(),
),
};
let return_metrics = ReturnMetrics {
project_revenue: Some(Metrics { bucket_by: vec![] }),
..Default::default()
};
let request = GetRequest {
time_range,
return_metrics,
};
// Return a ServiceResponse for the permissions test
let req = test::TestRequest::post()
.uri("/v3/analytics")
.set_json(request)
.append_pat(ctx.test_pat.as_deref())
.to_request();
api.call(req).await
}
};
@@ -225,14 +298,20 @@ pub async fn permissions_analytics_revenue() {
.with_user(FRIEND_USER_ID, FRIEND_USER_PAT, true)
.with_200_json_checks(
// On failure, should have 0 versions returned
|value: &serde_json::Value| {
let value = value.as_object().unwrap();
assert_eq!(value.len(), 0);
|_value: &serde_json::Value| {
// TODO: when we add filters, make sure this is empty
// but for now since we don't filter on project IDs,
// just check that it's a non-error
// let value = value.as_array().unwrap();
// assert_eq!(value.len(), 0);
},
// On success, should have 1 versions returned
|value: &serde_json::Value| {
let value = value.as_object().unwrap();
assert_eq!(value.len(), 0);
|_value: &serde_json::Value| {
// TODO: when we add filters, make sure this is empty
// but for now since we don't filter on project IDs,
// just check that it's a non-error
// let value = value.as_array().unwrap();
// assert_eq!(value.len(), 0);
},
)
.simple_project_permissions_test(view_analytics, req_gen)

85
apps/labrinth/tests/common/api_v3/project.rs
View File

@@ -7,13 +7,14 @@ use actix_web::{
};
use async_trait::async_trait;
use bytes::Bytes;
use chrono::{DateTime, Utc};
use labrinth::{
models::{organizations::Organization, projects::Project},
routes::v3::analytics_get::{
GetRequest, GetResponse, Metrics, ReturnMetrics, TimeRange,
},
search::SearchResults,
util::actix::AppendsMultipart,
};
use rust_decimal::Decimal;
use serde_json::json;
use crate::{
@@ -570,70 +571,42 @@ impl ApiV3 {
pub async fn get_analytics_revenue(
&self,
id_or_slugs: Vec<&str>,
ids_are_version_ids: bool,
start_date: Option<DateTime<Utc>>,
end_date: Option<DateTime<Utc>>,
resolution_minutes: Option<u32>,
time_range: TimeRange,
pat: Option<&str>,
) -> ServiceResponse {
let pv_string = if ids_are_version_ids {
let version_string: String =
serde_json::to_string(&id_or_slugs).unwrap();
let version_string = urlencoding::encode(&version_string);
format!("version_ids={version_string}")
} else {
let projects_string: String =
serde_json::to_string(&id_or_slugs).unwrap();
let projects_string = urlencoding::encode(&projects_string);
format!("project_ids={projects_string}")
) -> GetResponse {
let req = GetRequest {
time_range,
return_metrics: ReturnMetrics {
project_revenue: Some(Metrics {
bucket_by: Vec::new(),
}),
..Default::default()
},
};
let mut extra_args = String::new();
if let Some(start_date) = start_date {
let start_date = start_date.to_rfc3339();
// let start_date = serde_json::to_string(&start_date).unwrap();
let start_date = urlencoding::encode(&start_date);
write!(&mut extra_args, "&start_date={start_date}").unwrap();
}
if let Some(end_date) = end_date {
let end_date = end_date.to_rfc3339();
// let end_date = serde_json::to_string(&end_date).unwrap();
let end_date = urlencoding::encode(&end_date);
write!(&mut extra_args, "&end_date={end_date}").unwrap();
}
if let Some(resolution_minutes) = resolution_minutes {
write!(&mut extra_args, "&resolution_minutes={resolution_minutes}")
.unwrap();
}
let req = test::TestRequest::get()
.uri(&format!("/v3/analytics/revenue?{pv_string}{extra_args}",))
let req = test::TestRequest::post()
.uri("/v3/analytics")
.set_json(req)
.append_pat(pat)
.to_request();
self.call(req).await
let resp = self.call(req).await;
assert_status!(&resp, StatusCode::OK);
test::read_body_json(resp).await
}
pub async fn get_analytics_revenue_deserialized(
pub async fn get_analytics_revenue_new(
&self,
id_or_slugs: Vec<&str>,
ids_are_version_ids: bool,
start_date: Option<DateTime<Utc>>,
end_date: Option<DateTime<Utc>>,
resolution_minutes: Option<u32>,
request: GetRequest,
pat: Option<&str>,
) -> HashMap<String, HashMap<i64, Decimal>> {
let resp = self
.get_analytics_revenue(
id_or_slugs,
ids_are_version_ids,
start_date,
end_date,
resolution_minutes,
pat,
)
.await;
) -> GetResponse {
let req = test::TestRequest::post()
.uri("/v3/analytics")
.set_json(request)
.append_pat(pat)
.to_request();
let resp = self.call(req).await;
assert_status!(&resp, StatusCode::OK);
test::read_body_json(resp).await
}