From e76b6c3bde74bfd31a60fb3e6d47e2ba825d3d53 Mon Sep 17 00:00:00 2001 From: Geometrically <18202329+Geometrically@users.noreply.github.com> Date: Mon, 4 Dec 2023 19:45:17 -0700 Subject: [PATCH] Optimize country analytics (#782) --- src/clickhouse/fetch.rs | 96 +++++++++++++++------------------- src/routes/v3/analytics_get.rs | 14 ++--- 2 files changed, 48 insertions(+), 62 deletions(-) diff --git a/src/clickhouse/fetch.rs b/src/clickhouse/fetch.rs index 8f9de4d42..7dbdd8ea7 100644 --- a/src/clickhouse/fetch.rs +++ b/src/clickhouse/fetch.rs @@ -5,32 +5,17 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; #[derive(clickhouse::Row, Serialize, Deserialize, Clone, Debug)] -pub struct ReturnPlaytimes { +pub struct ReturnIntervals { pub time: u32, pub id: u64, - pub total_seconds: u64, + pub total: u64, } #[derive(clickhouse::Row, Serialize, Deserialize, Clone, Debug)] pub struct ReturnCountry { pub country: String, pub id: u64, - pub total_views: u64, - pub total_downloads: u64, -} - -#[derive(clickhouse::Row, Serialize, Deserialize, Clone, Debug)] -pub struct ReturnViews { - pub time: u32, - pub id: u64, - pub total_views: u64, -} - -#[derive(clickhouse::Row, Serialize, Deserialize, Clone, Debug)] -pub struct ReturnDownloads { - pub time: u32, - pub id: u64, - pub total_downloads: u64, + pub total: u64, } // Only one of project_id or version_id should be used @@ -41,14 +26,14 @@ pub async fn fetch_playtimes( end_date: DateTime, resolution_minute: u32, client: Arc, -) -> Result, ApiError> { +) -> Result, ApiError> { let query = client .query( " SELECT toUnixTimestamp(toStartOfInterval(recorded, toIntervalMinute(?))) AS time, project_id AS id, - SUM(seconds) AS total_seconds + SUM(seconds) AS total FROM playtime WHERE recorded BETWEEN ? AND ? AND project_id IN ? @@ -72,14 +57,14 @@ pub async fn fetch_views( end_date: DateTime, resolution_minutes: u32, client: Arc, -) -> Result, ApiError> { +) -> Result, ApiError> { let query = client .query( " SELECT toUnixTimestamp(toStartOfInterval(recorded, toIntervalMinute(?))) AS time, project_id AS id, - count(1) AS total_views + count(1) AS total FROM views WHERE recorded BETWEEN ? AND ? AND project_id IN ? @@ -102,14 +87,14 @@ pub async fn fetch_downloads( end_date: DateTime, resolution_minutes: u32, client: Arc, -) -> Result, ApiError> { +) -> Result, ApiError> { let query = client .query( " SELECT toUnixTimestamp(toStartOfInterval(recorded, toIntervalMinute(?))) AS time, project_id as id, - count(1) AS total_downloads + count(1) AS total FROM downloads WHERE recorded BETWEEN ? AND ? AND project_id IN ? @@ -124,50 +109,51 @@ pub async fn fetch_downloads( Ok(query.fetch_all().await?) } -// Fetches countries as a Vec of ReturnCountry -pub async fn fetch_countries( +pub async fn fetch_countries_downloads( projects: Vec, start_date: DateTime, end_date: DateTime, client: Arc, ) -> Result, ApiError> { let query = client.query( - " - WITH view_grouping AS ( + " SELECT country, project_id, - count(1) AS total_views - FROM views - WHERE recorded BETWEEN ? AND ? - GROUP BY - country, - project_id - ), - download_grouping AS ( - SELECT - country, - project_id, - count(1) AS total_downloads + count(1) AS total FROM downloads - WHERE recorded BETWEEN ? AND ? + WHERE recorded BETWEEN ? AND ? AND project_id IN ? GROUP BY country, - project_id - ) - - SELECT - v.country, - v.project_id, - v.total_views, - d.total_downloads - FROM view_grouping AS v - LEFT JOIN download_grouping AS d ON (v.country = d.country) AND (v.project_id = d.project_id) - WHERE project_id IN ? + project_id; " - ) - .bind(start_date.timestamp()) - .bind(end_date.timestamp()) + ) + .bind(start_date.timestamp()) + .bind(end_date.timestamp()) + .bind(projects.iter().map(|x| x.0).collect::>()); + + Ok(query.fetch_all().await?) +} + +pub async fn fetch_countries_views( + projects: Vec, + start_date: DateTime, + end_date: DateTime, + client: Arc, +) -> Result, ApiError> { + let query = client.query( + " + SELECT + country, + project_id, + count(1) AS total + FROM views + WHERE recorded BETWEEN ? AND ? AND project_id IN ? + GROUP BY + country, + project_id; + " + ) .bind(start_date.timestamp()) .bind(end_date.timestamp()) .bind(projects.iter().map(|x| x.0).collect::>()); diff --git a/src/routes/v3/analytics_get.rs b/src/routes/v3/analytics_get.rs index 281cf853b..dc7542798 100644 --- a/src/routes/v3/analytics_get.rs +++ b/src/routes/v3/analytics_get.rs @@ -117,7 +117,7 @@ pub async fn playtimes_get( hm.insert(id_string.clone(), HashMap::new()); } if let Some(hm) = hm.get_mut(&id_string) { - hm.insert(playtime.time, playtime.total_seconds); + hm.insert(playtime.time, playtime.total); } } @@ -183,7 +183,7 @@ pub async fn views_get( hm.insert(id_string.clone(), HashMap::new()); } if let Some(hm) = hm.get_mut(&id_string) { - hm.insert(views.time, views.total_views); + hm.insert(views.time, views.total); } } @@ -249,7 +249,7 @@ pub async fn downloads_get( hm.insert(id_string.clone(), HashMap::new()); } if let Some(hm) = hm.get_mut(&id_string) { - hm.insert(downloads.time, downloads.total_downloads); + hm.insert(downloads.time, downloads.total); } } @@ -394,7 +394,7 @@ pub async fn countries_downloads_get( let project_ids = filter_allowed_ids(project_ids, user, &pool, &redis).await?; // Get the countries - let countries = crate::clickhouse::fetch_countries( + let countries = crate::clickhouse::fetch_countries_downloads( project_ids.unwrap_or_default(), start_date, end_date, @@ -409,7 +409,7 @@ pub async fn countries_downloads_get( hm.insert(id_string.clone(), HashMap::new()); } if let Some(hm) = hm.get_mut(&id_string) { - hm.insert(views.country, views.total_downloads); + hm.insert(views.country, views.total); } } @@ -461,7 +461,7 @@ pub async fn countries_views_get( let project_ids = filter_allowed_ids(project_ids, user, &pool, &redis).await?; // Get the countries - let countries = crate::clickhouse::fetch_countries( + let countries = crate::clickhouse::fetch_countries_views( project_ids.unwrap_or_default(), start_date, end_date, @@ -476,7 +476,7 @@ pub async fn countries_views_get( hm.insert(id_string.clone(), HashMap::new()); } if let Some(hm) = hm.get_mut(&id_string) { - hm.insert(views.country, views.total_views); + hm.insert(views.country, views.total); } }