eightequal/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Optimize country analytics (#782)

Browse Source
This commit is contained in:
Geometrically
2023-12-04 19:45:17 -07:00
committed by GitHub
parent 4630d175d7
commit e76b6c3bde
2 changed files with 48 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
src/clickhouse/fetch.rs
View File

@@ -5,32 +5,17 @@ use chrono::{DateTime, Utc};
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
#[derive(clickhouse::Row, Serialize, Deserialize, Clone, Debug)] #[derive(clickhouse::Row, Serialize, Deserialize, Clone, Debug)]
pub struct ReturnPlaytimes { pub struct ReturnIntervals {
pub time: u32, pub time: u32,
pub id: u64, pub id: u64,
pub total_seconds: u64, pub total: u64,
} }
#[derive(clickhouse::Row, Serialize, Deserialize, Clone, Debug)] #[derive(clickhouse::Row, Serialize, Deserialize, Clone, Debug)]
pub struct ReturnCountry { pub struct ReturnCountry {
pub country: String, pub country: String,
pub id: u64, pub id: u64,
pub total_views: u64, pub total: u64,
pub total_downloads: u64,
}
#[derive(clickhouse::Row, Serialize, Deserialize, Clone, Debug)]
pub struct ReturnViews {
pub time: u32,
pub id: u64,
pub total_views: u64,
}
#[derive(clickhouse::Row, Serialize, Deserialize, Clone, Debug)]
pub struct ReturnDownloads {
pub time: u32,
pub id: u64,
pub total_downloads: u64,
} }
// Only one of project_id or version_id should be used // Only one of project_id or version_id should be used
@@ -41,14 +26,14 @@ pub async fn fetch_playtimes(
end_date: DateTime<Utc>, end_date: DateTime<Utc>,
resolution_minute: u32, resolution_minute: u32,
client: Arc<clickhouse::Client>, client: Arc<clickhouse::Client>,
) -> Result<Vec<ReturnPlaytimes>, ApiError> { ) -> Result<Vec<ReturnIntervals>, ApiError> {
let query = client let query = client
.query( .query(
" "
SELECT SELECT
toUnixTimestamp(toStartOfInterval(recorded, toIntervalMinute(?))) AS time, toUnixTimestamp(toStartOfInterval(recorded, toIntervalMinute(?))) AS time,
project_id AS id, project_id AS id,
SUM(seconds) AS total_seconds SUM(seconds) AS total
FROM playtime FROM playtime
WHERE recorded BETWEEN ? AND ? WHERE recorded BETWEEN ? AND ?
AND project_id IN ? AND project_id IN ?
@@ -72,14 +57,14 @@ pub async fn fetch_views(
end_date: DateTime<Utc>, end_date: DateTime<Utc>,
resolution_minutes: u32, resolution_minutes: u32,
client: Arc<clickhouse::Client>, client: Arc<clickhouse::Client>,
) -> Result<Vec<ReturnViews>, ApiError> { ) -> Result<Vec<ReturnIntervals>, ApiError> {
let query = client let query = client
.query( .query(
" "
SELECT SELECT
toUnixTimestamp(toStartOfInterval(recorded, toIntervalMinute(?))) AS time, toUnixTimestamp(toStartOfInterval(recorded, toIntervalMinute(?))) AS time,
project_id AS id, project_id AS id,
count(1) AS total_views count(1) AS total
FROM views FROM views
WHERE recorded BETWEEN ? AND ? WHERE recorded BETWEEN ? AND ?
AND project_id IN ? AND project_id IN ?
@@ -102,14 +87,14 @@ pub async fn fetch_downloads(
end_date: DateTime<Utc>, end_date: DateTime<Utc>,
resolution_minutes: u32, resolution_minutes: u32,
client: Arc<clickhouse::Client>, client: Arc<clickhouse::Client>,
) -> Result<Vec<ReturnDownloads>, ApiError> { ) -> Result<Vec<ReturnIntervals>, ApiError> {
let query = client let query = client
.query( .query(
" "
SELECT SELECT
toUnixTimestamp(toStartOfInterval(recorded, toIntervalMinute(?))) AS time, toUnixTimestamp(toStartOfInterval(recorded, toIntervalMinute(?))) AS time,
project_id as id, project_id as id,
count(1) AS total_downloads count(1) AS total
FROM downloads FROM downloads
WHERE recorded BETWEEN ? AND ? WHERE recorded BETWEEN ? AND ?
AND project_id IN ? AND project_id IN ?
@@ -124,50 +109,51 @@ pub async fn fetch_downloads(
Ok(query.fetch_all().await?) Ok(query.fetch_all().await?)
} }
// Fetches countries as a Vec of ReturnCountry pub async fn fetch_countries_downloads(
pub async fn fetch_countries(
projects: Vec<ProjectId>, projects: Vec<ProjectId>,
start_date: DateTime<Utc>, start_date: DateTime<Utc>,
end_date: DateTime<Utc>, end_date: DateTime<Utc>,
client: Arc<clickhouse::Client>, client: Arc<clickhouse::Client>,
) -> Result<Vec<ReturnCountry>, ApiError> { ) -> Result<Vec<ReturnCountry>, ApiError> {
let query = client.query( let query = client.query(
" "
WITH view_grouping AS (
SELECT SELECT
country, country,
project_id, project_id,
count(1) AS total_views count(1) AS total
FROM views
WHERE recorded BETWEEN ? AND ?
GROUP BY
country,
project_id
),
download_grouping AS (
SELECT
country,
project_id,
count(1) AS total_downloads
FROM downloads FROM downloads
WHERE recorded BETWEEN ? AND ? WHERE recorded BETWEEN ? AND ? AND project_id IN ?
GROUP BY GROUP BY
country, country,
project_id project_id;
)
SELECT
v.country,
v.project_id,
v.total_views,
d.total_downloads
FROM view_grouping AS v
LEFT JOIN download_grouping AS d ON (v.country = d.country) AND (v.project_id = d.project_id)
WHERE project_id IN ?
" "
) )
.bind(start_date.timestamp()) .bind(start_date.timestamp())
.bind(end_date.timestamp()) .bind(end_date.timestamp())
.bind(projects.iter().map(|x| x.0).collect::<Vec<_>>());
Ok(query.fetch_all().await?)
}
pub async fn fetch_countries_views(
projects: Vec<ProjectId>,
start_date: DateTime<Utc>,
end_date: DateTime<Utc>,
client: Arc<clickhouse::Client>,
) -> Result<Vec<ReturnCountry>, ApiError> {
let query = client.query(
"
SELECT
country,
project_id,
count(1) AS total
FROM views
WHERE recorded BETWEEN ? AND ? AND project_id IN ?
GROUP BY
country,
project_id;
"
)
.bind(start_date.timestamp()) .bind(start_date.timestamp())
.bind(end_date.timestamp()) .bind(end_date.timestamp())
.bind(projects.iter().map(|x| x.0).collect::<Vec<_>>()); .bind(projects.iter().map(|x| x.0).collect::<Vec<_>>());

14
src/routes/v3/analytics_get.rs
View File

@@ -117,7 +117,7 @@ pub async fn playtimes_get(
hm.insert(id_string.clone(), HashMap::new()); hm.insert(id_string.clone(), HashMap::new());
} }
if let Some(hm) = hm.get_mut(&id_string) { if let Some(hm) = hm.get_mut(&id_string) {
hm.insert(playtime.time, playtime.total_seconds); hm.insert(playtime.time, playtime.total);
} }
} }
@@ -183,7 +183,7 @@ pub async fn views_get(
hm.insert(id_string.clone(), HashMap::new()); hm.insert(id_string.clone(), HashMap::new());
} }
if let Some(hm) = hm.get_mut(&id_string) { if let Some(hm) = hm.get_mut(&id_string) {
hm.insert(views.time, views.total_views); hm.insert(views.time, views.total);
} }
} }
@@ -249,7 +249,7 @@ pub async fn downloads_get(
hm.insert(id_string.clone(), HashMap::new()); hm.insert(id_string.clone(), HashMap::new());
} }
if let Some(hm) = hm.get_mut(&id_string) { if let Some(hm) = hm.get_mut(&id_string) {
hm.insert(downloads.time, downloads.total_downloads); hm.insert(downloads.time, downloads.total);
} }
} }
@@ -394,7 +394,7 @@ pub async fn countries_downloads_get(
let project_ids = filter_allowed_ids(project_ids, user, &pool, &redis).await?; let project_ids = filter_allowed_ids(project_ids, user, &pool, &redis).await?;
// Get the countries // Get the countries
let countries = crate::clickhouse::fetch_countries( let countries = crate::clickhouse::fetch_countries_downloads(
project_ids.unwrap_or_default(), project_ids.unwrap_or_default(),
start_date, start_date,
end_date, end_date,
@@ -409,7 +409,7 @@ pub async fn countries_downloads_get(
hm.insert(id_string.clone(), HashMap::new()); hm.insert(id_string.clone(), HashMap::new());
} }
if let Some(hm) = hm.get_mut(&id_string) { if let Some(hm) = hm.get_mut(&id_string) {
hm.insert(views.country, views.total_downloads); hm.insert(views.country, views.total);
} }
} }
@@ -461,7 +461,7 @@ pub async fn countries_views_get(
let project_ids = filter_allowed_ids(project_ids, user, &pool, &redis).await?; let project_ids = filter_allowed_ids(project_ids, user, &pool, &redis).await?;
// Get the countries // Get the countries
let countries = crate::clickhouse::fetch_countries( let countries = crate::clickhouse::fetch_countries_views(
project_ids.unwrap_or_default(), project_ids.unwrap_or_default(),
start_date, start_date,
end_date, end_date,
@@ -476,7 +476,7 @@ pub async fn countries_views_get(
hm.insert(id_string.clone(), HashMap::new()); hm.insert(id_string.clone(), HashMap::new());
} }
if let Some(hm) = hm.get_mut(&id_string) { if let Some(hm) = hm.get_mut(&id_string) {
hm.insert(views.country, views.total_views); hm.insert(views.country, views.total);
} }
} }