You've already forked AstralRinth
Add analytics meta for downloading dependent projects (#6318)
* Send dependent mod info to backend * Parse meta from query * condense dependent_on and modpack * Analytics dependents response
This commit is contained in:
@@ -197,11 +197,13 @@ export async function add_project_from_version(
|
||||
path: string,
|
||||
versionId: string,
|
||||
reason: DownloadReason,
|
||||
dependentOnVersionId?: string,
|
||||
): Promise<string> {
|
||||
return await invoke('plugin:profile|profile_add_project_from_version', {
|
||||
path,
|
||||
versionId,
|
||||
reason,
|
||||
dependentOnVersionId,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -71,7 +71,7 @@ export const installVersionDependencies = async (profile, version, reason, onDep
|
||||
return installed
|
||||
}
|
||||
|
||||
const queueInstall = async (projectId, resolvedVersion) => {
|
||||
const queueInstall = async (projectId, resolvedVersion, dependentOn) => {
|
||||
if (!resolvedVersion?.id) return false
|
||||
|
||||
const versionId = resolvedVersion.id
|
||||
@@ -91,7 +91,11 @@ export const installVersionDependencies = async (profile, version, reason, onDep
|
||||
if (resolvedProjectId) {
|
||||
queuedProjectVersions.set(resolvedProjectId, versionId)
|
||||
}
|
||||
queuedInstalls.push({ versionId, projectId: resolvedProjectId })
|
||||
queuedInstalls.push({
|
||||
versionId,
|
||||
projectId: resolvedProjectId,
|
||||
dependentOnVersionId: dependentOn?.id,
|
||||
})
|
||||
return true
|
||||
}
|
||||
|
||||
@@ -159,7 +163,7 @@ export const installVersionDependencies = async (profile, version, reason, onDep
|
||||
if (!resolved) continue
|
||||
|
||||
const { depVersion, depProjectId } = resolved
|
||||
const queued = await queueInstall(depProjectId, depVersion)
|
||||
const queued = await queueInstall(depProjectId, depVersion, inputVersion)
|
||||
if (queued && depProjectId) {
|
||||
await announceDependency(depProjectId, depVersion)
|
||||
}
|
||||
@@ -176,8 +180,8 @@ export const installVersionDependencies = async (profile, version, reason, onDep
|
||||
for (let i = 0; i < queuedInstalls.length; i += batchSize) {
|
||||
const batch = queuedInstalls.slice(i, i + batchSize)
|
||||
await Promise.all(
|
||||
batch.map(async ({ versionId }) => {
|
||||
await add_project_from_version(profile.path, versionId, reason)
|
||||
batch.map(async ({ versionId, dependentOnVersionId }) => {
|
||||
await add_project_from_version(profile.path, versionId, reason, dependentOnVersionId)
|
||||
}),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -251,8 +251,15 @@ pub async fn profile_add_project_from_version(
|
||||
path: &str,
|
||||
version_id: &str,
|
||||
reason: DownloadReason,
|
||||
dependent_on_version_id: Option<String>,
|
||||
) -> Result<String> {
|
||||
Ok(profile::add_project_from_version(path, version_id, reason).await?)
|
||||
Ok(profile::add_project_from_version(
|
||||
path,
|
||||
version_id,
|
||||
reason,
|
||||
dependent_on_version_id,
|
||||
)
|
||||
.await?)
|
||||
}
|
||||
|
||||
// Adds a project to a profile from a path
|
||||
|
||||
Generated
+22
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"db_name": "PostgreSQL",
|
||||
"query": "\n SELECT id FROM versions\n WHERE mod_id = ANY($1)\n ",
|
||||
"describe": {
|
||||
"columns": [
|
||||
{
|
||||
"ordinal": 0,
|
||||
"name": "id",
|
||||
"type_info": "Int8"
|
||||
}
|
||||
],
|
||||
"parameters": {
|
||||
"Left": [
|
||||
"Int8Array"
|
||||
]
|
||||
},
|
||||
"nullable": [
|
||||
false
|
||||
]
|
||||
},
|
||||
"hash": "3eacabccb1da975ceba03932880681c39ef3190c365e292c49dfe4acd7671395"
|
||||
}
|
||||
@@ -247,7 +247,8 @@ pub async fn init_client_with_database(
|
||||
ALTER TABLE {database}.{DOWNLOADS} {cluster_line}
|
||||
ADD COLUMN IF NOT EXISTS reason String,
|
||||
ADD COLUMN IF NOT EXISTS game_version String,
|
||||
ADD COLUMN IF NOT EXISTS loader String
|
||||
ADD COLUMN IF NOT EXISTS loader String,
|
||||
ADD COLUMN IF NOT EXISTS dependent_on_version_id UInt64
|
||||
"
|
||||
))
|
||||
.execute()
|
||||
|
||||
@@ -29,6 +29,7 @@ pub struct Download {
|
||||
pub reason: String,
|
||||
pub game_version: String,
|
||||
pub loader: String,
|
||||
pub dependent_on_version_id: u64,
|
||||
}
|
||||
|
||||
/// Why a project was downloaded.
|
||||
|
||||
@@ -2,7 +2,7 @@ use crate::auth::validate::get_user_record_from_bearer_token;
|
||||
use crate::database::PgPool;
|
||||
use crate::database::redis::RedisPool;
|
||||
use crate::models::analytics::{Download, DownloadReason};
|
||||
use crate::models::ids::ProjectId;
|
||||
use crate::models::ids::{ProjectId, VersionId};
|
||||
use crate::models::pats::Scopes;
|
||||
use crate::queue::analytics::AnalyticsQueue;
|
||||
use crate::queue::session::AuthQueue;
|
||||
@@ -13,10 +13,12 @@ use crate::util::error::Context;
|
||||
use crate::util::guards::admin_key_guard;
|
||||
use crate::util::tags::valid_download_tags;
|
||||
use actix_web::{HttpRequest, HttpResponse, patch, post, web};
|
||||
use ariadne::ids::base62_impl::parse_base62;
|
||||
use eyre::eyre;
|
||||
use serde::Deserialize;
|
||||
use std::collections::HashMap;
|
||||
use std::net::Ipv4Addr;
|
||||
use std::str::FromStr;
|
||||
use std::sync::Arc;
|
||||
use tracing::trace;
|
||||
|
||||
@@ -45,10 +47,86 @@ pub struct DownloadMeta {
|
||||
pub reason: Option<DownloadReason>,
|
||||
pub game_version: Option<String>,
|
||||
pub loader: Option<String>,
|
||||
pub dependent_on: Option<VersionId>,
|
||||
}
|
||||
|
||||
pub const DOWNLOAD_META_HEADER: &str = "modrinth-download-meta";
|
||||
|
||||
fn parse_download_meta_version(
|
||||
version_id: &str,
|
||||
field: &str,
|
||||
) -> Result<VersionId, ApiError> {
|
||||
parse_base62(version_id)
|
||||
.map(VersionId)
|
||||
.wrap_request_err_with(|| {
|
||||
eyre!("invalid `{field}` version id '{version_id}'")
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_download_meta_from_query(
|
||||
url: &url::Url,
|
||||
) -> Result<Option<DownloadMeta>, ApiError> {
|
||||
let mut meta = DownloadMeta {
|
||||
reason: None,
|
||||
game_version: None,
|
||||
loader: None,
|
||||
dependent_on: None,
|
||||
};
|
||||
|
||||
for (key, value) in url.query_pairs() {
|
||||
match key.as_ref() {
|
||||
"mr_download_reason" => {
|
||||
meta.reason =
|
||||
Some(DownloadReason::from_str(&value).map_err(|_| {
|
||||
ApiError::Request(eyre!(
|
||||
"invalid download reason specified"
|
||||
))
|
||||
})?);
|
||||
}
|
||||
"mr_game_version" => {
|
||||
meta.game_version = Some(value.into_owned());
|
||||
}
|
||||
"mr_loader" => {
|
||||
meta.loader = Some(value.into_owned());
|
||||
}
|
||||
"mr_dependent_on" => {
|
||||
meta.dependent_on =
|
||||
Some(parse_download_meta_version(&value, "dependent_on")?);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
Ok((meta.reason.is_some()
|
||||
|| meta.game_version.is_some()
|
||||
|| meta.loader.is_some()
|
||||
|| meta.dependent_on.is_some())
|
||||
.then_some(meta))
|
||||
}
|
||||
|
||||
async fn resolve_download_attribution_version(
|
||||
pool: &PgPool,
|
||||
redis: &RedisPool,
|
||||
version_id: Option<VersionId>,
|
||||
field: &str,
|
||||
) -> Result<u64, ApiError> {
|
||||
let Some(version_id) = version_id else {
|
||||
return Ok(0);
|
||||
};
|
||||
|
||||
let version_id =
|
||||
crate::database::models::ids::DBVersionId::from(version_id);
|
||||
|
||||
crate::database::models::DBVersion::get(version_id, pool, redis)
|
||||
.await
|
||||
.wrap_internal_err("failed to fetch download attribution version")?
|
||||
.ok_or_else(|| {
|
||||
ApiError::Request(eyre!("invalid `{field}` version specified"))
|
||||
})?;
|
||||
|
||||
Ok(version_id.0 as u64)
|
||||
}
|
||||
|
||||
// This is an internal route, cannot be used without key
|
||||
#[utoipa::path(
|
||||
patch,
|
||||
@@ -89,10 +167,9 @@ pub async fn count_download(
|
||||
let project_id: crate::database::models::ids::DBProjectId =
|
||||
download_body.project_id.into();
|
||||
|
||||
let id_option =
|
||||
ariadne::ids::base62_impl::parse_base62(&download_body.version_name)
|
||||
.ok()
|
||||
.map(|x| x as i64);
|
||||
let id_option = parse_base62(&download_body.version_name)
|
||||
.ok()
|
||||
.map(|x| x as i64);
|
||||
|
||||
let (version_id, project_id) = if let Some(version) = sqlx::query!(
|
||||
"
|
||||
@@ -138,7 +215,7 @@ pub async fn count_download(
|
||||
.map(Some)
|
||||
.wrap_request_err("invalid download meta")?
|
||||
} else {
|
||||
None
|
||||
parse_download_meta_from_query(&url)?
|
||||
};
|
||||
|
||||
if let Some(meta) = &meta {
|
||||
@@ -162,6 +239,14 @@ pub async fn count_download(
|
||||
}
|
||||
}
|
||||
|
||||
let dependent_on_version_id = resolve_download_attribution_version(
|
||||
&pool,
|
||||
&redis,
|
||||
meta.as_ref().and_then(|m| m.dependent_on),
|
||||
"dependent_on",
|
||||
)
|
||||
.await?;
|
||||
|
||||
let download = Download {
|
||||
recorded: get_current_tenths_of_ms(),
|
||||
domain: url.host_str().unwrap_or_default().to_string(),
|
||||
@@ -212,6 +297,7 @@ pub async fn count_download(
|
||||
.and_then(|m| m.loader.as_ref())
|
||||
.map(|s| s.to_string())
|
||||
.unwrap_or_default(),
|
||||
dependent_on_version_id,
|
||||
};
|
||||
trace!("added download {download:#?}");
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
collections::{HashMap, HashSet},
|
||||
sync::{
|
||||
LazyLock,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
@@ -12,9 +12,16 @@ use regex::Regex;
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer, de::Error as _};
|
||||
|
||||
use crate::{
|
||||
database::models::{DBProjectId, DBVersionId},
|
||||
models::{ids::VersionId, v3::analytics::DownloadReason},
|
||||
database::{
|
||||
PgPool,
|
||||
models::{DBProjectId, DBVersion, DBVersionId},
|
||||
},
|
||||
models::{
|
||||
ids::{ProjectId, VersionId},
|
||||
v3::analytics::DownloadReason,
|
||||
},
|
||||
routes::ApiError,
|
||||
util::error::Context,
|
||||
};
|
||||
|
||||
use super::super::{
|
||||
@@ -39,6 +46,8 @@ pub enum ProjectDownloadsField {
|
||||
ProjectId,
|
||||
/// Version ID of this project.
|
||||
VersionId,
|
||||
/// Project ID that caused this project to be downloaded.
|
||||
DependentProjectId,
|
||||
/// Referrer domain which linked to this project.
|
||||
Domain,
|
||||
/// Normalized user agent used to download this project.
|
||||
@@ -63,6 +72,9 @@ pub struct ProjectDownloadsFilters {
|
||||
/// Version IDs to include.
|
||||
#[serde(default)]
|
||||
pub version_id: Vec<VersionId>,
|
||||
/// Dependent project IDs to include.
|
||||
#[serde(default)]
|
||||
pub dependent_project_id: Vec<ProjectId>,
|
||||
/// Referrer domains to include.
|
||||
#[serde(default)]
|
||||
pub domain: Vec<String>,
|
||||
@@ -98,6 +110,9 @@ pub struct ProjectDownloads {
|
||||
/// [`ProjectDownloadsField::VersionId`].
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) version_id: Option<VersionId>,
|
||||
/// [`ProjectDownloadsField::DependentProjectId`].
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) dependent_project_id: Option<ProjectId>,
|
||||
/// [`ProjectDownloadsField::Monetized`].
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) monetized: Option<bool>,
|
||||
@@ -175,6 +190,7 @@ struct DownloadRow {
|
||||
domain: String,
|
||||
user_agent: String,
|
||||
version_id: DBVersionId,
|
||||
dependent_on_version_id: DBVersionId,
|
||||
monetized: i8,
|
||||
country: String,
|
||||
reason: String,
|
||||
@@ -188,6 +204,7 @@ const DOWNLOADS: &str = {
|
||||
const USE_DOMAIN: &str = "{use_domain: Bool}";
|
||||
const USE_USER_AGENT: &str = "{use_user_agent: Bool}";
|
||||
const USE_VERSION_ID: &str = "{use_version_id: Bool}";
|
||||
const USE_DEPENDENT_PROJECT_ID: &str = "{use_dependent_project_id: Bool}";
|
||||
const USE_MONETIZED: &str = "{use_monetized: Bool}";
|
||||
const USE_COUNTRY: &str = "{use_country: Bool}";
|
||||
const USE_REASON: &str = "{use_reason: Bool}";
|
||||
@@ -195,6 +212,8 @@ const DOWNLOADS: &str = {
|
||||
const USE_LOADER: &str = "{use_loader: Bool}";
|
||||
const FILTER_DOMAIN: &str = "filter_domain";
|
||||
const FILTER_VERSION_ID: &str = "filter_version_id";
|
||||
const FILTER_DEPENDENT_ON_VERSION_ID: &str =
|
||||
"filter_dependent_on_version_id";
|
||||
const FILTER_MONETIZED: &str = "{filter_monetized: UInt8}";
|
||||
const FILTER_COUNTRY: &str = "filter_country";
|
||||
const FILTER_REASON: &str = "filter_reason";
|
||||
@@ -206,6 +225,7 @@ const DOWNLOADS: &str = {
|
||||
? AS {PROJECT_IDS},
|
||||
? AS {FILTER_DOMAIN},
|
||||
? AS {FILTER_VERSION_ID},
|
||||
? AS {FILTER_DEPENDENT_ON_VERSION_ID},
|
||||
? AS {FILTER_COUNTRY},
|
||||
? AS {FILTER_REASON},
|
||||
? AS {FILTER_GAME_VERSION},
|
||||
@@ -217,6 +237,7 @@ const DOWNLOADS: &str = {
|
||||
if({USE_DOMAIN}, domain, '') AS domain,
|
||||
if({USE_USER_AGENT}, user_agent, '') AS user_agent,
|
||||
if({USE_VERSION_ID}, version_id, 0) AS version_id,
|
||||
if({USE_DEPENDENT_PROJECT_ID}, dependent_on_version_id, 0) AS dependent_on_version_id,
|
||||
if({USE_MONETIZED}, CAST(user_id != 0 AS Int8), -1) AS monetized,
|
||||
if({USE_COUNTRY}, country, '') AS country,
|
||||
if({USE_REASON}, reason, '') AS reason,
|
||||
@@ -233,12 +254,13 @@ const DOWNLOADS: &str = {
|
||||
AND downloads.project_id IN {PROJECT_IDS}
|
||||
AND (empty({FILTER_DOMAIN}) OR downloads.domain IN {FILTER_DOMAIN})
|
||||
AND (empty({FILTER_VERSION_ID}) OR downloads.version_id IN {FILTER_VERSION_ID})
|
||||
AND (empty({FILTER_DEPENDENT_ON_VERSION_ID}) OR downloads.dependent_on_version_id IN {FILTER_DEPENDENT_ON_VERSION_ID})
|
||||
AND ({FILTER_MONETIZED} = 2 OR CAST(downloads.user_id != 0 AS UInt8) = {FILTER_MONETIZED})
|
||||
AND (empty({FILTER_COUNTRY}) OR downloads.country IN {FILTER_COUNTRY})
|
||||
AND (empty({FILTER_REASON}) OR downloads.reason IN {FILTER_REASON})
|
||||
AND (empty({FILTER_GAME_VERSION}) OR downloads.game_version IN {FILTER_GAME_VERSION})
|
||||
AND (empty({FILTER_LOADER}) OR downloads.loader IN {FILTER_LOADER})
|
||||
GROUP BY bucket, source_project_id, project_id, domain, user_agent, version_id, monetized, country, reason, game_version, loader"
|
||||
GROUP BY bucket, source_project_id, project_id, domain, user_agent, version_id, dependent_on_version_id, monetized, country, reason, game_version, loader"
|
||||
)
|
||||
};
|
||||
|
||||
@@ -249,6 +271,7 @@ struct DownloadBucket {
|
||||
domain: Option<String>,
|
||||
user_agent: Option<DownloadSource>,
|
||||
version_id: Option<DBVersionId>,
|
||||
dependent_project_id: Option<DBProjectId>,
|
||||
monetized: Option<bool>,
|
||||
country: Option<String>,
|
||||
reason: Option<DownloadReason>,
|
||||
@@ -256,12 +279,79 @@ struct DownloadBucket {
|
||||
loader: Option<String>,
|
||||
}
|
||||
|
||||
async fn fetch_dependent_on_version_filter(
|
||||
metrics: &Metrics<ProjectDownloadsField, ProjectDownloadsFilters>,
|
||||
pool: &PgPool,
|
||||
) -> Result<Vec<VersionId>, ApiError> {
|
||||
if metrics.filter_by.dependent_project_id.is_empty() {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let project_ids = metrics
|
||||
.filter_by
|
||||
.dependent_project_id
|
||||
.iter()
|
||||
.map(|id| DBProjectId::from(*id).0)
|
||||
.collect::<Vec<_>>();
|
||||
let versions = sqlx::query!(
|
||||
"
|
||||
SELECT id FROM versions
|
||||
WHERE mod_id = ANY($1)
|
||||
",
|
||||
&project_ids
|
||||
)
|
||||
.fetch_all(pool)
|
||||
.await
|
||||
.wrap_internal_err("failed to fetch dependent project versions")?;
|
||||
|
||||
Ok(versions
|
||||
.into_iter()
|
||||
.map(|version| DBVersionId(version.id).into())
|
||||
.collect())
|
||||
}
|
||||
|
||||
async fn fetch_dependent_version_projects(
|
||||
rows: &[DownloadRow],
|
||||
cx: &QueryClickhouseContext<'_>,
|
||||
) -> Result<HashMap<DBVersionId, DBProjectId>, ApiError> {
|
||||
let dependent_on_version_ids = rows
|
||||
.iter()
|
||||
.filter_map(|row| {
|
||||
(row.dependent_on_version_id.0 != 0)
|
||||
.then_some(row.dependent_on_version_id)
|
||||
})
|
||||
.collect::<HashSet<_>>();
|
||||
|
||||
if dependent_on_version_ids.is_empty() {
|
||||
return Ok(HashMap::new());
|
||||
}
|
||||
|
||||
let dependent_on_version_ids =
|
||||
dependent_on_version_ids.into_iter().collect::<Vec<_>>();
|
||||
let versions =
|
||||
DBVersion::get_many(&dependent_on_version_ids, cx.pool, cx.redis)
|
||||
.await?;
|
||||
|
||||
Ok(versions
|
||||
.into_iter()
|
||||
.map(|version| (version.inner.id, version.inner.project_id))
|
||||
.collect())
|
||||
}
|
||||
|
||||
pub(crate) async fn fetch(
|
||||
cx: &mut QueryClickhouseContext<'_>,
|
||||
metrics: &Metrics<ProjectDownloadsField, ProjectDownloadsFilters>,
|
||||
) -> Result<(), ApiError> {
|
||||
use ProjectDownloadsField as F;
|
||||
let uses = |field| metrics.bucket_by.contains(&field);
|
||||
let dependent_on_version_filter =
|
||||
fetch_dependent_on_version_filter(metrics, cx.pool).await?;
|
||||
if !metrics.filter_by.dependent_project_id.is_empty()
|
||||
&& dependent_on_version_filter.is_empty()
|
||||
{
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let use_columns = &[
|
||||
("use_project_id", uses(F::ProjectId)),
|
||||
("use_domain", uses(F::Domain)),
|
||||
@@ -270,6 +360,7 @@ pub(crate) async fn fetch(
|
||||
uses(F::UserAgent) || !metrics.filter_by.user_agent.is_empty(),
|
||||
),
|
||||
("use_version_id", uses(F::VersionId)),
|
||||
("use_dependent_project_id", uses(F::DependentProjectId)),
|
||||
("use_monetized", uses(F::Monetized)),
|
||||
("use_country", uses(F::Country)),
|
||||
("use_reason", uses(F::Reason)),
|
||||
@@ -290,6 +381,7 @@ pub(crate) async fn fetch(
|
||||
for filter_param in [
|
||||
ClickhouseFilterParam::String(&metrics.filter_by.domain),
|
||||
ClickhouseFilterParam::VersionId(&metrics.filter_by.version_id),
|
||||
ClickhouseFilterParam::VersionId(&dependent_on_version_filter),
|
||||
ClickhouseFilterParam::Bool(
|
||||
"filter_monetized",
|
||||
&metrics.filter_by.monetized,
|
||||
@@ -308,9 +400,17 @@ pub(crate) async fn fetch(
|
||||
.any(|(column_name, used)| *column_name == name && *used)
|
||||
};
|
||||
let mut cursor = query.fetch::<DownloadRow>()?;
|
||||
let mut buckets = HashMap::<DownloadBucket, u64>::new();
|
||||
let mut rows = Vec::new();
|
||||
|
||||
while let Some(row) = cursor.next().await? {
|
||||
rows.push(row);
|
||||
}
|
||||
|
||||
let dependent_version_projects =
|
||||
fetch_dependent_version_projects(&rows, cx).await?;
|
||||
let mut buckets = HashMap::<DownloadBucket, u64>::new();
|
||||
|
||||
for row in rows {
|
||||
let normalized_source = normalize_download_source(&row.user_agent);
|
||||
if !metrics.filter_by.user_agent.is_empty()
|
||||
&& !normalized_source.as_ref().is_some_and(|source| {
|
||||
@@ -328,6 +428,15 @@ pub(crate) async fn fetch(
|
||||
.then_some(normalized_source)
|
||||
.flatten(),
|
||||
version_id: uses_column("use_version_id").then_some(row.version_id),
|
||||
dependent_project_id: if uses(F::DependentProjectId)
|
||||
&& row.dependent_on_version_id.0 != 0
|
||||
{
|
||||
dependent_version_projects
|
||||
.get(&row.dependent_on_version_id)
|
||||
.copied()
|
||||
} else {
|
||||
None
|
||||
},
|
||||
monetized: if uses_column("use_monetized") {
|
||||
match row.monetized {
|
||||
0 => Some(false),
|
||||
@@ -382,6 +491,9 @@ pub(crate) async fn fetch(
|
||||
version_id: key
|
||||
.version_id
|
||||
.and_then(none_if_zero_version_id),
|
||||
dependent_project_id: key
|
||||
.dependent_project_id
|
||||
.map(Into::into),
|
||||
monetized: key.monetized,
|
||||
country: key.country,
|
||||
reason: key.reason,
|
||||
@@ -468,6 +580,7 @@ static DOWNLOAD_SOURCE_PATTERNS: LazyLock<Vec<(Regex, DownloadSourcePattern)>> =
|
||||
(r"^unsup", P::Named("unsup")),
|
||||
(r"nothub/mrpack-install", P::Named("mrpack-install")),
|
||||
(r"^(packwiz-installer|packwiz/)", P::Named("Packwiz")),
|
||||
(r"^mrpack4server", P::Named("mrpack4server")),
|
||||
(
|
||||
r"^(Mozilla/|Chrome/|Chromium/|Firefox/|Safari/|AppleWebKit/|Edg/|OPR/)",
|
||||
P::Website,
|
||||
|
||||
@@ -24,7 +24,8 @@ use serde::{Deserialize, Serialize};
|
||||
|
||||
use crate::{
|
||||
auth::{
|
||||
AuthenticationError, checks::filter_visible_version_ids,
|
||||
AuthenticationError,
|
||||
checks::{filter_visible_project_ids, filter_visible_version_ids},
|
||||
get_user_from_headers,
|
||||
},
|
||||
database::{
|
||||
@@ -42,7 +43,7 @@ use crate::{
|
||||
projects::ProjectStatus,
|
||||
teams::ProjectPermissions,
|
||||
threads::MessageBody,
|
||||
v3::analytics::DownloadReason,
|
||||
v3::{analytics::DownloadReason, projects::Project},
|
||||
},
|
||||
queue::session::AuthQueue,
|
||||
routes::ApiError,
|
||||
@@ -127,6 +128,9 @@ pub struct GetResponse {
|
||||
/// time interval of metrics collection. The number of slices is determined
|
||||
/// by [`GetRequest::time_range`].
|
||||
pub metrics: Vec<TimeSlice>,
|
||||
/// Project metadata for projects referenced in the response metrics.
|
||||
#[serde(default)]
|
||||
pub projects: HashMap<ProjectId, Project>,
|
||||
/// List of events associated with projects that were requested.
|
||||
pub project_events: Vec<ProjectAnalyticsEvent>,
|
||||
}
|
||||
@@ -318,6 +322,8 @@ pub async fn fetch_analytics(
|
||||
|
||||
let mut query_clickhouse_cx = QueryClickhouseContext {
|
||||
clickhouse: &clickhouse,
|
||||
pool: &pool,
|
||||
redis: &redis,
|
||||
req: &req,
|
||||
time_slices: &mut time_slices,
|
||||
project_ids: &project_ids,
|
||||
@@ -392,8 +398,12 @@ pub async fn fetch_analytics(
|
||||
.await?;
|
||||
}
|
||||
|
||||
let projects =
|
||||
fetch_response_projects(&mut time_slices, &user, &pool, &redis).await?;
|
||||
|
||||
Ok(web::Json(GetResponse {
|
||||
metrics: time_slices,
|
||||
projects,
|
||||
project_events,
|
||||
}))
|
||||
}
|
||||
@@ -462,6 +472,88 @@ pub(crate) fn normalize_loader_for_project(
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_response_projects(
|
||||
time_slices: &mut [TimeSlice],
|
||||
user: &crate::models::users::User,
|
||||
pool: &PgPool,
|
||||
redis: &RedisPool,
|
||||
) -> Result<HashMap<ProjectId, Project>, ApiError> {
|
||||
let mut project_ids = HashSet::<DBProjectId>::new();
|
||||
|
||||
for time_slice in &*time_slices {
|
||||
for data in &time_slice.0 {
|
||||
let AnalyticsData::Project(project) = data else {
|
||||
continue;
|
||||
};
|
||||
|
||||
let source_project_id = DBProjectId::from(project.source_project);
|
||||
if source_project_id.0 != 0 {
|
||||
project_ids.insert(source_project_id);
|
||||
}
|
||||
if let ProjectMetrics::Downloads(downloads) = &project.metrics
|
||||
&& let Some(dependent_project_id) =
|
||||
downloads.dependent_project_id
|
||||
{
|
||||
project_ids.insert(dependent_project_id.into());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let project_ids = project_ids.into_iter().collect::<Vec<_>>();
|
||||
let projects = DBProject::get_many_ids(&project_ids, pool, redis).await?;
|
||||
let visible_project_ids = filter_visible_project_ids(
|
||||
projects.iter().map(|project| &project.inner).collect(),
|
||||
&Some(user.clone()),
|
||||
pool,
|
||||
false,
|
||||
)
|
||||
.await?
|
||||
.into_iter()
|
||||
.collect::<HashSet<_>>();
|
||||
|
||||
filter_response_project_ids(time_slices, &visible_project_ids);
|
||||
|
||||
Ok(projects
|
||||
.into_iter()
|
||||
.filter(|project| visible_project_ids.contains(&project.inner.id))
|
||||
.map(|project| {
|
||||
let project_id = project.inner.id.into();
|
||||
(project_id, Project::from(project))
|
||||
})
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn filter_response_project_ids(
|
||||
time_slices: &mut [TimeSlice],
|
||||
visible_project_ids: &HashSet<DBProjectId>,
|
||||
) {
|
||||
for time_slice in time_slices {
|
||||
time_slice.0.retain_mut(|data| {
|
||||
let AnalyticsData::Project(project) = data else {
|
||||
return true;
|
||||
};
|
||||
|
||||
let source_project_id = DBProjectId::from(project.source_project);
|
||||
if source_project_id.0 != 0
|
||||
&& !visible_project_ids.contains(&source_project_id)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if let ProjectMetrics::Downloads(downloads) = &mut project.metrics
|
||||
&& let Some(dependent_project_id) =
|
||||
downloads.dependent_project_id
|
||||
&& !visible_project_ids
|
||||
.contains(&DBProjectId::from(dependent_project_id))
|
||||
{
|
||||
downloads.dependent_project_id = None;
|
||||
}
|
||||
|
||||
true
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_project_status_change_events(
|
||||
project_ids: &[DBProjectId],
|
||||
time_range: &TimeRange,
|
||||
@@ -516,6 +608,8 @@ async fn fetch_project_status_change_events(
|
||||
|
||||
pub(crate) struct QueryClickhouseContext<'a> {
|
||||
pub(crate) clickhouse: &'a clickhouse::Client,
|
||||
pub(crate) pool: &'a PgPool,
|
||||
pub(crate) redis: &'a RedisPool,
|
||||
pub(crate) req: &'a GetRequest,
|
||||
pub(crate) time_slices: &'a mut [TimeSlice],
|
||||
pub(crate) project_ids: &'a [DBProjectId],
|
||||
@@ -875,6 +969,7 @@ mod tests {
|
||||
}),
|
||||
})]),
|
||||
],
|
||||
projects: HashMap::new(),
|
||||
project_events: vec![],
|
||||
};
|
||||
let target = json!({
|
||||
@@ -901,6 +996,7 @@ mod tests {
|
||||
}
|
||||
]
|
||||
],
|
||||
"projects": {},
|
||||
"project_events": []
|
||||
});
|
||||
|
||||
|
||||
@@ -315,6 +315,7 @@ pub async fn generate_pack_from_version_id(
|
||||
reason,
|
||||
game_version: profile.game_version.clone(),
|
||||
loader: profile.loader.as_str().to_string(),
|
||||
dependent_on: Some(version_id.clone()),
|
||||
};
|
||||
|
||||
let file = fetch_advanced(
|
||||
|
||||
@@ -387,8 +387,8 @@ pub async fn install_zipped_mrpack_files(
|
||||
profile_path: profile_path.clone(),
|
||||
pack_name: pack.name.clone(),
|
||||
icon,
|
||||
pack_id: project_id,
|
||||
pack_version: version_id,
|
||||
pack_id: project_id.clone(),
|
||||
pack_version: version_id.clone(),
|
||||
},
|
||||
100.0,
|
||||
"Downloading modpack",
|
||||
@@ -409,6 +409,7 @@ pub async fn install_zipped_mrpack_files(
|
||||
reason,
|
||||
game_version: profile.game_version.clone(),
|
||||
loader: profile.loader.as_str().to_string(),
|
||||
dependent_on: version_id.clone(),
|
||||
};
|
||||
|
||||
let num_files = pack.files.len();
|
||||
|
||||
@@ -462,6 +462,7 @@ pub async fn update_project(
|
||||
profile_path,
|
||||
update_version,
|
||||
fetch::DownloadReason::Update,
|
||||
None,
|
||||
&state.pool,
|
||||
&state.fetch_semaphore,
|
||||
&state.io_semaphore,
|
||||
@@ -503,6 +504,7 @@ pub async fn add_project_from_version(
|
||||
profile_path: &str,
|
||||
version_id: &str,
|
||||
reason: fetch::DownloadReason,
|
||||
dependent_on_version_id: Option<String>,
|
||||
) -> crate::Result<String> {
|
||||
let state = State::get().await?;
|
||||
|
||||
@@ -510,6 +512,7 @@ pub async fn add_project_from_version(
|
||||
profile_path,
|
||||
version_id,
|
||||
reason,
|
||||
dependent_on_version_id,
|
||||
&state.pool,
|
||||
&state.fetch_semaphore,
|
||||
&state.io_semaphore,
|
||||
|
||||
@@ -888,6 +888,7 @@ async fn get_modpack_identifiers(
|
||||
reason: DownloadReason::Modpack,
|
||||
game_version: profile.game_version.clone(),
|
||||
loader: profile.loader.as_str().to_string(),
|
||||
dependent_on: Some(version_id.to_string()),
|
||||
};
|
||||
|
||||
let mrpack_bytes = fetch_mirrors(
|
||||
|
||||
@@ -1336,6 +1336,7 @@ impl Profile {
|
||||
profile_path: &str,
|
||||
version_id: &str,
|
||||
reason: util::fetch::DownloadReason,
|
||||
dependent_on_version_id: Option<String>,
|
||||
pool: &SqlitePool,
|
||||
fetch_semaphore: &FetchSemaphore,
|
||||
io_semaphore: &IoSemaphore,
|
||||
@@ -1352,6 +1353,7 @@ impl Profile {
|
||||
reason,
|
||||
game_version: profile.game_version.clone(),
|
||||
loader: profile.loader.as_str().to_string(),
|
||||
dependent_on: dependent_on_version_id,
|
||||
};
|
||||
|
||||
let version =
|
||||
|
||||
@@ -35,6 +35,7 @@ pub struct DownloadMeta {
|
||||
pub reason: DownloadReason,
|
||||
pub game_version: String,
|
||||
pub loader: String,
|
||||
pub dependent_on: Option<String>,
|
||||
}
|
||||
|
||||
impl DownloadMeta {
|
||||
|
||||
Reference in New Issue
Block a user