You've already forked AstralRinth
forked from didirus/AstralRinth
Add launcher analytics (#661)
* Add more analytics * finish hydra move * Finish websocket flow * add minecraft account flow * Finish playtime vals + payout automation
This commit is contained in:
@@ -1,7 +1,6 @@
|
||||
use actix_web::{get, HttpResponse};
|
||||
use actix_web::HttpResponse;
|
||||
use serde_json::json;
|
||||
|
||||
#[get("/")]
|
||||
pub async fn index_get() -> HttpResponse {
|
||||
let data = json!({
|
||||
"name": "modrinth-labrinth",
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
use crate::file_hosting::FileHostingError;
|
||||
use crate::util::cors::default_cors;
|
||||
use actix_web::http::StatusCode;
|
||||
use actix_web::{web, HttpResponse};
|
||||
use futures::FutureExt;
|
||||
@@ -14,11 +15,21 @@ mod updates;
|
||||
pub use self::not_found::not_found;
|
||||
|
||||
pub fn root_config(cfg: &mut web::ServiceConfig) {
|
||||
cfg.service(index::index_get);
|
||||
cfg.service(web::scope("maven").configure(maven::config));
|
||||
cfg.service(web::scope("updates").configure(updates::config));
|
||||
cfg.route("", web::get().wrap(default_cors()).to(index::index_get));
|
||||
cfg.service(
|
||||
web::scope("api/v1").wrap_fn(|req, _srv| {
|
||||
web::scope("maven")
|
||||
.wrap(default_cors())
|
||||
.configure(maven::config),
|
||||
);
|
||||
cfg.service(
|
||||
web::scope("updates")
|
||||
.wrap(default_cors())
|
||||
.configure(updates::config),
|
||||
);
|
||||
cfg.service(
|
||||
web::scope("api/v1")
|
||||
.wrap(default_cors())
|
||||
.wrap_fn(|req, _srv| {
|
||||
async {
|
||||
Ok(req.into_response(
|
||||
HttpResponse::Gone()
|
||||
@@ -40,6 +51,8 @@ pub enum ApiError {
|
||||
Database(#[from] crate::database::models::DatabaseError),
|
||||
#[error("Database Error: {0}")]
|
||||
SqlxDatabase(#[from] sqlx::Error),
|
||||
#[error("Clickhouse Error: {0}")]
|
||||
Clickhouse(#[from] clickhouse::error::Error),
|
||||
#[error("Internal server error: {0}")]
|
||||
Xml(String),
|
||||
#[error("Deserialization error: {0}")]
|
||||
@@ -56,8 +69,6 @@ pub enum ApiError {
|
||||
Search(#[from] meilisearch_sdk::errors::Error),
|
||||
#[error("Indexing Error: {0}")]
|
||||
Indexing(#[from] crate::search::indexing::IndexingError),
|
||||
#[error("Ariadne Error: {0}")]
|
||||
Analytics(String),
|
||||
#[error("Payments Error: {0}")]
|
||||
Payments(String),
|
||||
#[error("Discord Error: {0}")]
|
||||
@@ -82,6 +93,7 @@ impl actix_web::ResponseError for ApiError {
|
||||
ApiError::Env(..) => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
ApiError::Database(..) => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
ApiError::SqlxDatabase(..) => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
ApiError::Clickhouse(..) => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
ApiError::Authentication(..) => StatusCode::UNAUTHORIZED,
|
||||
ApiError::CustomAuthentication(..) => StatusCode::UNAUTHORIZED,
|
||||
ApiError::Xml(..) => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
@@ -91,7 +103,6 @@ impl actix_web::ResponseError for ApiError {
|
||||
ApiError::FileHosting(..) => StatusCode::INTERNAL_SERVER_ERROR,
|
||||
ApiError::InvalidInput(..) => StatusCode::BAD_REQUEST,
|
||||
ApiError::Validation(..) => StatusCode::BAD_REQUEST,
|
||||
ApiError::Analytics(..) => StatusCode::FAILED_DEPENDENCY,
|
||||
ApiError::Payments(..) => StatusCode::FAILED_DEPENDENCY,
|
||||
ApiError::Discord(..) => StatusCode::FAILED_DEPENDENCY,
|
||||
ApiError::Turnstile => StatusCode::BAD_REQUEST,
|
||||
@@ -118,7 +129,6 @@ impl actix_web::ResponseError for ApiError {
|
||||
ApiError::FileHosting(..) => "file_hosting_error",
|
||||
ApiError::InvalidInput(..) => "invalid_input",
|
||||
ApiError::Validation(..) => "invalid_input",
|
||||
ApiError::Analytics(..) => "analytics_error",
|
||||
ApiError::Payments(..) => "payments_error",
|
||||
ApiError::Discord(..) => "discord_error",
|
||||
ApiError::Turnstile => "turnstile_error",
|
||||
@@ -127,6 +137,7 @@ impl actix_web::ResponseError for ApiError {
|
||||
ApiError::PasswordHashing(..) => "password_hashing_error",
|
||||
ApiError::PasswordStrengthCheck(..) => "strength_check_error",
|
||||
ApiError::Mail(..) => "mail_error",
|
||||
ApiError::Clickhouse(..) => "clickhouse_error",
|
||||
},
|
||||
description: &self.to_string(),
|
||||
})
|
||||
|
||||
@@ -1,23 +1,24 @@
|
||||
use crate::database::models::{User, UserId};
|
||||
use crate::auth::validate::get_user_record_from_bearer_token;
|
||||
use crate::models::analytics::Download;
|
||||
use crate::models::ids::ProjectId;
|
||||
use crate::models::projects::MonetizationStatus;
|
||||
use crate::models::pats::Scopes;
|
||||
use crate::queue::analytics::AnalyticsQueue;
|
||||
use crate::queue::maxmind::MaxMindIndexer;
|
||||
use crate::queue::session::AuthQueue;
|
||||
use crate::routes::ApiError;
|
||||
use crate::util::guards::admin_key_guard;
|
||||
use crate::DownloadQueue;
|
||||
use actix_web::{patch, post, web, HttpResponse};
|
||||
use chrono::{DateTime, SecondsFormat, Utc};
|
||||
use rust_decimal::Decimal;
|
||||
use actix_web::{patch, web, HttpRequest, HttpResponse};
|
||||
use chrono::Utc;
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use sqlx::PgPool;
|
||||
use std::collections::HashMap;
|
||||
use std::net::Ipv4Addr;
|
||||
use std::sync::Arc;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub fn config(cfg: &mut web::ServiceConfig) {
|
||||
cfg.service(
|
||||
web::scope("admin")
|
||||
.service(count_download)
|
||||
.service(process_payout),
|
||||
);
|
||||
cfg.service(web::scope("admin").service(count_download));
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -32,11 +33,28 @@ pub struct DownloadBody {
|
||||
|
||||
// This is an internal route, cannot be used without key
|
||||
#[patch("/_count-download", guard = "admin_key_guard")]
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn count_download(
|
||||
req: HttpRequest,
|
||||
pool: web::Data<PgPool>,
|
||||
redis: web::Data<deadpool_redis::Pool>,
|
||||
maxmind: web::Data<Arc<MaxMindIndexer>>,
|
||||
analytics_queue: web::Data<Arc<AnalyticsQueue>>,
|
||||
session_queue: web::Data<AuthQueue>,
|
||||
download_body: web::Json<DownloadBody>,
|
||||
download_queue: web::Data<DownloadQueue>,
|
||||
) -> Result<HttpResponse, ApiError> {
|
||||
let token = download_body
|
||||
.headers
|
||||
.iter()
|
||||
.find(|x| x.0.to_lowercase() == "authorization")
|
||||
.map(|x| &**x.1);
|
||||
|
||||
let user = get_user_record_from_bearer_token(&req, token, &**pool, &redis, &session_queue)
|
||||
.await
|
||||
.ok()
|
||||
.flatten();
|
||||
|
||||
let project_id: crate::database::models::ids::ProjectId = download_body.project_id.into();
|
||||
|
||||
let id_option = crate::models::ids::base62_impl::parse_base62(&download_body.version_name)
|
||||
@@ -83,322 +101,44 @@ pub async fn count_download(
|
||||
.await;
|
||||
}
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let url = url::Url::parse(&download_body.url)
|
||||
.map_err(|_| ApiError::InvalidInput("invalid download URL specified!".to_string()))?;
|
||||
|
||||
client
|
||||
.post(format!("{}download", dotenvy::var("ARIADNE_URL")?))
|
||||
.header("Modrinth-Admin", dotenvy::var("ARIADNE_ADMIN_KEY")?)
|
||||
.json(&json!({
|
||||
"ip": download_body.ip,
|
||||
"url": download_body.url,
|
||||
"project_id": download_body.project_id,
|
||||
"version_id": crate::models::projects::VersionId(version_id as u64).to_string(),
|
||||
"headers": download_body.headers
|
||||
}))
|
||||
.send()
|
||||
.await
|
||||
.ok();
|
||||
|
||||
Ok(HttpResponse::NoContent().body(""))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct PayoutData {
|
||||
amount: Decimal,
|
||||
date: DateTime<Utc>,
|
||||
}
|
||||
|
||||
#[post("/_process_payout", guard = "admin_key_guard")]
|
||||
pub async fn process_payout(
|
||||
pool: web::Data<PgPool>,
|
||||
redis: web::Data<deadpool_redis::Pool>,
|
||||
data: web::Json<PayoutData>,
|
||||
) -> Result<HttpResponse, ApiError> {
|
||||
let start: DateTime<Utc> = DateTime::from_utc(
|
||||
data.date
|
||||
.date_naive()
|
||||
.and_hms_nano_opt(0, 0, 0, 0)
|
||||
.unwrap_or_default(),
|
||||
Utc,
|
||||
);
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let mut transaction = pool.begin().await?;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct PayoutMultipliers {
|
||||
sum: u64,
|
||||
values: HashMap<String, u64>,
|
||||
}
|
||||
|
||||
let multipliers: PayoutMultipliers = client
|
||||
.get(format!("{}multipliers", dotenvy::var("ARIADNE_URL")?,))
|
||||
.header("Modrinth-Admin", dotenvy::var("ARIADNE_ADMIN_KEY")?)
|
||||
.query(&[(
|
||||
"start_date",
|
||||
start.to_rfc3339_opts(SecondsFormat::Nanos, true),
|
||||
)])
|
||||
.send()
|
||||
.await
|
||||
.map_err(|_| ApiError::Analytics("Error while fetching payout multipliers!".to_string()))?
|
||||
.json()
|
||||
.await
|
||||
.map_err(|_| {
|
||||
ApiError::Analytics("Error while deserializing payout multipliers!".to_string())
|
||||
})?;
|
||||
|
||||
struct Project {
|
||||
project_type: String,
|
||||
// user_id, payouts_split
|
||||
team_members: Vec<(i64, Decimal)>,
|
||||
// user_id, payouts_split, actual_project_id
|
||||
split_team_members: Vec<(i64, Decimal, i64)>,
|
||||
}
|
||||
|
||||
let mut projects_map: HashMap<i64, Project> = HashMap::new();
|
||||
|
||||
use futures::TryStreamExt;
|
||||
|
||||
sqlx::query!(
|
||||
"
|
||||
SELECT m.id id, tm.user_id user_id, tm.payouts_split payouts_split, pt.name project_type
|
||||
FROM mods m
|
||||
INNER JOIN team_members tm on m.team_id = tm.team_id AND tm.accepted = TRUE
|
||||
INNER JOIN project_types pt ON pt.id = m.project_type
|
||||
WHERE m.id = ANY($1) AND m.monetization_status = $2
|
||||
",
|
||||
&multipliers
|
||||
.values
|
||||
.keys()
|
||||
.flat_map(|x| x.parse::<i64>().ok())
|
||||
.collect::<Vec<i64>>(),
|
||||
MonetizationStatus::Monetized.as_str(),
|
||||
)
|
||||
.fetch_many(&mut *transaction)
|
||||
.try_for_each(|e| {
|
||||
if let Some(row) = e.right() {
|
||||
if let Some(project) = projects_map.get_mut(&row.id) {
|
||||
project.team_members.push((row.user_id, row.payouts_split));
|
||||
} else {
|
||||
projects_map.insert(
|
||||
row.id,
|
||||
Project {
|
||||
project_type: row.project_type,
|
||||
team_members: vec![(row.user_id, row.payouts_split)],
|
||||
split_team_members: Default::default(),
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
futures::future::ready(Ok(()))
|
||||
})
|
||||
.await?;
|
||||
|
||||
// Specific Payout Conditions (ex: modpack payout split)
|
||||
let mut projects_split_dependencies = Vec::new();
|
||||
|
||||
for (id, project) in &projects_map {
|
||||
if project.project_type == "modpack" {
|
||||
projects_split_dependencies.push(*id);
|
||||
}
|
||||
}
|
||||
|
||||
if !projects_split_dependencies.is_empty() {
|
||||
// (dependent_id, (dependency_id, times_depended))
|
||||
let mut project_dependencies: HashMap<i64, Vec<(i64, i64)>> = HashMap::new();
|
||||
// dependency_ids to fetch team members from
|
||||
let mut fetch_team_members: Vec<i64> = Vec::new();
|
||||
|
||||
sqlx::query!(
|
||||
"
|
||||
SELECT mv.mod_id, m.id, COUNT(m.id) times_depended FROM versions mv
|
||||
INNER JOIN dependencies d ON d.dependent_id = mv.id
|
||||
INNER JOIN versions v ON d.dependency_id = v.id
|
||||
INNER JOIN mods m ON v.mod_id = m.id OR d.mod_dependency_id = m.id
|
||||
WHERE mv.mod_id = ANY($1)
|
||||
group by mv.mod_id, m.id;
|
||||
",
|
||||
&projects_split_dependencies
|
||||
)
|
||||
.fetch_many(&mut *transaction)
|
||||
.try_for_each(|e| {
|
||||
if let Some(row) = e.right() {
|
||||
fetch_team_members.push(row.id);
|
||||
|
||||
if let Some(project) = project_dependencies.get_mut(&row.mod_id) {
|
||||
project.push((row.id, row.times_depended.unwrap_or(0)));
|
||||
} else {
|
||||
project_dependencies
|
||||
.insert(row.mod_id, vec![(row.id, row.times_depended.unwrap_or(0))]);
|
||||
}
|
||||
}
|
||||
|
||||
futures::future::ready(Ok(()))
|
||||
})
|
||||
.await?;
|
||||
|
||||
// (project_id, (user_id, payouts_split))
|
||||
let mut team_members: HashMap<i64, Vec<(i64, Decimal)>> = HashMap::new();
|
||||
|
||||
sqlx::query!(
|
||||
"
|
||||
SELECT m.id id, tm.user_id user_id, tm.payouts_split payouts_split
|
||||
FROM mods m
|
||||
INNER JOIN team_members tm on m.team_id = tm.team_id AND tm.accepted = TRUE
|
||||
WHERE m.id = ANY($1)
|
||||
",
|
||||
&*fetch_team_members
|
||||
)
|
||||
.fetch_many(&mut *transaction)
|
||||
.try_for_each(|e| {
|
||||
if let Some(row) = e.right() {
|
||||
if let Some(project) = team_members.get_mut(&row.id) {
|
||||
project.push((row.user_id, row.payouts_split));
|
||||
} else {
|
||||
team_members.insert(row.id, vec![(row.user_id, row.payouts_split)]);
|
||||
}
|
||||
}
|
||||
|
||||
futures::future::ready(Ok(()))
|
||||
})
|
||||
.await?;
|
||||
|
||||
for (project_id, dependencies) in project_dependencies {
|
||||
let dep_sum: i64 = dependencies.iter().map(|x| x.1).sum();
|
||||
|
||||
let project = projects_map.get_mut(&project_id);
|
||||
|
||||
if let Some(project) = project {
|
||||
if dep_sum > 0 {
|
||||
for dependency in dependencies {
|
||||
let project_multiplier: Decimal =
|
||||
Decimal::from(dependency.1) / Decimal::from(dep_sum);
|
||||
|
||||
if let Some(members) = team_members.get(&dependency.0) {
|
||||
let members_sum: Decimal = members.iter().map(|x| x.1).sum();
|
||||
|
||||
if members_sum > Decimal::ZERO {
|
||||
for member in members {
|
||||
let member_multiplier: Decimal = member.1 / members_sum;
|
||||
project.split_team_members.push((
|
||||
member.0,
|
||||
member_multiplier * project_multiplier,
|
||||
project_id,
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (id, project) in projects_map {
|
||||
if let Some(value) = &multipliers.values.get(&id.to_string()) {
|
||||
let project_multiplier: Decimal =
|
||||
Decimal::from(**value) / Decimal::from(multipliers.sum);
|
||||
|
||||
let default_split_given = Decimal::ONE;
|
||||
let split_given = Decimal::ONE / Decimal::from(5);
|
||||
let split_retention = Decimal::from(4) / Decimal::from(5);
|
||||
|
||||
let sum_splits: Decimal = project.team_members.iter().map(|x| x.1).sum();
|
||||
let sum_tm_splits: Decimal = project.split_team_members.iter().map(|x| x.1).sum();
|
||||
|
||||
let mut clear_cache_users = Vec::new();
|
||||
|
||||
if sum_splits > Decimal::ZERO {
|
||||
for (user_id, split) in project.team_members {
|
||||
let payout: Decimal = data.amount
|
||||
* project_multiplier
|
||||
* (split / sum_splits)
|
||||
* (if !project.split_team_members.is_empty() {
|
||||
&split_given
|
||||
} else {
|
||||
&default_split_given
|
||||
});
|
||||
|
||||
if payout > Decimal::ZERO {
|
||||
sqlx::query!(
|
||||
"
|
||||
INSERT INTO payouts_values (user_id, mod_id, amount, created)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
",
|
||||
user_id,
|
||||
id,
|
||||
payout,
|
||||
start
|
||||
)
|
||||
.execute(&mut *transaction)
|
||||
.await?;
|
||||
|
||||
sqlx::query!(
|
||||
"
|
||||
UPDATE users
|
||||
SET balance = balance + $1
|
||||
WHERE id = $2
|
||||
",
|
||||
payout,
|
||||
user_id
|
||||
)
|
||||
.execute(&mut *transaction)
|
||||
.await?;
|
||||
clear_cache_users.push(user_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if sum_tm_splits > Decimal::ZERO {
|
||||
for (user_id, split, project_id) in project.split_team_members {
|
||||
let payout: Decimal = data.amount
|
||||
* project_multiplier
|
||||
* (split / sum_tm_splits)
|
||||
* split_retention;
|
||||
|
||||
if payout > Decimal::ZERO {
|
||||
sqlx::query!(
|
||||
"
|
||||
INSERT INTO payouts_values (user_id, mod_id, amount, created)
|
||||
VALUES ($1, $2, $3, $4)
|
||||
",
|
||||
user_id,
|
||||
project_id,
|
||||
payout,
|
||||
start
|
||||
)
|
||||
.execute(&mut *transaction)
|
||||
.await?;
|
||||
|
||||
sqlx::query!(
|
||||
"
|
||||
UPDATE users
|
||||
SET balance = balance + $1
|
||||
WHERE id = $2
|
||||
",
|
||||
payout,
|
||||
user_id
|
||||
)
|
||||
.execute(&mut *transaction)
|
||||
.await?;
|
||||
clear_cache_users.push(user_id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
User::clear_caches(
|
||||
&clear_cache_users
|
||||
.into_iter()
|
||||
.map(|x| (UserId(x), None))
|
||||
.collect::<Vec<_>>(),
|
||||
&redis,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
}
|
||||
|
||||
transaction.commit().await?;
|
||||
let ip = super::analytics::convert_to_ip_v6(&download_body.ip)
|
||||
.unwrap_or_else(|_| Ipv4Addr::new(127, 0, 0, 1).to_ipv6_mapped());
|
||||
|
||||
analytics_queue
|
||||
.add_download(Download {
|
||||
id: Uuid::new_v4(),
|
||||
recorded: Utc::now().timestamp_nanos() / 100_000,
|
||||
domain: url.host_str().unwrap_or_default().to_string(),
|
||||
site_path: url.path().to_string(),
|
||||
user_id: user
|
||||
.and_then(|(scopes, x)| {
|
||||
if scopes.contains(Scopes::PERFORM_ANALYTICS) {
|
||||
Some(x.id.0 as u64)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.unwrap_or(0),
|
||||
project_id: project_id as u64,
|
||||
version_id: version_id as u64,
|
||||
ip,
|
||||
country: maxmind.query(ip).await.unwrap_or_default(),
|
||||
user_agent: download_body
|
||||
.headers
|
||||
.get("user-agent")
|
||||
.cloned()
|
||||
.unwrap_or_default(),
|
||||
headers: download_body
|
||||
.headers
|
||||
.clone()
|
||||
.into_iter()
|
||||
.filter(|x| !super::analytics::FILTERED_HEADERS.contains(&&*x.0.to_lowercase()))
|
||||
.collect(),
|
||||
})
|
||||
.await;
|
||||
|
||||
Ok(HttpResponse::NoContent().body(""))
|
||||
}
|
||||
|
||||
248
src/routes/v2/analytics.rs
Normal file
248
src/routes/v2/analytics.rs
Normal file
@@ -0,0 +1,248 @@
|
||||
use crate::auth::get_user_from_headers;
|
||||
use crate::models::analytics::{PageView, Playtime};
|
||||
use crate::models::pats::Scopes;
|
||||
use crate::queue::maxmind::MaxMindIndexer;
|
||||
use crate::queue::session::AuthQueue;
|
||||
use crate::routes::ApiError;
|
||||
use crate::util::env::parse_strings_from_var;
|
||||
use crate::AnalyticsQueue;
|
||||
use actix_cors::Cors;
|
||||
use actix_web::{post, web};
|
||||
use actix_web::{HttpRequest, HttpResponse};
|
||||
use chrono::Utc;
|
||||
use serde::Deserialize;
|
||||
use sqlx::PgPool;
|
||||
use std::collections::HashMap;
|
||||
use std::net::{AddrParseError, IpAddr, Ipv4Addr, Ipv6Addr};
|
||||
use std::sync::Arc;
|
||||
use url::Url;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub fn config(cfg: &mut web::ServiceConfig) {
|
||||
cfg.service(
|
||||
web::scope("v2/analytics")
|
||||
.wrap(
|
||||
Cors::default()
|
||||
.allowed_origin_fn(|origin, _req_head| {
|
||||
let allowed_origins =
|
||||
parse_strings_from_var("ANALYTICS_ALLOWED_ORIGINS").unwrap_or_default();
|
||||
|
||||
allowed_origins.contains(&"*".to_string())
|
||||
|| allowed_origins
|
||||
.contains(&origin.to_str().unwrap_or_default().to_string())
|
||||
})
|
||||
.allowed_methods(vec!["GET", "POST"])
|
||||
.allowed_headers(vec![
|
||||
actix_web::http::header::AUTHORIZATION,
|
||||
actix_web::http::header::ACCEPT,
|
||||
actix_web::http::header::CONTENT_TYPE,
|
||||
])
|
||||
.max_age(3600),
|
||||
)
|
||||
.service(page_view_ingest)
|
||||
.service(playtime_ingest),
|
||||
);
|
||||
}
|
||||
|
||||
pub const FILTERED_HEADERS: &[&str] = &[
|
||||
"authorization",
|
||||
"cookie",
|
||||
"modrinth-admin",
|
||||
// we already retrieve/use these elsewhere- so they are unneeded
|
||||
"user-agent",
|
||||
"cf-connecting-ip",
|
||||
"cf-ipcountry",
|
||||
"x-forwarded-for",
|
||||
"x-real-ip",
|
||||
// We don't need the information vercel provides from its headers
|
||||
"x-vercel-ip-city",
|
||||
"x-vercel-ip-timezone",
|
||||
"x-vercel-ip-longitude",
|
||||
"x-vercel-proxy-signature",
|
||||
"x-vercel-ip-country-region",
|
||||
"x-vercel-forwarded-for",
|
||||
"x-vercel-proxied-for",
|
||||
"x-vercel-proxy-signature-ts",
|
||||
"x-vercel-ip-latitude",
|
||||
"x-vercel-ip-country",
|
||||
];
|
||||
|
||||
pub fn convert_to_ip_v6(src: &str) -> Result<Ipv6Addr, AddrParseError> {
|
||||
let ip_addr: IpAddr = src.parse()?;
|
||||
|
||||
Ok(match ip_addr {
|
||||
IpAddr::V4(x) => x.to_ipv6_mapped(),
|
||||
IpAddr::V6(x) => x,
|
||||
})
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct UrlInput {
|
||||
url: String,
|
||||
}
|
||||
|
||||
//this route should be behind the cloudflare WAF to prevent non-browsers from calling it
|
||||
#[post("view")]
|
||||
pub async fn page_view_ingest(
|
||||
req: HttpRequest,
|
||||
maxmind: web::Data<Arc<MaxMindIndexer>>,
|
||||
analytics_queue: web::Data<Arc<AnalyticsQueue>>,
|
||||
session_queue: web::Data<AuthQueue>,
|
||||
url_input: web::Json<UrlInput>,
|
||||
pool: web::Data<PgPool>,
|
||||
redis: web::Data<deadpool_redis::Pool>,
|
||||
) -> Result<HttpResponse, ApiError> {
|
||||
let user = get_user_from_headers(&req, &**pool, &redis, &session_queue, None)
|
||||
.await
|
||||
.ok();
|
||||
let conn_info = req.connection_info().peer_addr().map(|x| x.to_string());
|
||||
|
||||
let url = Url::parse(&url_input.url)
|
||||
.map_err(|_| ApiError::InvalidInput("invalid page view URL specified!".to_string()))?;
|
||||
|
||||
let domain = url
|
||||
.host_str()
|
||||
.ok_or_else(|| ApiError::InvalidInput("invalid page view URL specified!".to_string()))?;
|
||||
|
||||
let allowed_origins = parse_strings_from_var("CORS_ALLOWED_ORIGINS").unwrap_or_default();
|
||||
if !(domain.ends_with(".modrinth.com")
|
||||
|| domain == "modrinth.com"
|
||||
|| allowed_origins.contains(&"*".to_string()))
|
||||
{
|
||||
return Err(ApiError::InvalidInput(
|
||||
"invalid page view URL specified!".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let headers = req
|
||||
.headers()
|
||||
.into_iter()
|
||||
.map(|(key, val)| {
|
||||
(
|
||||
key.to_string().to_lowercase(),
|
||||
val.to_str().unwrap_or_default().to_string(),
|
||||
)
|
||||
})
|
||||
.collect::<HashMap<String, String>>();
|
||||
|
||||
let ip = convert_to_ip_v6(if let Some(header) = headers.get("cf-connecting-ip") {
|
||||
header
|
||||
} else {
|
||||
conn_info.as_deref().unwrap_or_default()
|
||||
})
|
||||
.unwrap_or_else(|_| Ipv4Addr::new(127, 0, 0, 1).to_ipv6_mapped());
|
||||
|
||||
let mut view = PageView {
|
||||
id: Uuid::new_v4(),
|
||||
recorded: Utc::now().timestamp_nanos() / 100_000,
|
||||
domain: domain.to_string(),
|
||||
site_path: url.path().to_string(),
|
||||
user_id: 0,
|
||||
project_id: 0,
|
||||
ip,
|
||||
country: maxmind.query(ip).await.unwrap_or_default(),
|
||||
user_agent: headers.get("user-agent").cloned().unwrap_or_default(),
|
||||
headers: headers
|
||||
.into_iter()
|
||||
.filter(|x| !FILTERED_HEADERS.contains(&&*x.0))
|
||||
.collect(),
|
||||
};
|
||||
|
||||
if let Some(segments) = url.path_segments() {
|
||||
let segments_vec = segments.collect::<Vec<_>>();
|
||||
|
||||
if segments_vec.len() >= 2 {
|
||||
const PROJECT_TYPES: &[&str] = &[
|
||||
"mod",
|
||||
"modpack",
|
||||
"plugin",
|
||||
"resourcepack",
|
||||
"shader",
|
||||
"datapack",
|
||||
];
|
||||
|
||||
if PROJECT_TYPES.contains(&segments_vec[0]) {
|
||||
let project =
|
||||
crate::database::models::Project::get(segments_vec[1], &**pool, &redis).await?;
|
||||
|
||||
if let Some(project) = project {
|
||||
view.project_id = project.inner.id.0 as u64;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if let Some((_, user)) = user {
|
||||
view.user_id = user.id.0;
|
||||
}
|
||||
|
||||
analytics_queue.add_view(view).await;
|
||||
|
||||
Ok(HttpResponse::NoContent().body(""))
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct PlaytimeInput {
|
||||
seconds: u16,
|
||||
loader: String,
|
||||
game_version: String,
|
||||
parent: Option<crate::models::ids::VersionId>,
|
||||
}
|
||||
|
||||
#[post("playtime")]
|
||||
pub async fn playtime_ingest(
|
||||
req: HttpRequest,
|
||||
analytics_queue: web::Data<Arc<AnalyticsQueue>>,
|
||||
session_queue: web::Data<AuthQueue>,
|
||||
playtime_input: web::Json<HashMap<crate::models::ids::VersionId, PlaytimeInput>>,
|
||||
pool: web::Data<PgPool>,
|
||||
redis: web::Data<deadpool_redis::Pool>,
|
||||
) -> Result<HttpResponse, ApiError> {
|
||||
let (_, user) = get_user_from_headers(
|
||||
&req,
|
||||
&**pool,
|
||||
&redis,
|
||||
&session_queue,
|
||||
Some(&[Scopes::PERFORM_ANALYTICS]),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let playtimes = playtime_input.0;
|
||||
|
||||
if playtimes.len() > 2000 {
|
||||
return Err(ApiError::InvalidInput(
|
||||
"Too much playtime entered for version!".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let versions = crate::database::models::Version::get_many(
|
||||
&playtimes.iter().map(|x| (*x.0).into()).collect::<Vec<_>>(),
|
||||
&**pool,
|
||||
&redis,
|
||||
)
|
||||
.await?;
|
||||
|
||||
for (id, playtime) in playtimes {
|
||||
if playtime.seconds > 300 {
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(version) = versions.iter().find(|x| id == x.inner.id.into()) {
|
||||
analytics_queue
|
||||
.add_playtime(Playtime {
|
||||
id: Default::default(),
|
||||
recorded: Utc::now().timestamp_nanos() / 100_000,
|
||||
seconds: playtime.seconds,
|
||||
user_id: user.id.0,
|
||||
project_id: version.inner.id.0 as u64,
|
||||
version_id: version.inner.project_id.0 as u64,
|
||||
loader: playtime.loader,
|
||||
game_version: playtime.game_version,
|
||||
parent: playtime.parent.map(|x| x.0).unwrap_or(0),
|
||||
})
|
||||
.await;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(HttpResponse::NoContent().finish())
|
||||
}
|
||||
@@ -1,4 +1,5 @@
|
||||
mod admin;
|
||||
mod analytics;
|
||||
mod moderation;
|
||||
mod notifications;
|
||||
pub(crate) mod project_creation;
|
||||
@@ -14,10 +15,12 @@ mod version_file;
|
||||
mod versions;
|
||||
|
||||
pub use super::ApiError;
|
||||
use crate::util::cors::default_cors;
|
||||
|
||||
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
|
||||
cfg.service(
|
||||
actix_web::web::scope("v2")
|
||||
.wrap(default_cors())
|
||||
.configure(admin::config)
|
||||
.configure(crate::auth::session::config)
|
||||
.configure(crate::auth::flows::config)
|
||||
@@ -36,4 +39,6 @@ pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
|
||||
.configure(version_file::config)
|
||||
.configure(versions::config),
|
||||
);
|
||||
|
||||
cfg.configure(analytics::config);
|
||||
}
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
pub use super::ApiError;
|
||||
use crate::util::cors::default_cors;
|
||||
use actix_web::{web, HttpResponse};
|
||||
use serde_json::json;
|
||||
|
||||
pub fn config(cfg: &mut web::ServiceConfig) {
|
||||
cfg.service(web::scope("v3").route("", web::get().to(hello_world)));
|
||||
cfg.service(
|
||||
web::scope("v3")
|
||||
.wrap(default_cors())
|
||||
.route("", web::get().to(hello_world)),
|
||||
);
|
||||
}
|
||||
|
||||
pub async fn hello_world() -> Result<HttpResponse, ApiError> {
|
||||
|
||||
Reference in New Issue
Block a user