eightequal/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Fix clippy errors + lint, use turbo CI

Browse Source
This commit is contained in:
Jai A
2024-10-18 16:07:35 -07:00
parent 663ab83b08
commit 8dd955563e
186 changed files with 10615 additions and 6433 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
apps/labrinth/src/queue/analytics.rs
View File

@@ -36,12 +36,14 @@ impl AnalyticsQueue {
fn strip_ip(ip: Ipv6Addr) -> u64 {
if let Some(ip) = ip.to_ipv4_mapped() {
let octets = ip.octets();
u64::from_be_bytes([octets[0], octets[1], octets[2], octets[3], 0, 0, 0, 0])
u64::from_be_bytes([
octets[0], octets[1], octets[2], octets[3], 0, 0, 0, 0,
])
} else {
let octets = ip.octets();
u64::from_be_bytes([
octets[0], octets[1], octets[2], octets[3], octets[4], octets[5], octets[6],
octets[7],
octets[0], octets[1], octets[2], octets[3], octets[4],
octets[5], octets[6], octets[7],
])
}
}
@@ -98,7 +100,8 @@ impl AnalyticsQueue {
raw_views.push((views, true));
}
let mut redis = redis.pool.get().await.map_err(DatabaseError::RedisPool)?;
let mut redis =
redis.pool.get().await.map_err(DatabaseError::RedisPool)?;
let results = cmd("MGET")
.arg(
@@ -107,7 +110,7 @@ impl AnalyticsQueue {
.map(|x| format!("{}:{}-{}", VIEWS_NAMESPACE, x.0, x.1))
.collect::<Vec<_>>(),
)
.query_async::<_, Vec<Option<u32>>>(&mut redis)
.query_async::<Vec<Option<u32>>>(&mut redis)
.await
.map_err(DatabaseError::CacheError)?;
@@ -115,24 +118,25 @@ impl AnalyticsQueue {
for (idx, count) in results.into_iter().enumerate() {
let key = &views_keys[idx];
let new_count = if let Some((views, monetized)) = raw_views.get_mut(idx) {
if let Some(count) = count {
if count > 3 {
*monetized = false;
continue;
}
let new_count =
if let Some((views, monetized)) = raw_views.get_mut(idx) {
if let Some(count) = count {
if count > 3 {
*monetized = false;
continue;
}
if (count + views.len() as u32) > 3 {
*monetized = false;
}
if (count + views.len() as u32) > 3 {
*monetized = false;
}
count + (views.len() as u32)
count + (views.len() as u32)
} else {
views.len() as u32
}
} else {
views.len() as u32
}
} else {
1
};
1
};
pipe.atomic().set_ex(
format!("{}:{}-{}", VIEWS_NAMESPACE, key.0, key.1),
@@ -140,7 +144,7 @@ impl AnalyticsQueue {
6 * 60 * 60,
);
}
pipe.query_async(&mut *redis)
pipe.query_async::<()>(&mut *redis)
.await
.map_err(DatabaseError::CacheError)?;
@@ -163,21 +167,26 @@ impl AnalyticsQueue {
let mut downloads_keys = Vec::new();
let raw_downloads = DashMap::new();
for (index, (key, download)) in downloads_queue.into_iter().enumerate() {
for (index, (key, download)) in
downloads_queue.into_iter().enumerate()
{
downloads_keys.push(key);
raw_downloads.insert(index, download);
}
let mut redis = redis.pool.get().await.map_err(DatabaseError::RedisPool)?;
let mut redis =
redis.pool.get().await.map_err(DatabaseError::RedisPool)?;
let results = cmd("MGET")
.arg(
downloads_keys
.iter()
.map(|x| format!("{}:{}-{}", DOWNLOADS_NAMESPACE, x.0, x.1))
.map(|x| {
format!("{}:{}-{}", DOWNLOADS_NAMESPACE, x.0, x.1)
})
.collect::<Vec<_>>(),
)
.query_async::<_, Vec<Option<u32>>>(&mut redis)
.query_async::<Vec<Option<u32>>>(&mut redis)
.await
.map_err(DatabaseError::CacheError)?;
@@ -202,7 +211,7 @@ impl AnalyticsQueue {
6 * 60 * 60,
);
}
pipe.query_async(&mut *redis)
pipe.query_async::<()>(&mut *redis)
.await
.map_err(DatabaseError::CacheError)?;

13
apps/labrinth/src/queue/maxmind.rs
View File

@@ -46,11 +46,13 @@ impl MaxMindIndexer {
if let Ok(entries) = archive.entries() {
for mut file in entries.flatten() {
if let Ok(path) = file.header().path() {
if path.extension().and_then(|x| x.to_str()) == Some("mmdb") {
if path.extension().and_then(|x| x.to_str()) == Some("mmdb")
{
let mut buf = Vec::new();
file.read_to_end(&mut buf).unwrap();
let reader = maxminddb::Reader::from_source(buf).unwrap();
let reader =
maxminddb::Reader::from_source(buf).unwrap();
return Ok(Some(reader));
}
@@ -71,10 +73,9 @@ impl MaxMindIndexer {
let maxmind = self.reader.read().await;
if let Some(ref maxmind) = *maxmind {
maxmind
.lookup::<Country>(ip.into())
.ok()
.and_then(|x| x.country.and_then(|x| x.iso_code.map(|x| x.to_string())))
maxmind.lookup::<Country>(ip.into()).ok().and_then(|x| {
x.country.and_then(|x| x.iso_code.map(|x| x.to_string()))
})
} else {
None
}

16
apps/labrinth/src/queue/moderation.rs
View File

@@ -128,10 +128,14 @@ impl ModerationMessage {
pub fn header(&self) -> &'static str {
match self {
ModerationMessage::NoPrimaryFile => "No primary files",
ModerationMessage::PackFilesNotAllowed { .. } => "Copyrighted Content",
ModerationMessage::PackFilesNotAllowed { .. } => {
"Copyrighted Content"
}
ModerationMessage::MissingGalleryImage => "Missing Gallery Images",
ModerationMessage::MissingLicense => "Missing License",
ModerationMessage::MissingCustomLicenseUrl { .. } => "Missing License URL",
ModerationMessage::MissingCustomLicenseUrl { .. } => {
"Missing License URL"
}
ModerationMessage::NoSideTypes => "Missing Environment Information",
}
}
@@ -806,7 +810,9 @@ impl ApprovalType {
pub fn from_string(string: &str) -> Option<Self> {
match string {
"yes" => Some(ApprovalType::Yes),
"with-attribution-and-source" => Some(ApprovalType::WithAttributionAndSource),
"with-attribution-and-source" => {
Some(ApprovalType::WithAttributionAndSource)
}
"with-attribution" => Some(ApprovalType::WithAttribution),
"no" => Some(ApprovalType::No),
"permanent-no" => Some(ApprovalType::PermanentNo),
@@ -818,7 +824,9 @@ impl ApprovalType {
pub(crate) fn as_str(&self) -> &'static str {
match self {
ApprovalType::Yes => "yes",
ApprovalType::WithAttributionAndSource => "with-attribution-and-source",
ApprovalType::WithAttributionAndSource => {
"with-attribution-and-source"
}
ApprovalType::WithAttribution => "with-attribution",
ApprovalType::No => "no",
ApprovalType::PermanentNo => "permanent-no",

119
apps/labrinth/src/queue/payouts.rs
View File

@@ -1,5 +1,6 @@
use crate::models::payouts::{
PayoutDecimal, PayoutInterval, PayoutMethod, PayoutMethodFee, PayoutMethodType,
PayoutDecimal, PayoutInterval, PayoutMethod, PayoutMethodFee,
PayoutMethodType,
};
use crate::models::projects::MonetizationStatus;
use crate::routes::ApiError;
@@ -81,12 +82,17 @@ impl PayoutsQueue {
.form(&form)
.send()
.await
.map_err(|_| ApiError::Payments("Error while authenticating with PayPal".to_string()))?
.map_err(|_| {
ApiError::Payments(
"Error while authenticating with PayPal".to_string(),
)
})?
.json()
.await
.map_err(|_| {
ApiError::Payments(
"Error while authenticating with PayPal (deser error)".to_string(),
"Error while authenticating with PayPal (deser error)"
.to_string(),
)
})?;
@@ -114,7 +120,9 @@ impl PayoutsQueue {
if credentials.expires < Utc::now() {
drop(read);
self.refresh_token().await.map_err(|_| {
ApiError::Payments("Error while authenticating with PayPal".to_string())
ApiError::Payments(
"Error while authenticating with PayPal".to_string(),
)
})?
} else {
credentials.clone()
@@ -122,7 +130,9 @@ impl PayoutsQueue {
} else {
drop(read);
self.refresh_token().await.map_err(|_| {
ApiError::Payments("Error while authenticating with PayPal".to_string())
ApiError::Payments(
"Error while authenticating with PayPal".to_string(),
)
})?
};
@@ -138,7 +148,10 @@ impl PayoutsQueue {
)
.header(
"Authorization",
format!("{} {}", credentials.token_type, credentials.access_token),
format!(
"{} {}",
credentials.token_type, credentials.access_token
),
);
if let Some(body) = body {
@@ -149,15 +162,16 @@ impl PayoutsQueue {
.body(body);
}
let resp = request
.send()
.await
.map_err(|_| ApiError::Payments("could not communicate with PayPal".to_string()))?;
let resp = request.send().await.map_err(|_| {
ApiError::Payments("could not communicate with PayPal".to_string())
})?;
let status = resp.status();
let value = resp.json::<Value>().await.map_err(|_| {
ApiError::Payments("could not retrieve PayPal response body".to_string())
ApiError::Payments(
"could not retrieve PayPal response body".to_string(),
)
})?;
if !status.is_success() {
@@ -173,14 +187,18 @@ impl PayoutsQueue {
pub error_description: String,
}
if let Ok(error) = serde_json::from_value::<PayPalError>(value.clone()) {
if let Ok(error) =
serde_json::from_value::<PayPalError>(value.clone())
{
return Err(ApiError::Payments(format!(
"error name: {}, message: {}",
error.name, error.message
)));
}
if let Ok(error) = serde_json::from_value::<PayPalIdentityError>(value) {
if let Ok(error) =
serde_json::from_value::<PayPalIdentityError>(value)
{
return Err(ApiError::Payments(format!(
"error name: {}, message: {}",
error.error, error.error_description
@@ -216,15 +234,18 @@ impl PayoutsQueue {
request = request.json(&body);
}
let resp = request
.send()
.await
.map_err(|_| ApiError::Payments("could not communicate with Tremendous".to_string()))?;
let resp = request.send().await.map_err(|_| {
ApiError::Payments(
"could not communicate with Tremendous".to_string(),
)
})?;
let status = resp.status();
let value = resp.json::<Value>().await.map_err(|_| {
ApiError::Payments("could not retrieve Tremendous response body".to_string())
ApiError::Payments(
"could not retrieve Tremendous response body".to_string(),
)
})?;
if !status.is_success() {
@@ -235,12 +256,15 @@ impl PayoutsQueue {
message: String,
}
let err =
serde_json::from_value::<TremendousError>(array.clone()).map_err(|_| {
ApiError::Payments(
"could not retrieve Tremendous error json body".to_string(),
)
})?;
let err = serde_json::from_value::<TremendousError>(
array.clone(),
)
.map_err(|_| {
ApiError::Payments(
"could not retrieve Tremendous error json body"
.to_string(),
)
})?;
return Err(ApiError::Payments(err.message));
}
@@ -254,8 +278,12 @@ impl PayoutsQueue {
Ok(serde_json::from_value(value)?)
}
pub async fn get_payout_methods(&self) -> Result<Vec<PayoutMethod>, ApiError> {
async fn refresh_payout_methods(queue: &PayoutsQueue) -> Result<PayoutMethods, ApiError> {
pub async fn get_payout_methods(
&self,
) -> Result<Vec<PayoutMethod>, ApiError> {
async fn refresh_payout_methods(
queue: &PayoutsQueue,
) -> Result<PayoutMethods, ApiError> {
let mut options = queue.payout_options.write().await;
let mut methods = Vec::new();
@@ -304,7 +332,11 @@ impl PayoutsQueue {
}
let response = queue
.make_tremendous_request::<(), TremendousResponse>(Method::GET, "products", None)
.make_tremendous_request::<(), TremendousResponse>(
Method::GET,
"products",
None,
)
.await?;
for product in response.products {
@@ -361,7 +393,11 @@ impl PayoutsQueue {
id: product.id,
type_: PayoutMethodType::Tremendous,
name: product.name.clone(),
supported_countries: product.countries.into_iter().map(|x| x.abbr).collect(),
supported_countries: product
.countries
.into_iter()
.map(|x| x.abbr)
.collect(),
image_url: product
.images
.into_iter()
@@ -412,7 +448,8 @@ impl PayoutsQueue {
methods.push(method);
}
const UPRANK_IDS: &[&str] = &["ET0ZVETV5ILN", "Q24BD9EZ332JT", "UIL1ZYJU5MKN"];
const UPRANK_IDS: &[&str] =
&["ET0ZVETV5ILN", "Q24BD9EZ332JT", "UIL1ZYJU5MKN"];
const DOWNRANK_IDS: &[&str] = &["EIPF8Q00EMM1", "OU2MWXYWPNWQ"];
methods.sort_by(|a, b| {
@@ -558,7 +595,10 @@ pub async fn make_aditude_request(
Ok(json)
}
pub async fn process_payout(pool: &PgPool, client: &clickhouse::Client) -> Result<(), ApiError> {
pub async fn process_payout(
pool: &PgPool,
client: &clickhouse::Client,
) -> Result<(), ApiError> {
let start: DateTime<Utc> = DateTime::from_naive_utc_and_offset(
(Utc::now() - Duration::days(1))
.date_naive()
@@ -750,8 +790,12 @@ pub async fn process_payout(pool: &PgPool, client: &clickhouse::Client) -> Resul
);
}
let aditude_res =
make_aditude_request(&["METRIC_IMPRESSIONS", "METRIC_REVENUE"], "Yesterday", "1d").await?;
let aditude_res = make_aditude_request(
&["METRIC_IMPRESSIONS", "METRIC_REVENUE"],
"Yesterday",
"1d",
)
.await?;
let aditude_amount: Decimal = aditude_res
.iter()
@@ -777,8 +821,9 @@ pub async fn process_payout(pool: &PgPool, client: &clickhouse::Client) -> Resul
// Clean.io fee (ad antimalware). Per 1000 impressions.
let clean_io_fee = Decimal::from(8) / Decimal::from(1000);
let net_revenue =
aditude_amount - (clean_io_fee * Decimal::from(aditude_impressions) / Decimal::from(1000));
let net_revenue = aditude_amount
- (clean_io_fee * Decimal::from(aditude_impressions)
/ Decimal::from(1000));
let payout = net_revenue * (Decimal::from(1) - modrinth_cut);
@@ -811,11 +856,13 @@ pub async fn process_payout(pool: &PgPool, client: &clickhouse::Client) -> Resul
let project_multiplier: Decimal =
Decimal::from(**value) / Decimal::from(multipliers.sum);
let sum_splits: Decimal = project.team_members.iter().map(|x| x.1).sum();
let sum_splits: Decimal =
project.team_members.iter().map(|x| x.1).sum();
if sum_splits > Decimal::ZERO {
for (user_id, split) in project.team_members {
let payout: Decimal = payout * project_multiplier * (split / sum_splits);
let payout: Decimal =
payout * project_multiplier * (split / sum_splits);
if payout > Decimal::ZERO {
insert_user_ids.push(user_id);

30
apps/labrinth/src/queue/session.rs
View File

@@ -1,6 +1,8 @@
use crate::database::models::pat_item::PersonalAccessToken;
use crate::database::models::session_item::Session;
use crate::database::models::{DatabaseError, OAuthAccessTokenId, PatId, SessionId, UserId};
use crate::database::models::{
DatabaseError, OAuthAccessTokenId, PatId, SessionId, UserId,
};
use crate::database::redis::RedisPool;
use crate::routes::internal::session::SessionMetadata;
use chrono::Utc;
@@ -38,7 +40,10 @@ impl AuthQueue {
self.pat_queue.lock().await.insert(id);
}
pub async fn add_oauth_access_token(&self, id: crate::database::models::OAuthAccessTokenId) {
pub async fn add_oauth_access_token(
&self,
id: crate::database::models::OAuthAccessTokenId,
) {
self.oauth_access_token_queue.lock().await.insert(id);
}
@@ -56,10 +61,15 @@ impl AuthQueue {
std::mem::replace(&mut queue, HashSet::with_capacity(len))
}
pub async fn index(&self, pool: &PgPool, redis: &RedisPool) -> Result<(), DatabaseError> {
pub async fn index(
&self,
pool: &PgPool,
redis: &RedisPool,
) -> Result<(), DatabaseError> {
let session_queue = self.take_sessions().await;
let pat_queue = Self::take_hashset(&self.pat_queue).await;
let oauth_access_token_queue = Self::take_hashset(&self.oauth_access_token_queue).await;
let oauth_access_token_queue =
Self::take_hashset(&self.oauth_access_token_queue).await;
if !session_queue.is_empty()
|| !pat_queue.is_empty()
@@ -104,7 +114,11 @@ impl AuthQueue {
.await?;
for (id, session, user_id) in expired_ids {
clear_cache_sessions.push((Some(id), Some(session), Some(user_id)));
clear_cache_sessions.push((
Some(id),
Some(session),
Some(user_id),
));
Session::remove(id, &mut transaction).await?;
}
@@ -128,7 +142,11 @@ impl AuthQueue {
.execute(&mut *transaction)
.await?;
update_oauth_access_token_last_used(oauth_access_token_queue, &mut transaction).await?;
update_oauth_access_token_last_used(
oauth_access_token_queue,
&mut transaction,
)
.await?;
transaction.commit().await?;
PersonalAccessToken::clear_cache(clear_cache_pats, redis).await?;