eightequal/AstralRinth
1
0
Fork 0
forked from didirus/AstralRinth
Code Pull Requests Activity

Tests 3 restructure (#754)

Browse Source 
* moved files

* moved files

* initial v3 additions

* moves req data

* tests passing, restructuring, remove v2

* fmt; clippy; prepare

* merge conflicts + issues

* merge conflict, fmt, clippy, prepare

* revs

* fixed failing test

* fixed tests
This commit is contained in:
Wyatt Verchere
2023-11-16 10:36:03 -08:00
committed by GitHub
parent f4880d0519
commit 74973e73e6
66 changed files with 4282 additions and 1033 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
src/auth/oauth/mod.rs
View File

@@ -15,7 +15,7 @@ use crate::models::ids::OAuthClientId;
use crate::models::pats::Scopes;
use crate::queue::session::AuthQueue;
use actix_web::http::header::LOCATION;
use actix_web::web::{scope, Data, Query, ServiceConfig};
use actix_web::web::{Data, Query, ServiceConfig};
use actix_web::{get, post, web, HttpRequest, HttpResponse};
use chrono::Duration;
use rand::distributions::Alphanumeric;
@@ -33,13 +33,10 @@ pub mod errors;
pub mod uris;
pub fn config(cfg: &mut ServiceConfig) {
cfg.service(
scope("auth/oauth")
.service(init_oauth)
.service(accept_client_scopes)
.service(reject_client_scopes)
.service(request_token),
);
cfg.service(init_oauth)
.service(accept_client_scopes)
.service(reject_client_scopes)
.service(request_token);
}
#[derive(Serialize, Deserialize)]

138
src/database/models/loader_fields.rs
View File

@@ -295,60 +295,97 @@ pub struct SideType {
impl LoaderField {
pub async fn get_field<'a, E>(
field: &str,
loader_ids: &[LoaderId],
exec: E,
redis: &RedisPool,
) -> Result<Option<LoaderField>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let fields = Self::get_fields(exec, redis).await?;
let fields = Self::get_fields(loader_ids, exec, redis).await?;
Ok(fields.into_iter().find(|f| f.field == field))
}
// Gets all fields for a given loader
// Gets all fields for a given loader(s)
// Returns all as this there are probably relatively few fields per loader
// TODO: in the future, this should be to get all fields in relation to something
// - e.g. get all fields for a given game?
pub async fn get_fields<'a, E>(
loader_ids: &[LoaderId],
exec: E,
redis: &RedisPool,
) -> Result<Vec<LoaderField>, DatabaseError>
where
E: sqlx::Executor<'a, Database = sqlx::Postgres>,
{
let cached_fields = redis
.get_deserialized_from_json(LOADER_FIELDS_NAMESPACE, 0) // 0 => whatever we search for fields by
.await?;
if let Some(cached_fields) = cached_fields {
return Ok(cached_fields);
type RedisLoaderFieldTuple = (LoaderId, Vec<LoaderField>);
let mut loader_ids = loader_ids.to_vec();
let cached_fields: Vec<RedisLoaderFieldTuple> = redis
.multi_get::<String, _>(LOADER_FIELDS_NAMESPACE, loader_ids.iter().map(|x| x.0))
.await?
.into_iter()
.flatten()
.filter_map(|x: String| serde_json::from_str::<RedisLoaderFieldTuple>(&x).ok())
.collect();
let mut found_loader_fields = vec![];
if !cached_fields.is_empty() {
for (loader_id, fields) in cached_fields {
if loader_ids.contains(&loader_id) {
found_loader_fields.extend(fields);
loader_ids.retain(|x| x != &loader_id);
}
}
}
let result = sqlx::query!(
"
SELECT lf.id, lf.field, lf.field_type, lf.optional, lf.min_val, lf.max_val, lf.enum_type
FROM loader_fields lf
",
)
.fetch_many(exec)
.try_filter_map(|e| async {
Ok(e.right().and_then(|r| {
Some(LoaderField {
id: LoaderFieldId(r.id),
field_type: LoaderFieldType::build(&r.field_type, r.enum_type)?,
field: r.field,
optional: r.optional,
min_val: r.min_val,
max_val: r.max_val,
})
}))
})
.try_collect::<Vec<LoaderField>>()
.await?;
redis
.set_serialized_to_json(LOADER_FIELDS_NAMESPACE, &0, &result, None)
if !loader_ids.is_empty() {
let result = sqlx::query!(
"
SELECT DISTINCT lf.id, lf.field, lf.field_type, lf.optional, lf.min_val, lf.max_val, lf.enum_type, lfl.loader_id
FROM loader_fields lf
LEFT JOIN loader_fields_loaders lfl ON lfl.loader_field_id = lf.id
WHERE lfl.loader_id = ANY($1)
",
&loader_ids.iter().map(|x| x.0).collect::<Vec<_>>()
)
.fetch_many(exec)
.try_filter_map(|e| async {
Ok(e.right().and_then(|r| {
Some((LoaderId(r.loader_id) ,LoaderField {
id: LoaderFieldId(r.id),
field_type: LoaderFieldType::build(&r.field_type, r.enum_type)?,
field: r.field,
optional: r.optional,
min_val: r.min_val,
max_val: r.max_val,
}))
}))
})
.try_collect::<Vec<(LoaderId, LoaderField)>>()
.await?;
let result: Vec<RedisLoaderFieldTuple> = result
.into_iter()
.fold(
HashMap::new(),
|mut acc: HashMap<LoaderId, Vec<LoaderField>>, x| {
acc.entry(x.0).or_default().push(x.1);
acc
},
)
.into_iter()
.collect_vec();
for (k, v) in result.into_iter() {
redis
.set_serialized_to_json(LOADER_FIELDS_NAMESPACE, k.0, (k, &v), None)
.await?;
found_loader_fields.extend(v);
}
}
let result = found_loader_fields
.into_iter()
.unique_by(|x| x.id)
.collect();
Ok(result)
}
}
@@ -641,6 +678,41 @@ impl VersionField {
enum_variants: Vec<LoaderFieldEnumValue>,
) -> Result<VersionField, String> {
let value = VersionFieldValue::parse(&loader_field, value, enum_variants)?;
// Ensure, if applicable, that the value is within the min/max bounds
let countable = match &value {
VersionFieldValue::Integer(i) => Some(*i),
VersionFieldValue::ArrayInteger(v) => Some(v.len() as i32),
VersionFieldValue::Text(_) => None,
VersionFieldValue::ArrayText(v) => Some(v.len() as i32),
VersionFieldValue::Boolean(_) => None,
VersionFieldValue::ArrayBoolean(v) => Some(v.len() as i32),
VersionFieldValue::Enum(_, _) => None,
VersionFieldValue::ArrayEnum(_, v) => Some(v.len() as i32),
};
if let Some(count) = countable {
if let Some(min) = loader_field.min_val {
if count < min {
return Err(format!(
"Provided value '{v}' for {field_name} is less than the minimum of {min}",
v = serde_json::to_string(&value).unwrap_or_default(),
field_name = loader_field.field,
));
}
}
if let Some(max) = loader_field.max_val {
if count > max {
return Err(format!(
"Provided value '{v}' for {field_name} is greater than the maximum of {max}",
v = serde_json::to_string(&value).unwrap_or_default(),
field_name = loader_field.field,
));
}
}
}
Ok(VersionField {
version_id,
field_id: loader_field.id,

2
src/models/v3/teams.rs
View File

@@ -42,7 +42,7 @@ bitflags_serde_impl!(ProjectPermissions, u64);
impl Default for ProjectPermissions {
fn default() -> ProjectPermissions {
ProjectPermissions::UPLOAD_VERSION | ProjectPermissions::DELETE_VERSION
ProjectPermissions::empty()
}
}

1
src/routes/v2/mod.rs
View File

@@ -26,6 +26,7 @@ pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
.wrap(default_cors())
.configure(admin::config)
.configure(analytics_get::config)
// Todo: separate these- they need to also follow v2-v3 conversion
.configure(crate::auth::session::config)
.configure(crate::auth::flows::config)
.configure(crate::auth::pats::config)

8
src/routes/v2/projects.rs
View File

@@ -67,13 +67,15 @@ pub async fn project_search(
facet
.into_iter()
.map(|facet| {
let version = match facet.split(':').nth(1) {
Some(version) => version,
let val = match facet.split(':').nth(1) {
Some(val) => val,
None => return facet.to_string(),
};
if facet.starts_with("versions:") {
format!("game_versions:{}", version)
format!("game_versions:{}", val)
} else if facet.starts_with("project_type:") {
format!("project_types:{}", val)
} else {
facet.to_string()
}

5
src/routes/v2/version_creation.rs
View File

@@ -95,6 +95,11 @@ pub async fn version_create(
json!(legacy_create.game_versions),
);
// TODO: will be overhauled with side-types overhaul
// TODO: if not, should default to previous version
fields.insert("client_side".to_string(), json!("required"));
fields.insert("server_side".to_string(), json!("optional"));
// TODO: Some kind of handling here to ensure project type is fine.
// We expect the version uploaded to be of loader type modpack, but there might not be a way to check here for that.
// After all, theoretically, they could be creating a genuine 'fabric' mod, and modpack no longer carries information on whether its a mod or modpack,

322
src/routes/v3/admin.rs Normal file
View File

@@ -0,0 +1,322 @@
use crate::auth::validate::get_user_record_from_bearer_token;
use crate::database::models::User;
use crate::database::redis::RedisPool;
use crate::models::analytics::Download;
use crate::models::ids::ProjectId;
use crate::models::pats::Scopes;
use crate::models::users::{PayoutStatus, RecipientStatus};
use crate::queue::analytics::AnalyticsQueue;
use crate::queue::maxmind::MaxMindIndexer;
use crate::queue::session::AuthQueue;
use crate::routes::ApiError;
use crate::search::SearchConfig;
use crate::util::date::get_current_tenths_of_ms;
use crate::util::guards::admin_key_guard;
use crate::util::routes::read_from_payload;
use actix_web::{patch, post, web, HttpRequest, HttpResponse};
use hex::ToHex;
use hmac::{Hmac, Mac, NewMac};
use serde::Deserialize;
use sha2::Sha256;
use sqlx::PgPool;
use std::collections::HashMap;
use std::net::Ipv4Addr;
use std::sync::Arc;
use uuid::Uuid;
pub fn config(cfg: &mut web::ServiceConfig) {
cfg.service(
web::scope("admin")
.service(count_download)
.service(trolley_webhook)
.service(force_reindex),
);
}
#[derive(Deserialize)]
pub struct DownloadBody {
pub url: String,
pub project_id: ProjectId,
pub version_name: String,
pub ip: String,
pub headers: HashMap<String, String>,
}
// This is an internal route, cannot be used without key
#[patch("/_count-download", guard = "admin_key_guard")]
#[allow(clippy::too_many_arguments)]
pub async fn count_download(
req: HttpRequest,
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
maxmind: web::Data<Arc<MaxMindIndexer>>,
analytics_queue: web::Data<Arc<AnalyticsQueue>>,
session_queue: web::Data<AuthQueue>,
download_body: web::Json<DownloadBody>,
) -> Result<HttpResponse, ApiError> {
let token = download_body
.headers
.iter()
.find(|x| x.0.to_lowercase() == "authorization")
.map(|x| &**x.1);
let user = get_user_record_from_bearer_token(&req, token, &**pool, &redis, &session_queue)
.await
.ok()
.flatten();
let project_id: crate::database::models::ids::ProjectId = download_body.project_id.into();
let id_option = crate::models::ids::base62_impl::parse_base62(&download_body.version_name)
.ok()
.map(|x| x as i64);
let (version_id, project_id) = if let Some(version) = sqlx::query!(
"
SELECT v.id id, v.mod_id mod_id FROM files f
INNER JOIN versions v ON v.id = f.version_id
WHERE f.url = $1
",
download_body.url,
)
.fetch_optional(pool.as_ref())
.await?
{
(version.id, version.mod_id)
} else if let Some(version) = sqlx::query!(
"
SELECT id, mod_id FROM versions
WHERE ((version_number = $1 OR id = $3) AND mod_id = $2)
",
download_body.version_name,
project_id as crate::database::models::ids::ProjectId,
id_option
)
.fetch_optional(pool.as_ref())
.await?
{
(version.id, version.mod_id)
} else {
return Err(ApiError::InvalidInput(
"Specified version does not exist!".to_string(),
));
};
let url = url::Url::parse(&download_body.url)
.map_err(|_| ApiError::InvalidInput("invalid download URL specified!".to_string()))?;
let ip = crate::routes::analytics::convert_to_ip_v6(&download_body.ip)
.unwrap_or_else(|_| Ipv4Addr::new(127, 0, 0, 1).to_ipv6_mapped());
analytics_queue.add_download(Download {
id: Uuid::new_v4(),
recorded: get_current_tenths_of_ms(),
domain: url.host_str().unwrap_or_default().to_string(),
site_path: url.path().to_string(),
user_id: user
.and_then(|(scopes, x)| {
if scopes.contains(Scopes::PERFORM_ANALYTICS) {
Some(x.id.0 as u64)
} else {
None
}
})
.unwrap_or(0),
project_id: project_id as u64,
version_id: version_id as u64,
ip,
country: maxmind.query(ip).await.unwrap_or_default(),
user_agent: download_body
.headers
.get("user-agent")
.cloned()
.unwrap_or_default(),
headers: download_body
.headers
.clone()
.into_iter()
.filter(|x| !crate::routes::analytics::FILTERED_HEADERS.contains(&&*x.0.to_lowercase()))
.collect(),
});
Ok(HttpResponse::NoContent().body(""))
}
#[derive(Deserialize)]
pub struct TrolleyWebhook {
model: String,
action: String,
body: HashMap<String, serde_json::Value>,
}
#[post("/_trolley")]
#[allow(clippy::too_many_arguments)]
pub async fn trolley_webhook(
req: HttpRequest,
pool: web::Data<PgPool>,
redis: web::Data<RedisPool>,
mut payload: web::Payload,
) -> Result<HttpResponse, ApiError> {
if let Some(signature) = req.headers().get("X-PaymentRails-Signature") {
let payload = read_from_payload(
&mut payload,
1 << 20,
"Webhook payload exceeds the maximum of 1MiB.",
)
.await?;
let mut signature = signature.to_str().ok().unwrap_or_default().split(',');
let timestamp = signature
.next()
.and_then(|x| x.split('=').nth(1))
.unwrap_or_default();
let v1 = signature
.next()
.and_then(|x| x.split('=').nth(1))
.unwrap_or_default();
let mut mac: Hmac<Sha256> =
Hmac::new_from_slice(dotenvy::var("TROLLEY_WEBHOOK_SIGNATURE")?.as_bytes())
.map_err(|_| ApiError::Payments("error initializing HMAC".to_string()))?;
mac.update(timestamp.as_bytes());
mac.update(&payload);
let request_signature = mac.finalize().into_bytes().encode_hex::<String>();
if &*request_signature == v1 {
let webhook = serde_json::from_slice::<TrolleyWebhook>(&payload)?;
if webhook.model == "recipient" {
#[derive(Deserialize)]
struct Recipient {
pub id: String,
pub email: Option<String>,
pub status: Option<RecipientStatus>,
}
if let Some(body) = webhook.body.get("recipient") {
if let Ok(recipient) = serde_json::from_value::<Recipient>(body.clone()) {
let value = sqlx::query!(
"SELECT id FROM users WHERE trolley_id = $1",
recipient.id
)
.fetch_optional(&**pool)
.await?;
if let Some(user) = value {
let user = User::get_id(
crate::database::models::UserId(user.id),
&**pool,
&redis,
)
.await?;
if let Some(user) = user {
let mut transaction = pool.begin().await?;
if webhook.action == "deleted" {
sqlx::query!(
"
UPDATE users
SET trolley_account_status = NULL, trolley_id = NULL
WHERE id = $1
",
user.id.0
)
.execute(&mut *transaction)
.await?;
} else {
sqlx::query!(
"
UPDATE users
SET email = $1, email_verified = $2, trolley_account_status = $3
WHERE id = $4
",
recipient.email.clone(),
user.email_verified && recipient.email == user.email,
recipient.status.map(|x| x.as_str()),
user.id.0
)
.execute(&mut *transaction).await?;
}
transaction.commit().await?;
User::clear_caches(&[(user.id, None)], &redis).await?;
}
}
}
}
}
if webhook.model == "payment" {
#[derive(Deserialize)]
struct Payment {
pub id: String,
pub status: PayoutStatus,
}
if let Some(body) = webhook.body.get("payment") {
if let Ok(payment) = serde_json::from_value::<Payment>(body.clone()) {
let value = sqlx::query!(
"SELECT id, amount, user_id, status FROM historical_payouts WHERE payment_id = $1",
payment.id
)
.fetch_optional(&**pool)
.await?;
if let Some(payout) = value {
let mut transaction = pool.begin().await?;
if payment.status.is_failed()
&& !PayoutStatus::from_string(&payout.status).is_failed()
{
sqlx::query!(
"
UPDATE users
SET balance = balance + $1
WHERE id = $2
",
payout.amount,
payout.user_id,
)
.execute(&mut *transaction)
.await?;
}
sqlx::query!(
"
UPDATE historical_payouts
SET status = $1
WHERE payment_id = $2
",
payment.status.as_str(),
payment.id,
)
.execute(&mut *transaction)
.await?;
transaction.commit().await?;
User::clear_caches(
&[(crate::database::models::UserId(payout.user_id), None)],
&redis,
)
.await?;
}
}
}
}
}
}
Ok(HttpResponse::NoContent().finish())
}
#[post("/_force_reindex", guard = "admin_key_guard")]
pub async fn force_reindex(
pool: web::Data<PgPool>,
config: web::Data<SearchConfig>,
) -> Result<HttpResponse, ApiError> {
use crate::search::indexing::index_projects;
index_projects(pool.as_ref().clone(), &config).await?;
Ok(HttpResponse::NoContent().finish())
}

17
src/routes/v3/mod.rs
View File

@@ -1,8 +1,9 @@
pub use super::ApiError;
use crate::{auth::oauth, util::cors::default_cors};
use crate::util::cors::default_cors;
use actix_web::{web, HttpResponse};
use serde_json::json;
pub mod admin;
pub mod analytics_get;
pub mod collections;
pub mod images;
@@ -27,20 +28,28 @@ pub fn config(cfg: &mut web::ServiceConfig) {
cfg.service(
web::scope("v3")
.wrap(default_cors())
.configure(admin::config)
.configure(analytics_get::config)
// TODO: write tests that catch these
.configure(oauth_clients::config)
.configure(crate::auth::session::config)
.configure(crate::auth::flows::config)
.configure(crate::auth::pats::config)
.configure(collections::config)
.configure(images::config)
.configure(moderation::config)
.configure(notifications::config)
.configure(organizations::config)
.configure(project_creation::config)
.configure(projects::config)
.configure(reports::config)
.configure(statistics::config)
.configure(tags::config)
.configure(teams::config)
.configure(threads::config)
.configure(users::config)
.configure(version_file::config)
.configure(versions::config)
.configure(oauth::config)
.configure(oauth_clients::config),
.configure(versions::config),
);
}

8
src/routes/v3/oauth_clients.rs
View File

@@ -43,20 +43,19 @@ use crate::database::models::oauth_client_item::OAuthClient as DBOAuthClient;
use crate::models::ids::OAuthClientId as ApiOAuthClientId;
pub fn config(cfg: &mut web::ServiceConfig) {
cfg.service(get_user_clients);
cfg.service(
scope("oauth")
.configure(crate::auth::oauth::config)
.service(revoke_oauth_authorization)
.service(oauth_client_create)
.service(oauth_client_edit)
.service(oauth_client_delete)
.service(get_client)
.service(get_clients)
.service(get_user_oauth_authorizations)
.service(revoke_oauth_authorization),
.service(get_user_oauth_authorizations),
);
}
#[get("user/{user_id}/oauth_apps")]
pub async fn get_user_clients(
req: HttpRequest,
info: web::Path<String>,
@@ -354,6 +353,7 @@ pub async fn revoke_oauth_authorization(
redis: web::Data<RedisPool>,
session_queue: web::Data<AuthQueue>,
) -> Result<HttpResponse, ApiError> {
println!("Inside revoke_oauth_authorization");
let current_user = get_user_from_headers(
&req,
&**pool,

4
src/routes/v3/organizations.rs
View File

@@ -23,15 +23,17 @@ use sqlx::PgPool;
use validator::Validate;
pub fn config(cfg: &mut web::ServiceConfig) {
cfg.route("organizations", web::get().to(organizations_get));
cfg.service(
web::scope("organization")
.route("", web::post().to(organization_create))
.route("{id}/projects", web::get().to(organization_projects_get))
.route("{id}", web::get().to(organization_get))
.route("{id}", web::patch().to(organizations_edit))
.route("{id}", web::delete().to(organization_delete))
.route("{id}/projects", web::post().to(organization_projects_add))
.route(
"{id}/projects",
"{id}/projects/{project_id}",
web::delete().to(organization_projects_remove),
)
.route("{id}/icon", web::patch().to(organization_icon_edit))

38
src/routes/v3/project_creation.rs
View File

@@ -1,8 +1,6 @@
use super::version_creation::InitialVersionData;
use super::version_creation::{try_create_version_fields, InitialVersionData};
use crate::auth::{get_user_from_headers, AuthenticationError};
use crate::database::models::loader_fields::{
Loader, LoaderField, LoaderFieldEnumValue, VersionField,
};
use crate::database::models::loader_fields::{Loader, LoaderField, LoaderFieldEnumValue};
use crate::database::models::thread_item::ThreadBuilder;
use crate::database::models::{self, image_item, User};
use crate::database::redis::RedisPool;
@@ -37,7 +35,7 @@ use thiserror::Error;
use validator::Validate;
pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
cfg.route("create", web::post().to(project_create));
cfg.route("project", web::post().to(project_create));
}
#[derive(Error, Debug)]
@@ -884,31 +882,19 @@ async fn create_initial_version(
})
.collect::<Result<Vec<models::LoaderId>, CreateError>>()?;
let loader_fields = LoaderField::get_fields(&mut **transaction, redis).await?;
let mut version_fields = vec![];
let loader_fields = LoaderField::get_fields(&loaders, &mut **transaction, redis).await?;
let mut loader_field_enum_values =
LoaderFieldEnumValue::list_many_loader_fields(&loader_fields, &mut **transaction, redis)
.await?;
for (key, value) in version_data.fields.iter() {
let loader_field = loader_fields
.iter()
.find(|lf| &lf.field == key)
.ok_or_else(|| {
CreateError::InvalidInput(format!("Loader field '{key}' does not exist!"))
})?;
let enum_variants = loader_field_enum_values
.remove(&loader_field.id)
.unwrap_or_default();
let vf: VersionField = VersionField::check_parse(
version_id.into(),
loader_field.clone(),
value.clone(),
enum_variants,
)
.map_err(CreateError::InvalidInput)?;
version_fields.push(vf);
}
let version_fields = try_create_version_fields(
version_id,
&version_data.fields,
&loader_fields,
&mut loader_field_enum_values,
)?;
println!("Made it past here");
let dependencies = version_data
.dependencies
.iter()

4
src/routes/v3/projects.rs
View File

@@ -43,7 +43,7 @@ pub fn config(cfg: &mut web::ServiceConfig) {
web::scope("project")
.route("{id}", web::get().to(project_get))
.route("{id}/check", web::get().to(project_get_check))
.route("{id}", web::delete().to(project_get))
.route("{id}", web::delete().to(project_delete))
.route("{id}", web::patch().to(project_edit))
.route("{id}/icon", web::patch().to(project_icon_edit))
.route("{id}/icon", web::delete().to(delete_project_icon))
@@ -59,7 +59,7 @@ pub fn config(cfg: &mut web::ServiceConfig) {
"members",
web::get().to(super::teams::team_members_get_project),
)
.route("versions", web::get().to(super::versions::version_list))
.route("version", web::get().to(super::versions::version_list))
.route(
"version/{slug}",
web::get().to(super::versions::version_project_get),

23
src/routes/v3/tags.rs
View File

@@ -7,6 +7,7 @@ use crate::database::models::loader_fields::{
};
use crate::database::redis::RedisPool;
use actix_web::{web, HttpResponse};
use itertools::Itertools;
use serde_json::Value;
use sqlx::PgPool;
@@ -17,7 +18,7 @@ pub fn config(cfg: &mut web::ServiceConfig) {
.route("loader", web::get().to(loader_list)),
)
.route("games", web::get().to(games_list))
.route("loader_fields", web::get().to(loader_fields_list))
.route("loader_field", web::get().to(loader_fields_list))
.route("license", web::get().to(license_list))
.route("license/{id}", web::get().to(license_text))
.route("donation_platform", web::get().to(donation_platform_list))
@@ -118,14 +119,20 @@ pub async fn loader_fields_list(
redis: web::Data<RedisPool>,
) -> Result<HttpResponse, ApiError> {
let query = query.into_inner();
let loader_field = LoaderField::get_field(&query.loader_field, &**pool, &redis)
let all_loader_ids = Loader::list(&**pool, &redis)
.await?
.ok_or_else(|| {
ApiError::InvalidInput(format!(
"'{}' was not a valid loader field.",
query.loader_field
))
})?;
.into_iter()
.map(|x| x.id)
.collect_vec();
let loader_field =
LoaderField::get_field(&query.loader_field, &all_loader_ids, &**pool, &redis)
.await?
.ok_or_else(|| {
ApiError::InvalidInput(format!(
"'{}' was not a valid loader field.",
query.loader_field
))
})?;
let loader_field_enum_id = match loader_field.field_type {
LoaderFieldType::Enum(enum_id) | LoaderFieldType::ArrayEnum(enum_id) => enum_id,

3
src/routes/v3/threads.rs
View File

@@ -24,8 +24,8 @@ use sqlx::PgPool;
pub fn config(cfg: &mut web::ServiceConfig) {
cfg.service(
web::scope("thread")
.route("{id}", web::get().to(thread_get))
.route("inbox", web::get().to(moderation_inbox))
.route("{id}", web::get().to(thread_get))
.route("{id}", web::post().to(thread_send_message))
.route("{id}/read", web::post().to(thread_read)),
);
@@ -517,7 +517,6 @@ pub async fn moderation_inbox(
Some(&[Scopes::THREAD_READ]),
)
.await?;
let ids = sqlx::query!(
"
SELECT id

5
src/routes/v3/users.rs
View File

@@ -26,7 +26,7 @@ use crate::{
util::{routes::read_from_payload, validate::validation_errors_to_string},
};
use super::ApiError;
use super::{oauth_clients::get_user_clients, ApiError};
pub fn config(cfg: &mut web::ServiceConfig) {
cfg.route("user", web::get().to(user_auth_get));
@@ -45,7 +45,8 @@ pub fn config(cfg: &mut web::ServiceConfig) {
.route("{id}/notifications", web::get().to(user_notifications))
.route("{id}/payouts", web::get().to(user_payouts))
.route("{id}/payouts_fees", web::get().to(user_payouts_fees))
.route("{id}/payouts", web::post().to(user_payouts_request)),
.route("{id}/payouts", web::post().to(user_payouts_request))
.route("{id}/oauth_apps", web::get().to(get_user_clients)),
);
}

97
src/routes/v3/version_creation.rs
View File

@@ -30,7 +30,7 @@ use futures::stream::StreamExt;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use sqlx::postgres::PgPool;
use std::collections::HashMap;
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use validator::Validate;
@@ -256,37 +256,6 @@ async fn version_create_inner(
let all_loaders =
models::loader_fields::Loader::list(&mut **transaction, redis).await?;
let loader_fields = LoaderField::get_fields(&mut **transaction, redis).await?;
let mut version_fields = vec![];
let mut loader_field_enum_values = LoaderFieldEnumValue::list_many_loader_fields(
&loader_fields,
&mut **transaction,
redis,
)
.await?;
for (key, value) in version_create_data.fields.iter() {
let loader_field = loader_fields
.iter()
.find(|lf| &lf.field == key)
.ok_or_else(|| {
CreateError::InvalidInput(format!(
"Loader field '{key}' does not exist!"
))
})?;
let enum_variants = loader_field_enum_values
.remove(&loader_field.id)
.unwrap_or_default();
let vf: VersionField = VersionField::check_parse(
version_id.into(),
loader_field.clone(),
value.clone(),
enum_variants,
)
.map_err(CreateError::InvalidInput)?;
version_fields.push(vf);
}
let loaders = version_create_data
.loaders
.iter()
@@ -299,7 +268,22 @@ async fn version_create_inner(
})
.collect::<Result<Vec<_>, _>>()?;
selected_loaders = Some(loaders.clone());
let loader_ids = loaders.iter().map(|y| y.id).collect_vec();
let loader_ids: Vec<models::LoaderId> = loaders.iter().map(|y| y.id).collect_vec();
let loader_fields =
LoaderField::get_fields(&loader_ids, &mut **transaction, redis).await?;
let mut loader_field_enum_values = LoaderFieldEnumValue::list_many_loader_fields(
&loader_fields,
&mut **transaction,
redis,
)
.await?;
let version_fields = try_create_version_fields(
version_id,
&version_create_data.fields,
&loader_fields,
&mut loader_field_enum_values,
)?;
let dependencies = version_create_data
.dependencies
@@ -966,3 +950,50 @@ pub fn get_name_ext(
};
Ok((file_name, file_extension))
}
// Reused functionality between project_creation and version_creation
// Create a list of VersionFields from the fetched data, and check that all mandatory fields are present
pub fn try_create_version_fields(
version_id: VersionId,
submitted_fields: &HashMap<String, serde_json::Value>,
loader_fields: &[LoaderField],
loader_field_enum_values: &mut HashMap<models::LoaderFieldId, Vec<LoaderFieldEnumValue>>,
) -> Result<Vec<VersionField>, CreateError> {
let mut version_fields = vec![];
let mut remaining_mandatory_loader_fields = loader_fields
.iter()
.filter(|lf| !lf.optional)
.map(|lf| lf.field.clone())
.collect::<HashSet<_>>();
for (key, value) in submitted_fields.iter() {
let loader_field = loader_fields
.iter()
.find(|lf| &lf.field == key)
.ok_or_else(|| {
CreateError::InvalidInput(format!(
"Loader field '{key}' does not exist for any loaders supplied,"
))
})?;
remaining_mandatory_loader_fields.remove(&loader_field.field);
let enum_variants = loader_field_enum_values
.remove(&loader_field.id)
.unwrap_or_default();
let vf: VersionField = VersionField::check_parse(
version_id.into(),
loader_field.clone(),
value.clone(),
enum_variants,
)
.map_err(CreateError::InvalidInput)?;
version_fields.push(vf);
}
if !remaining_mandatory_loader_fields.is_empty() {
return Err(CreateError::InvalidInput(format!(
"Missing mandatory loader fields: {}",
remaining_mandatory_loader_fields.iter().join(", ")
)));
}
Ok(version_fields)
}

8
src/routes/v3/version_file.rs
View File

@@ -19,7 +19,7 @@ use std::collections::HashMap;
pub fn config(cfg: &mut web::ServiceConfig) {
cfg.service(
web::scope("version_file")
.route("version_id", web::get().to(get_version_from_hash))
.route("{version_id}", web::get().to(get_version_from_hash))
.route("{version_id}/update", web::post().to(get_update_from_hash))
.route("project", web::post().to(get_projects_from_hashes))
.route("{version_id}", web::delete().to(delete_file))
@@ -380,7 +380,7 @@ pub async fn update_files(
Ok(HttpResponse::Ok().json(response))
}
#[derive(Deserialize)]
#[derive(Serialize, Deserialize)]
pub struct FileUpdateData {
pub hash: String,
pub loaders: Option<Vec<String>>,
@@ -388,7 +388,7 @@ pub struct FileUpdateData {
pub version_types: Option<Vec<VersionType>>,
}
#[derive(Deserialize)]
#[derive(Serialize, Deserialize)]
pub struct ManyFileUpdateData {
#[serde(default = "default_algorithm")]
pub algorithm: String,
@@ -461,6 +461,7 @@ pub async fn update_individual_files(
if let Some(loaders) = &query_file.loaders {
bool &= x.loaders.iter().any(|y| loaders.contains(y));
}
if let Some(loader_fields) = &query_file.loader_fields {
for (key, values) in loader_fields {
bool &= if let Some(x_vf) =
@@ -472,7 +473,6 @@ pub async fn update_individual_files(
};
}
}
bool
})
.sorted()

16
src/routes/v3/versions.rs
View File

@@ -5,7 +5,9 @@ use crate::auth::{
filter_authorized_versions, get_user_from_headers, is_authorized, is_authorized_version,
};
use crate::database;
use crate::database::models::loader_fields::{LoaderField, LoaderFieldEnumValue, VersionField};
use crate::database::models::loader_fields::{
self, LoaderField, LoaderFieldEnumValue, VersionField,
};
use crate::database::models::version_item::{DependencyBuilder, LoaderVersion};
use crate::database::models::{image_item, Organization};
use crate::database::redis::RedisPool;
@@ -22,6 +24,7 @@ use crate::util::img;
use crate::util::validate::validation_errors_to_string;
use actix_web::{web, HttpRequest, HttpResponse};
use chrono::{DateTime, Utc};
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use sqlx::PgPool;
use validator::Validate;
@@ -384,7 +387,14 @@ pub async fn version_edit_helper(
.map(|x| x.to_string())
.collect::<Vec<String>>();
let loader_fields = LoaderField::get_fields(&mut *transaction, &redis)
let all_loaders = loader_fields::Loader::list(&mut *transaction, &redis).await?;
let loader_ids = version_item
.loaders
.iter()
.filter_map(|x| all_loaders.iter().find(|y| &y.loader == x).map(|y| y.id))
.collect_vec();
let loader_fields = LoaderField::get_fields(&loader_ids, &mut *transaction, &redis)
.await?
.into_iter()
.filter(|lf| version_fields_names.contains(&lf.field))
@@ -417,7 +427,7 @@ pub async fn version_edit_helper(
.find(|lf| lf.field == vf_name)
.ok_or_else(|| {
ApiError::InvalidInput(format!(
"Loader field '{vf_name}' does not exist."
"Loader field '{vf_name}' does not exist for any loaders supplied."
))
})?;
let enum_variants = loader_field_enum_values

6
src/search/indexing/local_import.rs
View File

@@ -22,7 +22,7 @@ pub async fn index_local(
SELECT m.id id, v.id version_id, m.title title, m.description description, m.downloads downloads, m.follows follows,
m.icon_url icon_url, m.published published, m.approved approved, m.updated updated,
m.team_id team_id, m.license license, m.slug slug, m.status status_name, m.color color,
pt.name project_type_name, u.username username,
u.username username,
ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is false) categories,
ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null and mc.is_additional is true) additional_categories,
ARRAY_AGG(DISTINCT lo.loader) filter (where lo.loader is not null) loaders,
@@ -79,7 +79,7 @@ pub async fn index_local(
LEFT OUTER JOIN loader_field_enums lfe on lf.enum_type = lfe.id
LEFT OUTER JOIN loader_field_enum_values lfev on lfev.enum_id = lfe.id
WHERE v.status != ANY($1)
GROUP BY v.id, m.id, pt.id, u.id;
GROUP BY v.id, m.id, u.id;
",
&*crate::models::projects::VersionStatus::iterator().filter(|x| x.is_hidden()).map(|x| x.to_string()).collect::<Vec<String>>(),
&*crate::models::projects::ProjectStatus::iterator().filter(|x| x.is_searchable()).map(|x| x.to_string()).collect::<Vec<String>>(),
@@ -146,7 +146,7 @@ pub async fn index_local(
modified_timestamp: m.updated.timestamp(),
license,
slug: m.slug,
project_type: m.project_type_name,
project_types: m.project_types.unwrap_or_default(),
gallery: m.gallery.unwrap_or_default(),
display_categories,
open_source,

4
src/search/indexing/mod.rs
View File

@@ -176,7 +176,7 @@ fn default_settings() -> Settings {
const DEFAULT_DISPLAYED_ATTRIBUTES: &[&str] = &[
"project_id",
"version_id",
"project_type",
"project_types",
"slug",
"author",
"title",
@@ -200,7 +200,7 @@ const DEFAULT_SEARCHABLE_ATTRIBUTES: &[&str] = &["title", "description", "author
const DEFAULT_ATTRIBUTES_FOR_FACETING: &[&str] = &[
"categories",
"license",
"project_type",
"project_types",
"downloads",
"follows",
"author",

4
src/search/mod.rs
View File

@@ -74,7 +74,7 @@ impl SearchConfig {
pub struct UploadSearchProject {
pub version_id: String,
pub project_id: String,
pub project_type: String,
pub project_types: Vec<String>,
pub slug: Option<String>,
pub author: String,
pub title: String,
@@ -114,7 +114,7 @@ pub struct SearchResults {
pub struct ResultSearchProject {
pub version_id: String,
pub project_id: String,
pub project_type: String,
pub project_types: Vec<String>,
pub slug: Option<String>,
pub author: String,
pub title: String,

7
src/util/webhook.rs
View File

@@ -87,7 +87,7 @@ pub async fn send_discord_webhook(
"
SELECT m.id id, m.title title, m.description description, m.color color,
m.icon_url icon_url, m.slug slug,
pt.name project_type, u.username username, u.avatar_url avatar_url,
u.username username, u.avatar_url avatar_url,
ARRAY_AGG(DISTINCT c.category) filter (where c.category is not null) categories,
ARRAY_AGG(DISTINCT lo.loader) filter (where lo.loader is not null) loaders,
ARRAY_AGG(DISTINCT pt.name) filter (where pt.name is not null) project_types,
@@ -142,7 +142,7 @@ pub async fn send_discord_webhook(
LEFT OUTER JOIN loader_field_enums lfe on lf.enum_type = lfe.id
LEFT OUTER JOIN loader_field_enum_values lfev on lfev.enum_id = lfe.id
WHERE m.id = $1
GROUP BY m.id, pt.id, u.id;
GROUP BY m.id, u.id;
",
project_id.0 as i64,
&*crate::models::projects::VersionStatus::iterator().filter(|x| x.is_hidden()).map(|x| x.to_string()).collect::<Vec<String>>(),
@@ -246,7 +246,8 @@ pub async fn send_discord_webhook(
});
}
let mut project_type = project.project_type;
let mut project_types: Vec<String> = project.project_types.unwrap_or_default();
let mut project_type = project_types.pop().unwrap_or_default(); // TODO: Should this grab a not-first?
if loaders.iter().all(|x| PLUGIN_LOADERS.contains(&&**x)) {
project_type = "plugin".to_string();