From 9706f1597ba05a527168dc2bedbaaa254a4e369f Mon Sep 17 00:00:00 2001 From: aecsocket Date: Sat, 8 Nov 2025 15:27:31 -0800 Subject: [PATCH] Supporting documents for Mural payouts (#4721) * wip: gotenberg * Generate and provide supporting docs for Mural payouts * Correct docs * shear * update cargo lock because r-a complains otherwise * Remove local Gotenberg queue and use Redis instead * Store platform_id in database correctly * Address PR comments * Fix up CI * fix rebase * Add timeout to default env vars --- Cargo.toml | 1 + apps/labrinth/.env.docker-compose | 1 + apps/labrinth/.env.local | 6 +- apps/labrinth/src/database/redis.rs | 29 ++++- apps/labrinth/src/lib.rs | 1 + apps/labrinth/src/main.rs | 4 +- apps/labrinth/src/queue/payouts/mod.rs | 2 +- apps/labrinth/src/queue/payouts/mural.rs | 84 +++++++++++- .../labrinth/src/routes/internal/gotenberg.rs | 123 ++++++++++++++---- apps/labrinth/src/routes/v3/payouts.rs | 81 ++++++++---- apps/labrinth/src/test/mod.rs | 4 +- apps/labrinth/src/util/gotenberg.rs | 96 ++++++++++++-- docker-compose.yml | 23 +++- packages/muralpay/examples/muralpay.rs | 11 +- packages/muralpay/src/payout.rs | 24 +++- 15 files changed, 409 insertions(+), 81 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index cbabe4261..c6afc79c4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -249,6 +249,7 @@ redundant_clone = "warn" redundant_feature_names = "warn" redundant_type_annotations = "warn" todo = "warn" +too_many_arguments = "allow" uninlined_format_args = "warn" unnested_or_patterns = "warn" wildcard_dependencies = "warn" diff --git a/apps/labrinth/.env.docker-compose b/apps/labrinth/.env.docker-compose index 2368726e8..6271bd3cb 100644 --- a/apps/labrinth/.env.docker-compose +++ b/apps/labrinth/.env.docker-compose @@ -144,6 +144,7 @@ ANROK_API_URL=none GOTENBERG_URL=http://labrinth-gotenberg:13000 GOTENBERG_CALLBACK_BASE=http://host.docker.internal:8000/_internal/gotenberg +GOTENBERG_TIMEOUT=30000 ARCHON_URL=none diff --git a/apps/labrinth/.env.local b/apps/labrinth/.env.local index 37358ad82..f4115b4ce 100644 --- a/apps/labrinth/.env.local +++ b/apps/labrinth/.env.local @@ -19,7 +19,10 @@ MEILISEARCH_KEY=modrinth REDIS_URL=redis://localhost REDIS_MAX_CONNECTIONS=10000 -BIND_ADDR=127.0.0.1:8000 +# Must bind to broadcast, not localhost, because some +# Docker services (Gotenberg) must be able to reach the backend +# from a different network interface +BIND_ADDR=0.0.0.0:8000 SELF_ADDR=http://127.0.0.1:8000 MODERATION_SLACK_WEBHOOK= @@ -145,6 +148,7 @@ ANROK_API_URL=none GOTENBERG_URL=http://localhost:13000 GOTENBERG_CALLBACK_BASE=http://host.docker.internal:8000/_internal/gotenberg +GOTENBERG_TIMEOUT=30000 ARCHON_URL=none diff --git a/apps/labrinth/src/database/redis.rs b/apps/labrinth/src/database/redis.rs index cc01b0b20..5d299f8a1 100644 --- a/apps/labrinth/src/database/redis.rs +++ b/apps/labrinth/src/database/redis.rs @@ -5,7 +5,10 @@ use dashmap::DashMap; use deadpool_redis::{Config, Runtime}; use futures::future::Either; use prometheus::{IntGauge, Registry}; -use redis::{Cmd, ExistenceCheck, SetExpiry, SetOptions, cmd}; +use redis::{ + AsyncTypedCommands, Cmd, ExistenceCheck, SetExpiry, SetOptions, + ToRedisArgs, cmd, +}; use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -680,6 +683,30 @@ impl RedisConnection { Ok(()) } + + pub async fn lpush( + &mut self, + namespace: &str, + key: &str, + value: impl ToRedisArgs + Send + Sync, + ) -> Result<(), DatabaseError> { + let key = format!("{}_{namespace}:{key}", self.meta_namespace); + self.connection.lpush(key, value).await?; + Ok(()) + } + + pub async fn brpop( + &mut self, + namespace: &str, + key: &str, + timeout: Option, + ) -> Result, DatabaseError> { + let key = format!("{}_{namespace}:{key}", self.meta_namespace); + // a timeout of 0 is infinite + let timeout = timeout.unwrap_or(0.0); + let values = self.connection.brpop(key, timeout).await?; + Ok(values) + } } #[derive(Serialize, Deserialize)] diff --git a/apps/labrinth/src/lib.rs b/apps/labrinth/src/lib.rs index 992fa385a..a66e14264 100644 --- a/apps/labrinth/src/lib.rs +++ b/apps/labrinth/src/lib.rs @@ -503,6 +503,7 @@ pub fn check_env_vars() -> bool { failed |= check_var::("GOTENBERG_URL"); failed |= check_var::("GOTENBERG_CALLBACK_BASE"); + failed |= check_var::("GOTENBERG_TIMEOUT"); failed |= check_var::("STRIPE_API_KEY"); failed |= check_var::("STRIPE_WEBHOOK_SECRET"); diff --git a/apps/labrinth/src/main.rs b/apps/labrinth/src/main.rs index 210f5a520..a02e8bd29 100644 --- a/apps/labrinth/src/main.rs +++ b/apps/labrinth/src/main.rs @@ -153,8 +153,8 @@ async fn main() -> std::io::Result<()> { let email_queue = EmailQueue::init(pool.clone(), redis_pool.clone()).unwrap(); - let gotenberg_client = - GotenbergClient::from_env().expect("Failed to create Gotenberg client"); + let gotenberg_client = GotenbergClient::from_env(redis_pool.clone()) + .expect("Failed to create Gotenberg client"); if let Some(task) = args.run_background_task { info!("Running task {task:?} and exiting"); diff --git a/apps/labrinth/src/queue/payouts/mod.rs b/apps/labrinth/src/queue/payouts/mod.rs index d40a7d915..44d96f2b1 100644 --- a/apps/labrinth/src/queue/payouts/mod.rs +++ b/apps/labrinth/src/queue/payouts/mod.rs @@ -726,7 +726,7 @@ impl PayoutsQueue { } } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Copy)] pub struct PayoutFees { /// Fee which is taken by the underlying method we're using. /// diff --git a/apps/labrinth/src/queue/payouts/mural.rs b/apps/labrinth/src/queue/payouts/mural.rs index d7c08d7e7..2bb719ca1 100644 --- a/apps/labrinth/src/queue/payouts/mural.rs +++ b/apps/labrinth/src/queue/payouts/mural.rs @@ -1,13 +1,18 @@ use ariadne::ids::UserId; +use chrono::Utc; use eyre::{Result, eyre}; use muralpay::{MuralError, TokenFeeRequest}; -use rust_decimal::Decimal; +use rust_decimal::{Decimal, prelude::ToPrimitive}; use serde::{Deserialize, Serialize}; use crate::{ - queue::payouts::{AccountBalance, PayoutsQueue}, + database::models::DBPayoutId, + queue::payouts::{AccountBalance, PayoutFees, PayoutsQueue}, routes::ApiError, - util::error::Context, + util::{ + error::Context, + gotenberg::{GotenbergClient, PaymentStatement}, + }, }; #[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)] @@ -54,10 +59,13 @@ impl PayoutsQueue { pub async fn create_muralpay_payout_request( &self, + payout_id: DBPayoutId, user_id: UserId, - amount: muralpay::TokenAmount, + gross_amount: Decimal, + fees: PayoutFees, payout_details: MuralPayoutRequest, recipient_info: muralpay::PayoutRecipientInfo, + gotenberg: &GotenbergClient, ) -> Result { let muralpay = self.muralpay.load(); let muralpay = muralpay @@ -86,11 +94,71 @@ impl PayoutsQueue { } }; + // Mural takes `fees.method_fee` off the top of the amount we tell them to send + let sent_to_method = gross_amount - fees.platform_fee; + // ..so the net is `gross - platform_fee - method_fee` + let net_amount = gross_amount - fees.total_fee(); + + let recipient_address = recipient_info.physical_address(); + let recipient_email = recipient_info.email().to_string(); + let gross_amount_cents = gross_amount * Decimal::from(100); + let net_amount_cents = net_amount * Decimal::from(100); + let fees_cents = fees.total_fee() * Decimal::from(100); + let address_line_3 = format!( + "{}, {}, {}", + recipient_address.city, + recipient_address.state, + recipient_address.zip + ); + + let payment_statement = PaymentStatement { + payment_id: payout_id.into(), + recipient_address_line_1: Some(recipient_address.address1.clone()), + recipient_address_line_2: recipient_address.address2.clone(), + recipient_address_line_3: Some(address_line_3), + recipient_email, + payment_date: Utc::now(), + gross_amount_cents: gross_amount_cents + .to_i64() + .wrap_internal_err_with(|| eyre!("gross amount of cents `{gross_amount_cents}` cannot be expressed as an `i64`"))?, + net_amount_cents: net_amount_cents + .to_i64() + .wrap_internal_err_with(|| eyre!("net amount of cents `{net_amount_cents}` cannot be expressed as an `i64`"))?, + fees_cents: fees_cents + .to_i64() + .wrap_internal_err_with(|| eyre!("fees amount of cents `{fees_cents}` cannot be expressed as an `i64`"))?, + currency_code: "USD".into(), + }; + let payment_statement_doc = gotenberg + .wait_for_payment_statement(&payment_statement) + .await + .wrap_internal_err("failed to generate payment statement")?; + + // TODO + // std::fs::write( + // "/tmp/modrinth-payout-statement.pdf", + // base64::Engine::decode( + // &base64::engine::general_purpose::STANDARD, + // &payment_statement_doc.body, + // ) + // .unwrap(), + // ) + // .unwrap(); + let payout = muralpay::CreatePayout { - amount, + amount: muralpay::TokenAmount { + token_amount: sent_to_method, + token_symbol: muralpay::USDC.into(), + }, payout_details, recipient_info, - supporting_details: None, + supporting_details: Some(muralpay::SupportingDetails { + supporting_document: Some(format!( + "data:application/pdf;base64,{}", + payment_statement_doc.body + )), + payout_purpose: Some(muralpay::PayoutPurpose::VendorPayment), + }), }; let payout_request = muralpay @@ -103,7 +171,9 @@ impl PayoutsQueue { .await .map_err(|err| match err { MuralError::Api(err) => ApiError::Request(err.into()), - err => ApiError::Internal(err.into()), + err => ApiError::Internal( + eyre!(err).wrap_err("failed to create payout request"), + ), })?; // try to immediately execute the payout request... diff --git a/apps/labrinth/src/routes/internal/gotenberg.rs b/apps/labrinth/src/routes/internal/gotenberg.rs index 744a12567..149b25453 100644 --- a/apps/labrinth/src/routes/internal/gotenberg.rs +++ b/apps/labrinth/src/routes/internal/gotenberg.rs @@ -1,21 +1,35 @@ -use actix_web::{ - HttpMessage, HttpResponse, error::ParseError, http::header, post, web, -}; -use serde::Deserialize; +use std::fmt; + +use actix_web::{HttpMessage, error::ParseError, http::header, post, web}; +use ariadne::ids::base62_impl::parse_base62; +use base64::Engine; +use serde::{Deserialize, Serialize}; +use thiserror::Error; use tracing::trace; +use crate::database::redis::RedisPool; +use crate::models::ids::PayoutId; use crate::routes::ApiError; +use crate::util::error::Context; use crate::util::gotenberg::{ GeneratedPdfType, MODRINTH_GENERATED_PDF_TYPE, MODRINTH_PAYMENT_ID, + PAYMENT_STATEMENTS_NAMESPACE, }; use crate::util::guards::internal_network_guard; +/// Document generated by Gotenberg and returned to us. +#[derive(Debug, Serialize, Deserialize)] +pub struct GotenbergDocument { + /// Body of the document as a base 64 string. + pub body: String, +} + pub fn config(cfg: &mut actix_web::web::ServiceConfig) { - cfg.service(success).service(error); + cfg.service(success_callback).service(error_callback); } #[post("/gotenberg/success", guard = "internal_network_guard")] -pub async fn success( +pub async fn success_callback( web::Header(header::ContentDisposition { disposition, parameters: disposition_parameters, @@ -26,7 +40,8 @@ pub async fn success( >, maybe_payment_id: Option>, body: web::Bytes, -) -> Result { + redis: web::Data, +) -> Result<(), ApiError> { trace!( %trace, %disposition, @@ -37,25 +52,65 @@ pub async fn success( "Received Gotenberg generated PDF" ); - Ok(HttpResponse::Ok().finish()) + let payout_id = maybe_payment_id + .wrap_request_err("no payout ID for document")? + .0 + .0; + + let mut redis = redis + .connect() + .await + .wrap_internal_err("failed to get Redis connection")?; + + let body = base64::engine::general_purpose::STANDARD.encode(&body); + + let redis_msg = serde_json::to_string(&Ok::< + GotenbergDocument, + GotenbergError, + >(GotenbergDocument { body })) + .wrap_internal_err("failed to serialize document to JSON")?; + + redis + .lpush( + PAYMENT_STATEMENTS_NAMESPACE, + &payout_id.to_string(), + &redis_msg, + ) + .await + .wrap_internal_err("failed to send document over Redis")?; + + Ok(()) } -#[allow(dead_code)] -#[derive(Debug, Deserialize)] -pub struct ErrorBody { - status: Option, - message: Option, +#[derive(Debug, Clone, Serialize, Deserialize, Error)] +pub struct GotenbergError { + pub status: Option, + pub message: Option, +} + +impl fmt::Display for GotenbergError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match (&self.status, &self.message) { + (None, None) => write!(f, "(unknown error)"), + (Some(status), None) => write!(f, "status {status}"), + (None, Some(message)) => write!(f, "(no status) {message}"), + (Some(status), Some(message)) => { + write!(f, "status {status}: {message}") + } + } + } } #[post("/gotenberg/error", guard = "internal_network_guard")] -pub async fn error( +pub async fn error_callback( web::Header(GotenbergTrace(trace)): web::Header, web::Header(ModrinthGeneratedPdfType(r#type)): web::Header< ModrinthGeneratedPdfType, >, maybe_payment_id: Option>, - web::Json(error_body): web::Json, -) -> Result { + web::Json(error_body): web::Json, + redis: web::Data, +) -> Result<(), ApiError> { trace!( %trace, r#type = r#type.as_str(), @@ -64,7 +119,31 @@ pub async fn error( "Received Gotenberg error webhook" ); - Ok(HttpResponse::Ok().finish()) + let payout_id = maybe_payment_id + .wrap_request_err("no payout ID for document")? + .0 + .0; + let mut redis = redis + .connect() + .await + .wrap_internal_err("failed to get Redis connection")?; + + let redis_msg = serde_json::to_string(&Err::< + GotenbergDocument, + GotenbergError, + >(error_body)) + .wrap_internal_err("failed to serialize error to JSON")?; + + redis + .lpush( + PAYMENT_STATEMENTS_NAMESPACE, + &payout_id.to_string(), + &redis_msg, + ) + .await + .wrap_internal_err("failed to send error over Redis")?; + + Ok(()) } #[derive(Debug)] @@ -122,14 +201,14 @@ impl header::Header for ModrinthGeneratedPdfType { } } -#[derive(Debug)] -struct ModrinthPaymentId(String); +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct ModrinthPaymentId(pub PayoutId); impl header::TryIntoHeaderValue for ModrinthPaymentId { type Error = header::InvalidHeaderValue; fn try_into_value(self) -> Result { - header::HeaderValue::from_str(&self.0) + header::HeaderValue::from_str(&self.0.to_string()) } } @@ -144,7 +223,7 @@ impl header::Header for ModrinthPaymentId { .ok_or(ParseError::Header)? .to_str() .map_err(|_| ParseError::Header) - .map(ToOwned::to_owned) - .map(ModrinthPaymentId) + .and_then(|s| parse_base62(s).map_err(|_| ParseError::Header)) + .map(|id| Self(PayoutId(id))) } } diff --git a/apps/labrinth/src/routes/v3/payouts.rs b/apps/labrinth/src/routes/v3/payouts.rs index 75b4d0c9d..00384e92b 100644 --- a/apps/labrinth/src/routes/v3/payouts.rs +++ b/apps/labrinth/src/routes/v3/payouts.rs @@ -10,11 +10,12 @@ use crate::models::payouts::{ MuralPayDetails, PayoutMethodRequest, PayoutMethodType, PayoutStatus, TremendousDetails, TremendousForexResponse, }; -use crate::queue::payouts::PayoutsQueue; +use crate::queue::payouts::{PayoutFees, PayoutsQueue}; use crate::queue::session::AuthQueue; use crate::routes::ApiError; use crate::util::avalara1099; use crate::util::error::Context; +use crate::util::gotenberg::GotenbergClient; use actix_web::{HttpRequest, HttpResponse, delete, get, post, web}; use chrono::{DateTime, Duration, Utc}; use eyre::eyre; @@ -476,6 +477,7 @@ pub async fn create_payout( body: web::Json, session_queue: web::Data, payouts_queue: web::Data, + gotenberg: web::Data, ) -> Result<(), ApiError> { let (scopes, user) = get_user_record_from_bearer_token( &req, @@ -609,6 +611,8 @@ pub async fn create_payout( body: &body, user: &user, payout_id, + gross_amount: body.amount, + fees, amount_minus_fee, total_fee: fees.total_fee(), sent_to_method, @@ -623,7 +627,7 @@ pub async fn create_payout( tremendous_payout(payout_cx, method_details).await? } PayoutMethodRequest::MuralPay { method_details } => { - mural_pay_payout(payout_cx, method_details).await? + mural_pay_payout(payout_cx, method_details, &gotenberg).await? } }; @@ -648,6 +652,8 @@ struct PayoutContext<'a> { body: &'a Withdrawal, user: &'a DBUser, payout_id: DBPayoutId, + gross_amount: Decimal, + fees: PayoutFees, /// Set as the [`DBPayout::amount`] field. amount_minus_fee: Decimal, /// Set as the [`DBPayout::fee`] field. @@ -674,6 +680,8 @@ async fn tremendous_payout( body, user, payout_id, + gross_amount: _, + fees: _, amount_minus_fee, total_fee, sent_to_method, @@ -686,18 +694,6 @@ async fn tremendous_payout( ) -> Result { let user_email = get_verified_email(user)?; - let mut payout_item = DBPayout { - id: payout_id, - user_id: user.id, - created: Utc::now(), - status: PayoutStatus::InTransit, - amount: amount_minus_fee, - fee: Some(total_fee), - method: Some(PayoutMethodType::Tremendous), - method_address: Some(user_email.to_string()), - platform_id: None, - }; - #[derive(Deserialize)] struct Reward { pub id: String, @@ -766,36 +762,47 @@ async fn tremendous_payout( ) .await?; - if let Some(reward) = res.order.rewards.first() { - payout_item.platform_id = Some(reward.id.clone()) - } + let platform_id = res.order.rewards.first().map(|reward| reward.id.clone()); - Ok(payout_item) + Ok(DBPayout { + id: payout_id, + user_id: user.id, + created: Utc::now(), + status: PayoutStatus::InTransit, + amount: amount_minus_fee, + fee: Some(total_fee), + method: Some(PayoutMethodType::Tremendous), + method_address: Some(user_email.to_string()), + platform_id, + }) } async fn mural_pay_payout( PayoutContext { - body: _body, + body: _, user, payout_id, + gross_amount, + fees, amount_minus_fee, total_fee, - sent_to_method, + sent_to_method: _, payouts_queue, }: PayoutContext<'_>, details: &MuralPayDetails, + gotenberg: &GotenbergClient, ) -> Result { let user_email = get_verified_email(user)?; let payout_request = payouts_queue .create_muralpay_payout_request( + payout_id, user.id.into(), - muralpay::TokenAmount { - token_symbol: muralpay::USDC.into(), - token_amount: sent_to_method, - }, + gross_amount, + fees, details.payout_details.clone(), details.recipient_info.clone(), + gotenberg, ) .await?; @@ -817,6 +824,8 @@ async fn paypal_payout( body, user, payout_id, + gross_amount: _, + fees: _, amount_minus_fee, total_fee, sent_to_method, @@ -948,18 +957,39 @@ async fn paypal_payout( pub enum TransactionItem { /// User withdrew some of their available payout. Withdrawal { + /// ID of the payout. id: PayoutId, + /// Status of this payout. status: PayoutStatus, + /// When the payout was created. created: DateTime, + /// How much the user got from this payout, excluding fees. amount: Decimal, + /// How much the user paid in fees for this payout, on top of `amount`. fee: Option, + /// What payout method type was used for this. method_type: Option, + /// Payout-method-specific ID for the type of payout the user got. + /// + /// - Tremendous: the rewarded gift card ID. + /// - Mural: the payment rail used, i.e. crypto USDC or fiat USD. + /// - PayPal: `paypal_us` + /// - Venmo: `venmo` + /// + /// For legacy transactions, this may be [`None`] as we did not always + /// store this payout info. + method_id: Option, + /// Payout-method-specific address which the payout was sent to, like + /// an email address. method_address: Option, }, /// User got a payout available for them to withdraw. PayoutAvailable { + /// When this payout was made available for the user to withdraw. created: DateTime, + /// Where this payout came from. payout_source: PayoutSource, + /// How much the payout was worth. amount: Decimal, }, } @@ -1031,6 +1061,9 @@ pub async fn transaction_history( amount: payout.amount, fee: payout.fee, method_type: payout.method, + // TODO: store the `method_id` in the database, and return it here + // don't use the `platform_id`, that's something else + method_id: None, method_address: payout.method_address, }); diff --git a/apps/labrinth/src/test/mod.rs b/apps/labrinth/src/test/mod.rs index 1a10812e7..45cb2463a 100644 --- a/apps/labrinth/src/test/mod.rs +++ b/apps/labrinth/src/test/mod.rs @@ -48,8 +48,8 @@ pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig { let anrok_client = anrok::Client::from_env().unwrap(); let email_queue = EmailQueue::init(pool.clone(), redis_pool.clone()).unwrap(); - let gotenberg_client = - GotenbergClient::from_env().expect("Failed to create Gotenberg client"); + let gotenberg_client = GotenbergClient::from_env(redis_pool.clone()) + .expect("Failed to create Gotenberg client"); crate::app_setup( pool.clone(), diff --git a/apps/labrinth/src/util/gotenberg.rs b/apps/labrinth/src/util/gotenberg.rs index ec8c6b3d1..8fd55362a 100644 --- a/apps/labrinth/src/util/gotenberg.rs +++ b/apps/labrinth/src/util/gotenberg.rs @@ -1,22 +1,29 @@ +use crate::database::redis::RedisPool; +use crate::models::ids::PayoutId; use crate::routes::ApiError; +use crate::routes::internal::gotenberg::{GotenbergDocument, GotenbergError}; +use crate::util::env::env_var; use crate::util::error::Context; use actix_web::http::header::HeaderName; +use chrono::{DateTime, Datelike, Utc}; use serde::{Deserialize, Serialize}; use std::str::FromStr; +use std::time::Duration; pub const MODRINTH_GENERATED_PDF_TYPE: HeaderName = HeaderName::from_static("modrinth-generated-pdf-type"); pub const MODRINTH_PAYMENT_ID: HeaderName = HeaderName::from_static("modrinth-payment-id"); +pub const PAYMENT_STATEMENTS_NAMESPACE: &str = "payment_statements"; #[derive(Serialize, Deserialize, Debug, Clone)] pub struct PaymentStatement { - pub payment_id: String, + pub payment_id: PayoutId, pub recipient_address_line_1: Option, pub recipient_address_line_2: Option, pub recipient_address_line_3: Option, pub recipient_email: String, - pub payment_date: String, + pub payment_date: DateTime, pub gross_amount_cents: i64, pub net_amount_cents: i64, pub fees_cents: i64, @@ -53,28 +60,27 @@ pub struct GotenbergClient { gotenberg_url: String, site_url: String, callback_base: String, + redis: RedisPool, } impl GotenbergClient { /// Initialize the client from environment variables. - pub fn from_env() -> Result { + pub fn from_env(redis: RedisPool) -> eyre::Result { let client = reqwest::Client::builder() .user_agent("Modrinth") .build() - .wrap_internal_err("failed to build reqwest client")?; + .wrap_err("failed to build reqwest client")?; - let gotenberg_url = dotenvy::var("GOTENBERG_URL") - .wrap_internal_err("GOTENBERG_URL is not set")?; - let site_url = dotenvy::var("SITE_URL") - .wrap_internal_err("SITE_URL is not set")?; - let callback_base = dotenvy::var("GOTENBERG_CALLBACK_BASE") - .wrap_internal_err("GOTENBERG_CALLBACK_BASE is not set")?; + let gotenberg_url = env_var("GOTENBERG_URL")?; + let site_url = env_var("SITE_URL")?; + let callback_base = env_var("GOTENBERG_CALLBACK_BASE")?; Ok(Self { client, gotenberg_url: gotenberg_url.trim_end_matches('/').to_owned(), site_url: site_url.trim_end_matches('/').to_owned(), callback_base: callback_base.trim_end_matches('/').to_owned(), + redis, }) } @@ -140,7 +146,7 @@ impl GotenbergClient { ) .header( "Modrinth-Payment-Id", - &statement.payment_id, + statement.payment_id.to_string(), ) .header( "Gotenberg-Output-Filename", @@ -155,11 +161,67 @@ impl GotenbergClient { Ok(()) } + + /// Tells Gotenberg to generate a payment statement PDF, and waits until we + /// get a response for that PDF. + /// + /// This submits the PDF via [`GotenbergClient::generate_payment_statement`] + /// then waits until some Labrinth instance receives a response on the + /// Gotenberg webhook, sends the response over Redis to our instance, and + /// returns that from this function. + /// + /// In a local environment, the Labrinth instance that receives the webhook + /// response will be the same one that is waiting on the response, so the + /// Redis step is unnecessary, but in a real deployment, we have multiple + /// Labrinth instances so we need Redis in between. + /// + /// If Gotenberg does not return a response to us within `GOTENBERG_TIMEOUT` + /// number of milliseconds, this will fail. + pub async fn wait_for_payment_statement( + &self, + statement: &PaymentStatement, + ) -> Result { + let mut redis = self + .redis + .connect() + .await + .wrap_internal_err("failed to get Redis connection")?; + + self.generate_payment_statement(statement).await?; + + let timeout_ms = env_var("GOTENBERG_TIMEOUT") + .map_err(ApiError::Internal)? + .parse::() + .wrap_internal_err( + "`GOTENBERG_TIMEOUT` is not a valid number of milliseconds", + )?; + + let [_key, document] = tokio::time::timeout( + Duration::from_millis(timeout_ms), + redis.brpop( + PAYMENT_STATEMENTS_NAMESPACE, + &statement.payment_id.to_string(), + None, + ), + ) + .await + .wrap_internal_err("Gotenberg document generation timed out")? + .wrap_internal_err("failed to get document over Redis")? + .wrap_internal_err("no document was returned from Redis")?; + + let document = serde_json::from_str::< + Result, + >(&document) + .wrap_internal_err("failed to deserialize Redis document response")? + .wrap_internal_err("Gotenberg document generation failed")?; + + Ok(document) + } } fn fill_statement_template(html: &str, s: &PaymentStatement) -> String { let variables: Vec<(&str, String)> = vec![ - ("statement.payment_id", s.payment_id.clone()), + ("statement.payment_id", s.payment_id.to_string()), ( "statement.recipient_address_line_1", s.recipient_address_line_1.clone().unwrap_or_default(), @@ -173,7 +235,15 @@ fn fill_statement_template(html: &str, s: &PaymentStatement) -> String { s.recipient_address_line_3.clone().unwrap_or_default(), ), ("statement.recipient_email", s.recipient_email.clone()), - ("statement.payment_date", s.payment_date.clone()), + ( + "statement.payment_date", + format!( + "{:04}-{:02}-{:02}", + s.payment_date.year(), + s.payment_date.month(), + s.payment_date.day() + ), + ), ( "statement.gross_amount", format_money(s.gross_amount_cents, &s.currency_code), diff --git a/docker-compose.yml b/docker-compose.yml index ebca3de96..965599fde 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,7 +12,7 @@ services: POSTGRES_PASSWORD: labrinth POSTGRES_HOST_AUTH_METHOD: trust healthcheck: - test: [ 'CMD', 'pg_isready' ] + test: ['CMD', 'pg_isready'] interval: 3s timeout: 5s retries: 3 @@ -28,7 +28,7 @@ services: MEILI_MASTER_KEY: modrinth MEILI_HTTP_PAYLOAD_SIZE_LIMIT: 107374182400 healthcheck: - test: [ 'CMD', 'curl', '--fail', 'http://localhost:7700/health' ] + test: ['CMD', 'curl', '--fail', 'http://localhost:7700/health'] interval: 3s timeout: 5s retries: 3 @@ -41,7 +41,7 @@ services: volumes: - redis-data:/data healthcheck: - test: [ 'CMD', 'redis-cli', 'PING' ] + test: ['CMD', 'redis-cli', 'PING'] interval: 3s timeout: 5s retries: 3 @@ -54,7 +54,7 @@ services: CLICKHOUSE_USER: default CLICKHOUSE_PASSWORD: default healthcheck: - test: [ 'CMD-SHELL', 'clickhouse-client --query "SELECT 1"' ] + test: ['CMD-SHELL', 'clickhouse-client --query "SELECT 1"'] interval: 3s timeout: 5s retries: 3 @@ -67,7 +67,14 @@ services: environment: MP_ENABLE_SPAMASSASSIN: postmark healthcheck: - test: [ 'CMD', 'wget', '-q', '-O/dev/null', 'http://localhost:8025/api/v1/info' ] + test: + [ + 'CMD', + 'wget', + '-q', + '-O/dev/null', + 'http://localhost:8025/api/v1/info', + ] interval: 3s timeout: 5s retries: 3 @@ -75,7 +82,11 @@ services: image: gotenberg/gotenberg:8 container_name: labrinth-gotenberg ports: - - "3000:13000" + - '13000:3000' + extra_hosts: + # Gotenberg must send a message on a webhook to our backend, + # so it must have access to our local network + - 'host.docker.internal:host-gateway' labrinth: profiles: - with-labrinth diff --git a/packages/muralpay/examples/muralpay.rs b/packages/muralpay/examples/muralpay.rs index d3ae1afba..1af7f7bb6 100644 --- a/packages/muralpay/examples/muralpay.rs +++ b/packages/muralpay/examples/muralpay.rs @@ -5,7 +5,8 @@ use muralpay::{ AccountId, CounterpartyId, CreatePayout, CreatePayoutDetails, Dob, FiatAccountType, FiatAndRailCode, FiatAndRailDetails, FiatFeeRequest, FiatPayoutFee, MuralPay, PayoutMethodId, PayoutRecipientInfo, - PhysicalAddress, TokenAmount, TokenFeeRequest, TokenPayoutFee, UsdSymbol, + PayoutRequestId, PhysicalAddress, TokenAmount, TokenFeeRequest, + TokenPayoutFee, UsdSymbol, }; use rust_decimal::{Decimal, dec}; use serde::Serialize; @@ -54,6 +55,11 @@ enum PayoutCommand { /// List all payout requests #[clap(alias = "ls")] List, + /// Get details for a single payout request + Get { + /// ID of the payout request + payout_request_id: PayoutRequestId, + }, /// Create a payout request Create { /// ID of the Mural account to send from @@ -140,6 +146,9 @@ async fn main() -> Result<()> { Command::Payout { command: PayoutCommand::List, } => run(of, muralpay.search_payout_requests(None, None).await?), + Command::Payout { + command: PayoutCommand::Get { payout_request_id }, + } => run(of, muralpay.get_payout_request(payout_request_id).await?), Command::Payout { command: PayoutCommand::Create { diff --git a/packages/muralpay/src/payout.rs b/packages/muralpay/src/payout.rs index 063035def..876a18702 100644 --- a/packages/muralpay/src/payout.rs +++ b/packages/muralpay/src/payout.rs @@ -32,6 +32,8 @@ impl MuralPay { #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] struct Body { + // if we submit `null`, Mural errors; we have to explicitly exclude this field + #[serde(skip_serializing_if = "Option::is_none")] filter: Option, } @@ -48,7 +50,7 @@ impl MuralPay { &self, id: PayoutRequestId, ) -> Result { - self.http_get(|base| format!("{base}/api/payouts/{id}")) + self.http_get(|base| format!("{base}/api/payouts/payout/{id}")) .send_mural() .await } @@ -606,6 +608,26 @@ pub enum PayoutRecipientInfo { }, } +impl PayoutRecipientInfo { + pub fn email(&self) -> &str { + match self { + PayoutRecipientInfo::Individual { email, .. } => email, + PayoutRecipientInfo::Business { email, .. } => email, + } + } + + pub fn physical_address(&self) -> &PhysicalAddress { + match self { + PayoutRecipientInfo::Individual { + physical_address, .. + } => physical_address, + PayoutRecipientInfo::Business { + physical_address, .. + } => physical_address, + } + } +} + #[derive(Debug, Display, Clone, Copy, SerializeDisplay, DeserializeFromStr)] #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))] #[display("{year:04}-{month:02}-{day:02}")]