Supporting documents for Mural payouts (#4721)

* wip: gotenberg

* Generate and provide supporting docs for Mural payouts

* Correct docs

* shear

* update cargo lock because r-a complains otherwise

* Remove local Gotenberg queue and use Redis instead

* Store platform_id in database correctly

* Address PR comments

* Fix up CI

* fix rebase

* Add timeout to default env vars

Cargo.toml

@@ -249,6 +249,7 @@ redundant_clone = "warn"
 redundant_feature_names = "warn"
 redundant_type_annotations = "warn"
 todo = "warn"
+too_many_arguments = "allow"
 uninlined_format_args = "warn"
 unnested_or_patterns = "warn"
 wildcard_dependencies = "warn"

apps/labrinth/.env.docker-compose

@@ -144,6 +144,7 @@ ANROK_API_URL=none
 
 GOTENBERG_URL=http://labrinth-gotenberg:13000
 GOTENBERG_CALLBACK_BASE=http://host.docker.internal:8000/_internal/gotenberg
+GOTENBERG_TIMEOUT=30000
 
 ARCHON_URL=none
 

apps/labrinth/.env.local

@@ -19,7 +19,10 @@ MEILISEARCH_KEY=modrinth
 REDIS_URL=redis://localhost
 REDIS_MAX_CONNECTIONS=10000
 
-BIND_ADDR=127.0.0.1:8000
+# Must bind to broadcast, not localhost, because some
+# Docker services (Gotenberg) must be able to reach the backend
+# from a different network interface
+BIND_ADDR=0.0.0.0:8000
 SELF_ADDR=http://127.0.0.1:8000
 
 MODERATION_SLACK_WEBHOOK=
@@ -145,6 +148,7 @@ ANROK_API_URL=none
 
 GOTENBERG_URL=http://localhost:13000
 GOTENBERG_CALLBACK_BASE=http://host.docker.internal:8000/_internal/gotenberg
+GOTENBERG_TIMEOUT=30000
 
 ARCHON_URL=none
 

apps/labrinth/src/database/redis.rs

@@ -5,7 +5,10 @@ use dashmap::DashMap;
 use deadpool_redis::{Config, Runtime};
 use futures::future::Either;
 use prometheus::{IntGauge, Registry};
-use redis::{Cmd, ExistenceCheck, SetExpiry, SetOptions, cmd};
+use redis::{
+    AsyncTypedCommands, Cmd, ExistenceCheck, SetExpiry, SetOptions,
+    ToRedisArgs, cmd,
+};
 use serde::de::DeserializeOwned;
 use serde::{Deserialize, Serialize};
 use std::collections::HashMap;
@@ -680,6 +683,30 @@ impl RedisConnection {
 
         Ok(())
     }
+
+    pub async fn lpush(
+        &mut self,
+        namespace: &str,
+        key: &str,
+        value: impl ToRedisArgs + Send + Sync,
+    ) -> Result<(), DatabaseError> {
+        let key = format!("{}_{namespace}:{key}", self.meta_namespace);
+        self.connection.lpush(key, value).await?;
+        Ok(())
+    }
+
+    pub async fn brpop(
+        &mut self,
+        namespace: &str,
+        key: &str,
+        timeout: Option<f64>,
+    ) -> Result<Option<[String; 2]>, DatabaseError> {
+        let key = format!("{}_{namespace}:{key}", self.meta_namespace);
+        // a timeout of 0 is infinite
+        let timeout = timeout.unwrap_or(0.0);
+        let values = self.connection.brpop(key, timeout).await?;
+        Ok(values)
+    }
 }
 
 #[derive(Serialize, Deserialize)]

apps/labrinth/src/lib.rs

@@ -503,6 +503,7 @@ pub fn check_env_vars() -> bool {
 
     failed |= check_var::<String>("GOTENBERG_URL");
     failed |= check_var::<String>("GOTENBERG_CALLBACK_BASE");
+    failed |= check_var::<String>("GOTENBERG_TIMEOUT");
 
     failed |= check_var::<String>("STRIPE_API_KEY");
     failed |= check_var::<String>("STRIPE_WEBHOOK_SECRET");

apps/labrinth/src/main.rs

@@ -153,8 +153,8 @@ async fn main() -> std::io::Result<()> {
     let email_queue =
         EmailQueue::init(pool.clone(), redis_pool.clone()).unwrap();
 
-    let gotenberg_client =
+    let gotenberg_client = GotenbergClient::from_env(redis_pool.clone())
-        GotenbergClient::from_env().expect("Failed to create Gotenberg client");
+        .expect("Failed to create Gotenberg client");
 
     if let Some(task) = args.run_background_task {
         info!("Running task {task:?} and exiting");

apps/labrinth/src/queue/payouts/mod.rs

@@ -726,7 +726,7 @@ impl PayoutsQueue {
     }
 }
 
-#[derive(Debug, Clone)]
+#[derive(Debug, Clone, Copy)]
 pub struct PayoutFees {
     /// Fee which is taken by the underlying method we're using.
     ///

apps/labrinth/src/queue/payouts/mural.rs

@@ -1,13 +1,18 @@
 use ariadne::ids::UserId;
+use chrono::Utc;
 use eyre::{Result, eyre};
 use muralpay::{MuralError, TokenFeeRequest};
-use rust_decimal::Decimal;
+use rust_decimal::{Decimal, prelude::ToPrimitive};
 use serde::{Deserialize, Serialize};
 
 use crate::{
-    queue::payouts::{AccountBalance, PayoutsQueue},
+    database::models::DBPayoutId,
+    queue::payouts::{AccountBalance, PayoutFees, PayoutsQueue},
     routes::ApiError,
-    util::error::Context,
+    util::{
+        error::Context,
+        gotenberg::{GotenbergClient, PaymentStatement},
+    },
 };
 
 #[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)]
@@ -54,10 +59,13 @@ impl PayoutsQueue {
 
     pub async fn create_muralpay_payout_request(
         &self,
+        payout_id: DBPayoutId,
         user_id: UserId,
-        amount: muralpay::TokenAmount,
+        gross_amount: Decimal,
+        fees: PayoutFees,
         payout_details: MuralPayoutRequest,
         recipient_info: muralpay::PayoutRecipientInfo,
+        gotenberg: &GotenbergClient,
     ) -> Result<muralpay::PayoutRequest, ApiError> {
         let muralpay = self.muralpay.load();
         let muralpay = muralpay
@@ -86,11 +94,71 @@ impl PayoutsQueue {
             }
         };
 
+        // Mural takes `fees.method_fee` off the top of the amount we tell them to send
+        let sent_to_method = gross_amount - fees.platform_fee;
+        // ..so the net is `gross - platform_fee - method_fee`
+        let net_amount = gross_amount - fees.total_fee();
+
+        let recipient_address = recipient_info.physical_address();
+        let recipient_email = recipient_info.email().to_string();
+        let gross_amount_cents = gross_amount * Decimal::from(100);
+        let net_amount_cents = net_amount * Decimal::from(100);
+        let fees_cents = fees.total_fee() * Decimal::from(100);
+        let address_line_3 = format!(
+            "{}, {}, {}",
+            recipient_address.city,
+            recipient_address.state,
+            recipient_address.zip
+        );
+
+        let payment_statement = PaymentStatement {
+            payment_id: payout_id.into(),
+            recipient_address_line_1: Some(recipient_address.address1.clone()),
+            recipient_address_line_2: recipient_address.address2.clone(),
+            recipient_address_line_3: Some(address_line_3),
+            recipient_email,
+            payment_date: Utc::now(),
+            gross_amount_cents: gross_amount_cents
+                .to_i64()
+                .wrap_internal_err_with(|| eyre!("gross amount of cents `{gross_amount_cents}` cannot be expressed as an `i64`"))?,
+            net_amount_cents: net_amount_cents
+                .to_i64()
+                .wrap_internal_err_with(|| eyre!("net amount of cents `{net_amount_cents}` cannot be expressed as an `i64`"))?,
+            fees_cents: fees_cents
+                .to_i64()
+                .wrap_internal_err_with(|| eyre!("fees amount of cents `{fees_cents}` cannot be expressed as an `i64`"))?,
+            currency_code: "USD".into(),
+        };
+        let payment_statement_doc = gotenberg
+            .wait_for_payment_statement(&payment_statement)
+            .await
+            .wrap_internal_err("failed to generate payment statement")?;
+
+        // TODO
+        // std::fs::write(
+        //     "/tmp/modrinth-payout-statement.pdf",
+        //     base64::Engine::decode(
+        //         &base64::engine::general_purpose::STANDARD,
+        //         &payment_statement_doc.body,
+        //     )
+        //     .unwrap(),
+        // )
+        // .unwrap();
+
         let payout = muralpay::CreatePayout {
-            amount,
+            amount: muralpay::TokenAmount {
+                token_amount: sent_to_method,
+                token_symbol: muralpay::USDC.into(),
+            },
             payout_details,
             recipient_info,
-            supporting_details: None,
+            supporting_details: Some(muralpay::SupportingDetails {
+                supporting_document: Some(format!(
+                    "data:application/pdf;base64,{}",
+                    payment_statement_doc.body
+                )),
+                payout_purpose: Some(muralpay::PayoutPurpose::VendorPayment),
+            }),
         };
 
         let payout_request = muralpay
@@ -103,7 +171,9 @@ impl PayoutsQueue {
             .await
             .map_err(|err| match err {
                 MuralError::Api(err) => ApiError::Request(err.into()),
-                err => ApiError::Internal(err.into()),
+                err => ApiError::Internal(
+                    eyre!(err).wrap_err("failed to create payout request"),
+                ),
             })?;
 
         // try to immediately execute the payout request...

apps/labrinth/src/routes/internal/gotenberg.rs

@@ -1,21 +1,35 @@
-use actix_web::{
+use std::fmt;
-    HttpMessage, HttpResponse, error::ParseError, http::header, post, web,
+
-};
+use actix_web::{HttpMessage, error::ParseError, http::header, post, web};
-use serde::Deserialize;
+use ariadne::ids::base62_impl::parse_base62;
+use base64::Engine;
+use serde::{Deserialize, Serialize};
+use thiserror::Error;
 use tracing::trace;
 
+use crate::database::redis::RedisPool;
+use crate::models::ids::PayoutId;
 use crate::routes::ApiError;
+use crate::util::error::Context;
 use crate::util::gotenberg::{
     GeneratedPdfType, MODRINTH_GENERATED_PDF_TYPE, MODRINTH_PAYMENT_ID,
+    PAYMENT_STATEMENTS_NAMESPACE,
 };
 use crate::util::guards::internal_network_guard;
 
+/// Document generated by Gotenberg and returned to us.
+#[derive(Debug, Serialize, Deserialize)]
+pub struct GotenbergDocument {
+    /// Body of the document as a base 64 string.
+    pub body: String,
+}
+
 pub fn config(cfg: &mut actix_web::web::ServiceConfig) {
-    cfg.service(success).service(error);
+    cfg.service(success_callback).service(error_callback);
 }
 
 #[post("/gotenberg/success", guard = "internal_network_guard")]
-pub async fn success(
+pub async fn success_callback(
     web::Header(header::ContentDisposition {
         disposition,
         parameters: disposition_parameters,
@@ -26,7 +40,8 @@ pub async fn success(
     >,
     maybe_payment_id: Option<web::Header<ModrinthPaymentId>>,
     body: web::Bytes,
-) -> Result<HttpResponse, ApiError> {
+    redis: web::Data<RedisPool>,
+) -> Result<(), ApiError> {
     trace!(
         %trace,
         %disposition,
@@ -37,25 +52,65 @@ pub async fn success(
         "Received Gotenberg generated PDF"
     );
 
-    Ok(HttpResponse::Ok().finish())
+    let payout_id = maybe_payment_id
+        .wrap_request_err("no payout ID for document")?
+        .0
+        .0;
+
+    let mut redis = redis
+        .connect()
+        .await
+        .wrap_internal_err("failed to get Redis connection")?;
+
+    let body = base64::engine::general_purpose::STANDARD.encode(&body);
+
+    let redis_msg = serde_json::to_string(&Ok::<
+        GotenbergDocument,
+        GotenbergError,
+    >(GotenbergDocument { body }))
+    .wrap_internal_err("failed to serialize document to JSON")?;
+
+    redis
+        .lpush(
+            PAYMENT_STATEMENTS_NAMESPACE,
+            &payout_id.to_string(),
+            &redis_msg,
+        )
+        .await
+        .wrap_internal_err("failed to send document over Redis")?;
+
+    Ok(())
 }
 
-#[allow(dead_code)]
+#[derive(Debug, Clone, Serialize, Deserialize, Error)]
-#[derive(Debug, Deserialize)]
+pub struct GotenbergError {
-pub struct ErrorBody {
+    pub status: Option<String>,
-    status: Option<String>,
+    pub message: Option<String>,
-    message: Option<String>,
+}
+
+impl fmt::Display for GotenbergError {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match (&self.status, &self.message) {
+            (None, None) => write!(f, "(unknown error)"),
+            (Some(status), None) => write!(f, "status {status}"),
+            (None, Some(message)) => write!(f, "(no status) {message}"),
+            (Some(status), Some(message)) => {
+                write!(f, "status {status}: {message}")
+            }
+        }
+    }
 }
 
 #[post("/gotenberg/error", guard = "internal_network_guard")]
-pub async fn error(
+pub async fn error_callback(
     web::Header(GotenbergTrace(trace)): web::Header<GotenbergTrace>,
     web::Header(ModrinthGeneratedPdfType(r#type)): web::Header<
         ModrinthGeneratedPdfType,
     >,
     maybe_payment_id: Option<web::Header<ModrinthPaymentId>>,
-    web::Json(error_body): web::Json<ErrorBody>,
+    web::Json(error_body): web::Json<GotenbergError>,
-) -> Result<HttpResponse, ApiError> {
+    redis: web::Data<RedisPool>,
+) -> Result<(), ApiError> {
     trace!(
         %trace,
         r#type = r#type.as_str(),
@@ -64,7 +119,31 @@ pub async fn error(
         "Received Gotenberg error webhook"
     );
 
-    Ok(HttpResponse::Ok().finish())
+    let payout_id = maybe_payment_id
+        .wrap_request_err("no payout ID for document")?
+        .0
+        .0;
+    let mut redis = redis
+        .connect()
+        .await
+        .wrap_internal_err("failed to get Redis connection")?;
+
+    let redis_msg = serde_json::to_string(&Err::<
+        GotenbergDocument,
+        GotenbergError,
+    >(error_body))
+    .wrap_internal_err("failed to serialize error to JSON")?;
+
+    redis
+        .lpush(
+            PAYMENT_STATEMENTS_NAMESPACE,
+            &payout_id.to_string(),
+            &redis_msg,
+        )
+        .await
+        .wrap_internal_err("failed to send error over Redis")?;
+
+    Ok(())
 }
 
 #[derive(Debug)]
@@ -122,14 +201,14 @@ impl header::Header for ModrinthGeneratedPdfType {
     }
 }
 
-#[derive(Debug)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
-struct ModrinthPaymentId(String);
+pub struct ModrinthPaymentId(pub PayoutId);
 
 impl header::TryIntoHeaderValue for ModrinthPaymentId {
     type Error = header::InvalidHeaderValue;
 
     fn try_into_value(self) -> Result<header::HeaderValue, Self::Error> {
-        header::HeaderValue::from_str(&self.0)
+        header::HeaderValue::from_str(&self.0.to_string())
     }
 }
 
@@ -144,7 +223,7 @@ impl header::Header for ModrinthPaymentId {
             .ok_or(ParseError::Header)?
             .to_str()
             .map_err(|_| ParseError::Header)
-            .map(ToOwned::to_owned)
+            .and_then(|s| parse_base62(s).map_err(|_| ParseError::Header))
-            .map(ModrinthPaymentId)
+            .map(|id| Self(PayoutId(id)))
     }
 }

apps/labrinth/src/routes/v3/payouts.rs

@@ -10,11 +10,12 @@ use crate::models::payouts::{
     MuralPayDetails, PayoutMethodRequest, PayoutMethodType, PayoutStatus,
     TremendousDetails, TremendousForexResponse,
 };
-use crate::queue::payouts::PayoutsQueue;
+use crate::queue::payouts::{PayoutFees, PayoutsQueue};
 use crate::queue::session::AuthQueue;
 use crate::routes::ApiError;
 use crate::util::avalara1099;
 use crate::util::error::Context;
+use crate::util::gotenberg::GotenbergClient;
 use actix_web::{HttpRequest, HttpResponse, delete, get, post, web};
 use chrono::{DateTime, Duration, Utc};
 use eyre::eyre;
@@ -476,6 +477,7 @@ pub async fn create_payout(
     body: web::Json<Withdrawal>,
     session_queue: web::Data<AuthQueue>,
     payouts_queue: web::Data<PayoutsQueue>,
+    gotenberg: web::Data<GotenbergClient>,
 ) -> Result<(), ApiError> {
     let (scopes, user) = get_user_record_from_bearer_token(
         &req,
@@ -609,6 +611,8 @@ pub async fn create_payout(
         body: &body,
         user: &user,
         payout_id,
+        gross_amount: body.amount,
+        fees,
         amount_minus_fee,
         total_fee: fees.total_fee(),
         sent_to_method,
@@ -623,7 +627,7 @@ pub async fn create_payout(
             tremendous_payout(payout_cx, method_details).await?
         }
         PayoutMethodRequest::MuralPay { method_details } => {
-            mural_pay_payout(payout_cx, method_details).await?
+            mural_pay_payout(payout_cx, method_details, &gotenberg).await?
         }
     };
 
@@ -648,6 +652,8 @@ struct PayoutContext<'a> {
     body: &'a Withdrawal,
     user: &'a DBUser,
     payout_id: DBPayoutId,
+    gross_amount: Decimal,
+    fees: PayoutFees,
     /// Set as the [`DBPayout::amount`] field.
     amount_minus_fee: Decimal,
     /// Set as the [`DBPayout::fee`] field.
@@ -674,6 +680,8 @@ async fn tremendous_payout(
         body,
         user,
         payout_id,
+        gross_amount: _,
+        fees: _,
         amount_minus_fee,
         total_fee,
         sent_to_method,
@@ -686,18 +694,6 @@ async fn tremendous_payout(
 ) -> Result<DBPayout, ApiError> {
     let user_email = get_verified_email(user)?;
 
-    let mut payout_item = DBPayout {
-        id: payout_id,
-        user_id: user.id,
-        created: Utc::now(),
-        status: PayoutStatus::InTransit,
-        amount: amount_minus_fee,
-        fee: Some(total_fee),
-        method: Some(PayoutMethodType::Tremendous),
-        method_address: Some(user_email.to_string()),
-        platform_id: None,
-    };
-
     #[derive(Deserialize)]
     struct Reward {
         pub id: String,
@@ -766,36 +762,47 @@ async fn tremendous_payout(
         )
         .await?;
 
-    if let Some(reward) = res.order.rewards.first() {
+    let platform_id = res.order.rewards.first().map(|reward| reward.id.clone());
-        payout_item.platform_id = Some(reward.id.clone())
-    }
 
-    Ok(payout_item)
+    Ok(DBPayout {
+        id: payout_id,
+        user_id: user.id,
+        created: Utc::now(),
+        status: PayoutStatus::InTransit,
+        amount: amount_minus_fee,
+        fee: Some(total_fee),
+        method: Some(PayoutMethodType::Tremendous),
+        method_address: Some(user_email.to_string()),
+        platform_id,
+    })
 }
 
 async fn mural_pay_payout(
     PayoutContext {
-        body: _body,
+        body: _,
         user,
         payout_id,
+        gross_amount,
+        fees,
         amount_minus_fee,
         total_fee,
-        sent_to_method,
+        sent_to_method: _,
         payouts_queue,
     }: PayoutContext<'_>,
     details: &MuralPayDetails,
+    gotenberg: &GotenbergClient,
 ) -> Result<DBPayout, ApiError> {
     let user_email = get_verified_email(user)?;
 
     let payout_request = payouts_queue
         .create_muralpay_payout_request(
+            payout_id,
             user.id.into(),
-            muralpay::TokenAmount {
+            gross_amount,
-                token_symbol: muralpay::USDC.into(),
+            fees,
-                token_amount: sent_to_method,
-            },
             details.payout_details.clone(),
             details.recipient_info.clone(),
+            gotenberg,
         )
         .await?;
 
@@ -817,6 +824,8 @@ async fn paypal_payout(
         body,
         user,
         payout_id,
+        gross_amount: _,
+        fees: _,
         amount_minus_fee,
         total_fee,
         sent_to_method,
@@ -948,18 +957,39 @@ async fn paypal_payout(
 pub enum TransactionItem {
     /// User withdrew some of their available payout.
     Withdrawal {
+        /// ID of the payout.
         id: PayoutId,
+        /// Status of this payout.
         status: PayoutStatus,
+        /// When the payout was created.
         created: DateTime<Utc>,
+        /// How much the user got from this payout, excluding fees.
         amount: Decimal,
+        /// How much the user paid in fees for this payout, on top of `amount`.
         fee: Option<Decimal>,
+        /// What payout method type was used for this.
         method_type: Option<PayoutMethodType>,
+        /// Payout-method-specific ID for the type of payout the user got.
+        ///
+        /// - Tremendous: the rewarded gift card ID.
+        /// - Mural: the payment rail used, i.e. crypto USDC or fiat USD.
+        /// - PayPal: `paypal_us`
+        /// - Venmo: `venmo`
+        ///
+        /// For legacy transactions, this may be [`None`] as we did not always
+        /// store this payout info.
+        method_id: Option<String>,
+        /// Payout-method-specific address which the payout was sent to, like
+        /// an email address.
         method_address: Option<String>,
     },
     /// User got a payout available for them to withdraw.
     PayoutAvailable {
+        /// When this payout was made available for the user to withdraw.
         created: DateTime<Utc>,
+        /// Where this payout came from.
         payout_source: PayoutSource,
+        /// How much the payout was worth.
         amount: Decimal,
     },
 }
@@ -1031,6 +1061,9 @@ pub async fn transaction_history(
                 amount: payout.amount,
                 fee: payout.fee,
                 method_type: payout.method,
+                // TODO: store the `method_id` in the database, and return it here
+                // don't use the `platform_id`, that's something else
+                method_id: None,
                 method_address: payout.method_address,
             });
 

apps/labrinth/src/test/mod.rs

@@ -48,8 +48,8 @@ pub async fn setup(db: &database::TemporaryDatabase) -> LabrinthConfig {
     let anrok_client = anrok::Client::from_env().unwrap();
     let email_queue =
         EmailQueue::init(pool.clone(), redis_pool.clone()).unwrap();
-    let gotenberg_client =
+    let gotenberg_client = GotenbergClient::from_env(redis_pool.clone())
-        GotenbergClient::from_env().expect("Failed to create Gotenberg client");
+        .expect("Failed to create Gotenberg client");
 
     crate::app_setup(
         pool.clone(),

apps/labrinth/src/util/gotenberg.rs

@@ -1,22 +1,29 @@
+use crate::database::redis::RedisPool;
+use crate::models::ids::PayoutId;
 use crate::routes::ApiError;
+use crate::routes::internal::gotenberg::{GotenbergDocument, GotenbergError};
+use crate::util::env::env_var;
 use crate::util::error::Context;
 use actix_web::http::header::HeaderName;
+use chrono::{DateTime, Datelike, Utc};
 use serde::{Deserialize, Serialize};
 use std::str::FromStr;
+use std::time::Duration;
 
 pub const MODRINTH_GENERATED_PDF_TYPE: HeaderName =
     HeaderName::from_static("modrinth-generated-pdf-type");
 pub const MODRINTH_PAYMENT_ID: HeaderName =
     HeaderName::from_static("modrinth-payment-id");
+pub const PAYMENT_STATEMENTS_NAMESPACE: &str = "payment_statements";
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct PaymentStatement {
-    pub payment_id: String,
+    pub payment_id: PayoutId,
     pub recipient_address_line_1: Option<String>,
     pub recipient_address_line_2: Option<String>,
     pub recipient_address_line_3: Option<String>,
     pub recipient_email: String,
-    pub payment_date: String,
+    pub payment_date: DateTime<Utc>,
     pub gross_amount_cents: i64,
     pub net_amount_cents: i64,
     pub fees_cents: i64,
@@ -53,28 +60,27 @@ pub struct GotenbergClient {
     gotenberg_url: String,
     site_url: String,
     callback_base: String,
+    redis: RedisPool,
 }
 
 impl GotenbergClient {
     /// Initialize the client from environment variables.
-    pub fn from_env() -> Result<Self, ApiError> {
+    pub fn from_env(redis: RedisPool) -> eyre::Result<Self> {
         let client = reqwest::Client::builder()
             .user_agent("Modrinth")
             .build()
-            .wrap_internal_err("failed to build reqwest client")?;
+            .wrap_err("failed to build reqwest client")?;
 
-        let gotenberg_url = dotenvy::var("GOTENBERG_URL")
+        let gotenberg_url = env_var("GOTENBERG_URL")?;
-            .wrap_internal_err("GOTENBERG_URL is not set")?;
+        let site_url = env_var("SITE_URL")?;
-        let site_url = dotenvy::var("SITE_URL")
+        let callback_base = env_var("GOTENBERG_CALLBACK_BASE")?;
-            .wrap_internal_err("SITE_URL is not set")?;
-        let callback_base = dotenvy::var("GOTENBERG_CALLBACK_BASE")
-            .wrap_internal_err("GOTENBERG_CALLBACK_BASE is not set")?;
 
         Ok(Self {
             client,
             gotenberg_url: gotenberg_url.trim_end_matches('/').to_owned(),
             site_url: site_url.trim_end_matches('/').to_owned(),
             callback_base: callback_base.trim_end_matches('/').to_owned(),
+            redis,
         })
     }
 
@@ -140,7 +146,7 @@ impl GotenbergClient {
             )
             .header(
                 "Modrinth-Payment-Id",
-                &statement.payment_id,
+                statement.payment_id.to_string(),
             )
             .header(
                 "Gotenberg-Output-Filename",
@@ -155,11 +161,67 @@ impl GotenbergClient {
 
         Ok(())
     }
+
+    /// Tells Gotenberg to generate a payment statement PDF, and waits until we
+    /// get a response for that PDF.
+    ///
+    /// This submits the PDF via [`GotenbergClient::generate_payment_statement`]
+    /// then waits until some Labrinth instance receives a response on the
+    /// Gotenberg webhook, sends the response over Redis to our instance, and
+    /// returns that from this function.
+    ///
+    /// In a local environment, the Labrinth instance that receives the webhook
+    /// response will be the same one that is waiting on the response, so the
+    /// Redis step is unnecessary, but in a real deployment, we have multiple
+    /// Labrinth instances so we need Redis in between.
+    ///
+    /// If Gotenberg does not return a response to us within `GOTENBERG_TIMEOUT`
+    /// number of milliseconds, this will fail.
+    pub async fn wait_for_payment_statement(
+        &self,
+        statement: &PaymentStatement,
+    ) -> Result<GotenbergDocument, ApiError> {
+        let mut redis = self
+            .redis
+            .connect()
+            .await
+            .wrap_internal_err("failed to get Redis connection")?;
+
+        self.generate_payment_statement(statement).await?;
+
+        let timeout_ms = env_var("GOTENBERG_TIMEOUT")
+            .map_err(ApiError::Internal)?
+            .parse::<u64>()
+            .wrap_internal_err(
+                "`GOTENBERG_TIMEOUT` is not a valid number of milliseconds",
+            )?;
+
+        let [_key, document] = tokio::time::timeout(
+            Duration::from_millis(timeout_ms),
+            redis.brpop(
+                PAYMENT_STATEMENTS_NAMESPACE,
+                &statement.payment_id.to_string(),
+                None,
+            ),
+        )
+        .await
+        .wrap_internal_err("Gotenberg document generation timed out")?
+        .wrap_internal_err("failed to get document over Redis")?
+        .wrap_internal_err("no document was returned from Redis")?;
+
+        let document = serde_json::from_str::<
+            Result<GotenbergDocument, GotenbergError>,
+        >(&document)
+        .wrap_internal_err("failed to deserialize Redis document response")?
+        .wrap_internal_err("Gotenberg document generation failed")?;
+
+        Ok(document)
+    }
 }
 
 fn fill_statement_template(html: &str, s: &PaymentStatement) -> String {
     let variables: Vec<(&str, String)> = vec![
-        ("statement.payment_id", s.payment_id.clone()),
+        ("statement.payment_id", s.payment_id.to_string()),
         (
             "statement.recipient_address_line_1",
             s.recipient_address_line_1.clone().unwrap_or_default(),
@@ -173,7 +235,15 @@ fn fill_statement_template(html: &str, s: &PaymentStatement) -> String {
             s.recipient_address_line_3.clone().unwrap_or_default(),
         ),
         ("statement.recipient_email", s.recipient_email.clone()),
-        ("statement.payment_date", s.payment_date.clone()),
+        (
+            "statement.payment_date",
+            format!(
+                "{:04}-{:02}-{:02}",
+                s.payment_date.year(),
+                s.payment_date.month(),
+                s.payment_date.day()
+            ),
+        ),
         (
             "statement.gross_amount",
             format_money(s.gross_amount_cents, &s.currency_code),

docker-compose.yml

@@ -12,7 +12,7 @@ services:
       POSTGRES_PASSWORD: labrinth
       POSTGRES_HOST_AUTH_METHOD: trust
     healthcheck:
-      test: [ 'CMD', 'pg_isready' ]
+      test: ['CMD', 'pg_isready']
       interval: 3s
       timeout: 5s
       retries: 3
@@ -28,7 +28,7 @@ services:
       MEILI_MASTER_KEY: modrinth
       MEILI_HTTP_PAYLOAD_SIZE_LIMIT: 107374182400
     healthcheck:
-      test: [ 'CMD', 'curl', '--fail', 'http://localhost:7700/health' ]
+      test: ['CMD', 'curl', '--fail', 'http://localhost:7700/health']
       interval: 3s
       timeout: 5s
       retries: 3
@@ -41,7 +41,7 @@ services:
     volumes:
       - redis-data:/data
     healthcheck:
-      test: [ 'CMD', 'redis-cli', 'PING' ]
+      test: ['CMD', 'redis-cli', 'PING']
       interval: 3s
       timeout: 5s
       retries: 3
@@ -54,7 +54,7 @@ services:
       CLICKHOUSE_USER: default
       CLICKHOUSE_PASSWORD: default
     healthcheck:
-      test: [ 'CMD-SHELL', 'clickhouse-client --query "SELECT 1"' ]
+      test: ['CMD-SHELL', 'clickhouse-client --query "SELECT 1"']
       interval: 3s
       timeout: 5s
       retries: 3
@@ -67,7 +67,14 @@ services:
     environment:
       MP_ENABLE_SPAMASSASSIN: postmark
     healthcheck:
-      test: [ 'CMD', 'wget', '-q', '-O/dev/null', 'http://localhost:8025/api/v1/info' ]
+      test:
+        [
+          'CMD',
+          'wget',
+          '-q',
+          '-O/dev/null',
+          'http://localhost:8025/api/v1/info',
+        ]
       interval: 3s
       timeout: 5s
       retries: 3
@@ -75,7 +82,11 @@ services:
     image: gotenberg/gotenberg:8
     container_name: labrinth-gotenberg
     ports:
-      - "3000:13000"
+      - '13000:3000'
+    extra_hosts:
+      # Gotenberg must send a message on a webhook to our backend,
+      # so it must have access to our local network
+      - 'host.docker.internal:host-gateway'
   labrinth:
     profiles:
       - with-labrinth

packages/muralpay/examples/muralpay.rs

@@ -5,7 +5,8 @@ use muralpay::{
     AccountId, CounterpartyId, CreatePayout, CreatePayoutDetails, Dob,
     FiatAccountType, FiatAndRailCode, FiatAndRailDetails, FiatFeeRequest,
     FiatPayoutFee, MuralPay, PayoutMethodId, PayoutRecipientInfo,
-    PhysicalAddress, TokenAmount, TokenFeeRequest, TokenPayoutFee, UsdSymbol,
+    PayoutRequestId, PhysicalAddress, TokenAmount, TokenFeeRequest,
+    TokenPayoutFee, UsdSymbol,
 };
 use rust_decimal::{Decimal, dec};
 use serde::Serialize;
@@ -54,6 +55,11 @@ enum PayoutCommand {
     /// List all payout requests
     #[clap(alias = "ls")]
     List,
+    /// Get details for a single payout request
+    Get {
+        /// ID of the payout request
+        payout_request_id: PayoutRequestId,
+    },
     /// Create a payout request
     Create {
         /// ID of the Mural account to send from
@@ -140,6 +146,9 @@ async fn main() -> Result<()> {
         Command::Payout {
             command: PayoutCommand::List,
         } => run(of, muralpay.search_payout_requests(None, None).await?),
+        Command::Payout {
+            command: PayoutCommand::Get { payout_request_id },
+        } => run(of, muralpay.get_payout_request(payout_request_id).await?),
         Command::Payout {
             command:
                 PayoutCommand::Create {

packages/muralpay/src/payout.rs

@@ -32,6 +32,8 @@ impl MuralPay {
         #[derive(Debug, Serialize)]
         #[serde(rename_all = "camelCase")]
         struct Body {
+            // if we submit `null`, Mural errors; we have to explicitly exclude this field
+            #[serde(skip_serializing_if = "Option::is_none")]
             filter: Option<PayoutStatusFilter>,
         }
 
@@ -48,7 +50,7 @@ impl MuralPay {
         &self,
         id: PayoutRequestId,
     ) -> Result<PayoutRequest, MuralError> {
-        self.http_get(|base| format!("{base}/api/payouts/{id}"))
+        self.http_get(|base| format!("{base}/api/payouts/payout/{id}"))
             .send_mural()
             .await
     }
@@ -606,6 +608,26 @@ pub enum PayoutRecipientInfo {
     },
 }
 
+impl PayoutRecipientInfo {
+    pub fn email(&self) -> &str {
+        match self {
+            PayoutRecipientInfo::Individual { email, .. } => email,
+            PayoutRecipientInfo::Business { email, .. } => email,
+        }
+    }
+
+    pub fn physical_address(&self) -> &PhysicalAddress {
+        match self {
+            PayoutRecipientInfo::Individual {
+                physical_address, ..
+            } => physical_address,
+            PayoutRecipientInfo::Business {
+                physical_address, ..
+            } => physical_address,
+        }
+    }
+}
+
 #[derive(Debug, Display, Clone, Copy, SerializeDisplay, DeserializeFromStr)]
 #[cfg_attr(feature = "utoipa", derive(utoipa::ToSchema))]
 #[display("{year:04}-{month:02}-{day:02}")]