1
0
* computer switch

* some fixes; github action

* added pr to master

* sqlx database setup

* switched intial GHA test db

* removed sqlx database setup

* unfinished patch route

* bug fixes + tests

* more tests, more fixes, cargo fmt

* merge fixes

* more tests, full reorganization

* fmt, clippy

* sqlx-data

* revs

* removed comments

* delete revs
This commit is contained in:
Wyatt Verchere
2023-10-06 09:57:33 -07:00
committed by GitHub
parent a1b59d4545
commit 259c5ef3d0
69 changed files with 4167 additions and 1312 deletions

82
tests/common/actix.rs Normal file
View File

@@ -0,0 +1,82 @@
use actix_web::test::TestRequest;
use bytes::{Bytes, BytesMut};
// Multipart functionality (actix-test does not innately support multipart)
#[derive(Debug, Clone)]
pub struct MultipartSegment {
pub name: String,
pub filename: Option<String>,
pub content_type: Option<String>,
pub data: MultipartSegmentData,
}
#[derive(Debug, Clone)]
#[allow(dead_code)]
pub enum MultipartSegmentData {
Text(String),
Binary(Vec<u8>),
}
pub trait AppendsMultipart {
fn set_multipart(self, data: Vec<MultipartSegment>) -> Self;
}
impl AppendsMultipart for TestRequest {
fn set_multipart(self, data: Vec<MultipartSegment>) -> Self {
let (boundary, payload) = generate_multipart(data);
self.append_header((
"Content-Type",
format!("multipart/form-data; boundary={}", boundary),
))
.set_payload(payload)
}
}
fn generate_multipart(data: Vec<MultipartSegment>) -> (String, Bytes) {
let mut boundary = String::from("----WebKitFormBoundary");
boundary.push_str(&rand::random::<u64>().to_string());
boundary.push_str(&rand::random::<u64>().to_string());
boundary.push_str(&rand::random::<u64>().to_string());
let mut payload = BytesMut::new();
for segment in data {
payload.extend_from_slice(
format!(
"--{boundary}\r\nContent-Disposition: form-data; name=\"{name}\"",
boundary = boundary,
name = segment.name
)
.as_bytes(),
);
if let Some(filename) = &segment.filename {
payload.extend_from_slice(
format!("; filename=\"{filename}\"", filename = filename).as_bytes(),
);
}
if let Some(content_type) = &segment.content_type {
payload.extend_from_slice(
format!(
"\r\nContent-Type: {content_type}",
content_type = content_type
)
.as_bytes(),
);
}
payload.extend_from_slice(b"\r\n\r\n");
match &segment.data {
MultipartSegmentData::Text(text) => {
payload.extend_from_slice(text.as_bytes());
}
MultipartSegmentData::Binary(binary) => {
payload.extend_from_slice(binary);
}
}
payload.extend_from_slice(b"\r\n");
}
payload.extend_from_slice(format!("--{boundary}--\r\n", boundary = boundary).as_bytes());
(boundary, Bytes::from(payload))
}

134
tests/common/database.rs Normal file
View File

@@ -0,0 +1,134 @@
#![allow(dead_code)]
use labrinth::database::redis::RedisPool;
use sqlx::{postgres::PgPoolOptions, PgPool};
use std::time::Duration;
use url::Url;
// The dummy test database adds a fair bit of 'dummy' data to test with.
// Some constants are used to refer to that data, and are described here.
// The rest can be accessed in the TestEnvironment 'dummy' field.
// The user IDs are as follows:
pub const ADMIN_USER_ID: &str = "1";
pub const MOD_USER_ID: &str = "2";
pub const USER_USER_ID: &str = "3"; // This is the 'main' user ID, and is used for most tests.
pub const FRIEND_USER_ID: &str = "4"; // This is exactly the same as USER_USER_ID, but could be used for testing friend-only endpoints (ie: teams, etc)
pub const ENEMY_USER_ID: &str = "5"; // This is exactly the same as USER_USER_ID, but could be used for testing friend-only endpoints (ie: teams, etc)
pub const ADMIN_USER_ID_PARSED: i64 = 1;
pub const MOD_USER_ID_PARSED: i64 = 2;
pub const USER_USER_ID_PARSED: i64 = 3;
pub const FRIEND_USER_ID_PARSED: i64 = 4;
pub const ENEMY_USER_ID_PARSED: i64 = 5;
// These are full-scoped PATs- as if the user was logged in (including illegal scopes).
pub const ADMIN_USER_PAT: &str = "mrp_patadmin";
pub const MOD_USER_PAT: &str = "mrp_patmoderator";
pub const USER_USER_PAT: &str = "mrp_patuser";
pub const FRIEND_USER_PAT: &str = "mrp_patfriend";
pub const ENEMY_USER_PAT: &str = "mrp_patenemy";
pub struct TemporaryDatabase {
pub pool: PgPool,
pub redis_pool: RedisPool,
pub database_name: String,
}
impl TemporaryDatabase {
// Creates a temporary database like sqlx::test does
// 1. Logs into the main database
// 2. Creates a new randomly generated database
// 3. Runs migrations on the new database
// 4. (Optionally, by using create_with_dummy) adds dummy data to the database
// If a db is created with create_with_dummy, it must be cleaned up with cleanup.
// This means that dbs will only 'remain' if a test fails (for examination of the db), and will be cleaned up otherwise.
pub async fn create() -> Self {
let temp_database_name = generate_random_database_name();
println!("Creating temporary database: {}", &temp_database_name);
let database_url = dotenvy::var("DATABASE_URL").expect("No database URL");
let mut url = Url::parse(&database_url).expect("Invalid database URL");
let pool = PgPool::connect(&database_url)
.await
.expect("Connection to database failed");
// Create the temporary database
let create_db_query = format!("CREATE DATABASE {}", &temp_database_name);
sqlx::query(&create_db_query)
.execute(&pool)
.await
.expect("Database creation failed");
pool.close().await;
// Modify the URL to switch to the temporary database
url.set_path(&format!("/{}", &temp_database_name));
let temp_db_url = url.to_string();
let pool = PgPoolOptions::new()
.min_connections(0)
.max_connections(4)
.max_lifetime(Some(Duration::from_secs(60 * 60)))
.connect(&temp_db_url)
.await
.expect("Connection to temporary database failed");
// Performs migrations
let migrations = sqlx::migrate!("./migrations");
migrations.run(&pool).await.expect("Migrations failed");
// Gets new Redis pool
let redis_pool = RedisPool::new(Some(temp_database_name.clone()));
Self {
pool,
database_name: temp_database_name,
redis_pool,
}
}
// Deletes the temporary database
// If a temporary db is created, it must be cleaned up with cleanup.
// This means that dbs will only 'remain' if a test fails (for examination of the db), and will be cleaned up otherwise.
pub async fn cleanup(mut self) {
let database_url = dotenvy::var("DATABASE_URL").expect("No database URL");
self.pool.close().await;
self.pool = PgPool::connect(&database_url)
.await
.expect("Connection to main database failed");
// Forcibly terminate all existing connections to this version of the temporary database
// We are done and deleting it, so we don't need them anymore
let terminate_query = format!(
"SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE datname = '{}' AND pid <> pg_backend_pid()",
&self.database_name
);
sqlx::query(&terminate_query)
.execute(&self.pool)
.await
.unwrap();
// Execute the deletion query asynchronously
let drop_db_query = format!("DROP DATABASE IF EXISTS {}", &self.database_name);
sqlx::query(&drop_db_query)
.execute(&self.pool)
.await
.expect("Database deletion failed");
}
}
fn generate_random_database_name() -> String {
// Generate a random database name here
// You can use your logic to create a unique name
// For example, you can use a random string as you did before
// or append a timestamp, etc.
// We will use a random string starting with "labrinth_tests_db_"
// and append a 6-digit number to it.
let mut database_name = String::from("labrinth_tests_db_");
database_name.push_str(&rand::random::<u64>().to_string()[..6]);
database_name
}

229
tests/common/dummy_data.rs Normal file
View File

@@ -0,0 +1,229 @@
use actix_web::test::{self, TestRequest};
use labrinth::{models::projects::Project, models::projects::Version};
use serde_json::json;
use sqlx::Executor;
use crate::common::{
actix::AppendsMultipart,
database::{MOD_USER_PAT, USER_USER_PAT},
};
use super::{
actix::{MultipartSegment, MultipartSegmentData},
environment::TestEnvironment,
};
pub struct DummyData {
pub alpha_team_id: String,
pub beta_team_id: String,
pub alpha_project_id: String,
pub beta_project_id: String,
pub alpha_project_slug: String,
pub beta_project_slug: String,
pub alpha_version_id: String,
pub beta_version_id: String,
pub alpha_thread_id: String,
pub beta_thread_id: String,
pub alpha_file_hash: String,
pub beta_file_hash: String,
}
pub async fn add_dummy_data(test_env: &TestEnvironment) -> DummyData {
// Adds basic dummy data to the database directly with sql (user, pats)
let pool = &test_env.db.pool.clone();
pool.execute(include_str!("../files/dummy_data.sql"))
.await
.unwrap();
let (alpha_project, alpha_version) = add_project_alpha(test_env).await;
let (beta_project, beta_version) = add_project_beta(test_env).await;
DummyData {
alpha_team_id: alpha_project.team.to_string(),
beta_team_id: beta_project.team.to_string(),
alpha_project_id: alpha_project.id.to_string(),
beta_project_id: beta_project.id.to_string(),
alpha_project_slug: alpha_project.slug.unwrap(),
beta_project_slug: beta_project.slug.unwrap(),
alpha_version_id: alpha_version.id.to_string(),
beta_version_id: beta_version.id.to_string(),
alpha_thread_id: alpha_project.thread_id.to_string(),
beta_thread_id: beta_project.thread_id.to_string(),
alpha_file_hash: alpha_version.files[0].hashes["sha1"].clone(),
beta_file_hash: beta_version.files[0].hashes["sha1"].clone(),
}
}
pub async fn add_project_alpha(test_env: &TestEnvironment) -> (Project, Version) {
// Adds dummy data to the database with sqlx (projects, versions, threads)
// Generate test project data.
let json_data = json!(
{
"title": "Test Project Alpha",
"slug": "alpha",
"description": "A dummy project for testing with.",
"body": "This project is approved, and versions are listed.",
"client_side": "required",
"server_side": "optional",
"initial_versions": [{
"file_parts": ["dummy-project-alpha.jar"],
"version_number": "1.2.3",
"version_title": "start",
"dependencies": [],
"game_versions": ["1.20.1"] ,
"release_channel": "release",
"loaders": ["fabric"],
"featured": true
}],
"categories": [],
"license_id": "MIT"
}
);
// Basic json
let json_segment = MultipartSegment {
name: "data".to_string(),
filename: None,
content_type: Some("application/json".to_string()),
data: MultipartSegmentData::Text(serde_json::to_string(&json_data).unwrap()),
};
// Basic file
let file_segment = MultipartSegment {
name: "dummy-project-alpha.jar".to_string(),
filename: Some("dummy-project-alpha.jar".to_string()),
content_type: Some("application/java-archive".to_string()),
data: MultipartSegmentData::Binary(
include_bytes!("../../tests/files/dummy-project-alpha.jar").to_vec(),
),
};
// Add a project.
let req = TestRequest::post()
.uri("/v2/project")
.append_header(("Authorization", USER_USER_PAT))
.set_multipart(vec![json_segment.clone(), file_segment.clone()])
.to_request();
let resp = test_env.call(req).await;
assert_eq!(resp.status(), 200);
// Approve as a moderator.
let req = TestRequest::patch()
.uri("/v2/project/alpha")
.append_header(("Authorization", MOD_USER_PAT))
.set_json(json!(
{
"status": "approved"
}
))
.to_request();
let resp = test_env.call(req).await;
assert_eq!(resp.status(), 204);
// Get project
let req = TestRequest::get()
.uri("/v2/project/alpha")
.append_header(("Authorization", USER_USER_PAT))
.to_request();
let resp = test_env.call(req).await;
let project: Project = test::read_body_json(resp).await;
// Get project's versions
let req = TestRequest::get()
.uri("/v2/project/alpha/version")
.append_header(("Authorization", USER_USER_PAT))
.to_request();
let resp = test_env.call(req).await;
let versions: Vec<Version> = test::read_body_json(resp).await;
let version = versions.into_iter().next().unwrap();
(project, version)
}
pub async fn add_project_beta(test_env: &TestEnvironment) -> (Project, Version) {
// Adds dummy data to the database with sqlx (projects, versions, threads)
// Generate test project data.
let json_data = json!(
{
"title": "Test Project Beta",
"slug": "beta",
"description": "A dummy project for testing with.",
"body": "This project is not-yet-approved, and versions are draft.",
"client_side": "required",
"server_side": "optional",
"initial_versions": [{
"file_parts": ["dummy-project-beta.jar"],
"version_number": "1.2.3",
"version_title": "start",
"status": "unlisted",
"requested_status": "unlisted",
"dependencies": [],
"game_versions": ["1.20.1"] ,
"release_channel": "release",
"loaders": ["fabric"],
"featured": true
}],
"status": "private",
"requested_status": "private",
"categories": [],
"license_id": "MIT"
}
);
// Basic json
let json_segment = MultipartSegment {
name: "data".to_string(),
filename: None,
content_type: Some("application/json".to_string()),
data: MultipartSegmentData::Text(serde_json::to_string(&json_data).unwrap()),
};
// Basic file
let file_segment = MultipartSegment {
name: "dummy-project-beta.jar".to_string(),
filename: Some("dummy-project-beta.jar".to_string()),
content_type: Some("application/java-archive".to_string()),
data: MultipartSegmentData::Binary(
include_bytes!("../../tests/files/dummy-project-beta.jar").to_vec(),
),
};
// Add a project.
let req = TestRequest::post()
.uri("/v2/project")
.append_header(("Authorization", USER_USER_PAT))
.set_multipart(vec![json_segment.clone(), file_segment.clone()])
.to_request();
let resp = test_env.call(req).await;
assert_eq!(resp.status(), 200);
// Get project
let req = TestRequest::get()
.uri("/v2/project/beta")
.append_header(("Authorization", USER_USER_PAT))
.to_request();
let resp = test_env.call(req).await;
let project: Project = test::read_body_json(resp).await;
// Get project's versions
let req = TestRequest::get()
.uri("/v2/project/beta/version")
.append_header(("Authorization", USER_USER_PAT))
.to_request();
let resp = test_env.call(req).await;
let versions: Vec<Version> = test::read_body_json(resp).await;
let version = versions.into_iter().next().unwrap();
(project, version)
}

View File

@@ -0,0 +1,71 @@
#![allow(dead_code)]
use super::{database::TemporaryDatabase, dummy_data};
use crate::common::setup;
use actix_web::{dev::ServiceResponse, test, App};
// A complete test environment, with a test actix app and a database.
// Must be called in an #[actix_rt::test] context. It also simulates a
// temporary sqlx db like #[sqlx::test] would.
// Use .call(req) on it directly to make a test call as if test::call_service(req) were being used.
pub struct TestEnvironment {
test_app: Box<dyn LocalService>,
pub db: TemporaryDatabase,
pub dummy: Option<dummy_data::DummyData>,
}
impl TestEnvironment {
pub async fn build_with_dummy() -> Self {
let mut test_env = Self::build().await;
let dummy = dummy_data::add_dummy_data(&test_env).await;
test_env.dummy = Some(dummy);
test_env
}
pub async fn build() -> Self {
let db = TemporaryDatabase::create().await;
let labrinth_config = setup(&db).await;
let app = App::new().configure(|cfg| labrinth::app_config(cfg, labrinth_config.clone()));
let test_app = test::init_service(app).await;
Self {
test_app: Box::new(test_app),
db,
dummy: None,
}
}
pub async fn cleanup(self) {
self.db.cleanup().await;
}
pub async fn call(&self, req: actix_http::Request) -> ServiceResponse {
self.test_app.call(req).await.unwrap()
}
}
trait LocalService {
fn call(
&self,
req: actix_http::Request,
) -> std::pin::Pin<
Box<dyn std::future::Future<Output = Result<ServiceResponse, actix_web::Error>>>,
>;
}
impl<S> LocalService for S
where
S: actix_web::dev::Service<
actix_http::Request,
Response = ServiceResponse,
Error = actix_web::Error,
>,
S::Future: 'static,
{
fn call(
&self,
req: actix_http::Request,
) -> std::pin::Pin<
Box<dyn std::future::Future<Output = Result<ServiceResponse, actix_web::Error>>>,
> {
Box::pin(self.call(req))
}
}

40
tests/common/mod.rs Normal file
View File

@@ -0,0 +1,40 @@
use labrinth::{check_env_vars, clickhouse};
use labrinth::{file_hosting, queue, LabrinthConfig};
use std::sync::Arc;
use self::database::TemporaryDatabase;
pub mod actix;
pub mod database;
pub mod dummy_data;
pub mod environment;
pub mod pats;
pub mod scopes;
// Testing equivalent to 'setup' function, producing a LabrinthConfig
// If making a test, you should probably use environment::TestEnvironment::build_with_dummy() (which calls this)
pub async fn setup(db: &TemporaryDatabase) -> LabrinthConfig {
println!("Setting up labrinth config");
dotenvy::dotenv().ok();
if check_env_vars() {
println!("Some environment variables are missing!");
}
let pool = db.pool.clone();
let redis_pool = db.redis_pool.clone();
let file_host: Arc<dyn file_hosting::FileHost + Send + Sync> =
Arc::new(file_hosting::MockHost::new());
let mut clickhouse = clickhouse::init_client().await.unwrap();
let maxmind_reader = Arc::new(queue::maxmind::MaxMindIndexer::new().await.unwrap());
labrinth::app_setup(
pool.clone(),
redis_pool.clone(),
&mut clickhouse,
file_host.clone(),
maxmind_reader.clone(),
)
}

30
tests/common/pats.rs Normal file
View File

@@ -0,0 +1,30 @@
#![allow(dead_code)]
use chrono::Utc;
use labrinth::{
database::{self, models::generate_pat_id},
models::pats::Scopes,
};
use super::database::TemporaryDatabase;
// Creates a PAT with the given scopes, and returns the access token
// Interfacing with the db directly, rather than using a ourte,
// allows us to test with scopes that are not allowed to be created by PATs
pub async fn create_test_pat(scopes: Scopes, user_id: i64, db: &TemporaryDatabase) -> String {
let mut transaction = db.pool.begin().await.unwrap();
let id = generate_pat_id(&mut transaction).await.unwrap();
let pat = database::models::pat_item::PersonalAccessToken {
id,
name: format!("test_pat_{}", scopes.bits()),
access_token: format!("mrp_{}", id.0),
scopes,
user_id: database::models::ids::UserId(user_id),
created: Utc::now(),
expires: Utc::now() + chrono::Duration::days(1),
last_used: None,
};
pat.insert(&mut transaction).await.unwrap();
transaction.commit().await.unwrap();
pat.access_token
}

124
tests/common/scopes.rs Normal file
View File

@@ -0,0 +1,124 @@
#![allow(dead_code)]
use actix_web::test::{self, TestRequest};
use labrinth::models::pats::Scopes;
use super::{database::USER_USER_ID_PARSED, environment::TestEnvironment, pats::create_test_pat};
// A reusable test type that works for any scope test testing an endpoint that:
// - returns a known 'expected_failure_code' if the scope is not present (defaults to 401)
// - returns a 200-299 if the scope is present
// - returns failure and success JSON bodies for requests that are 200 (for performing non-simple follow-up tests on)
// This uses a builder format, so you can chain methods to set the parameters to non-defaults (most will probably be not need to be set).
pub struct ScopeTest<'a> {
test_env: &'a TestEnvironment,
// Scopes expected to fail on this test. By default, this is all scopes except the success scopes.
// (To ensure we have isolated the scope we are testing)
failure_scopes: Option<Scopes>,
// User ID to use for the PATs. By default, this is the USER_USER_ID_PARSED constant.
user_id: i64,
// The code that is expected to be returned if the scope is not present. By default, this is 401 (Unauthorized)
expected_failure_code: u16,
}
impl<'a> ScopeTest<'a> {
pub fn new(test_env: &'a TestEnvironment) -> Self {
Self {
test_env,
failure_scopes: None,
user_id: USER_USER_ID_PARSED,
expected_failure_code: 401,
}
}
// Set non-standard failure scopes
// If not set, it will be set to all scopes except the success scopes
// (eg: if a combination of scopes is needed, but you want to make sure that the endpoint does not work with all-but-one of them)
pub fn with_failure_scopes(mut self, scopes: Scopes) -> Self {
self.failure_scopes = Some(scopes);
self
}
// Set the user ID to use
// (eg: a moderator, or friend)
pub fn with_user_id(mut self, user_id: i64) -> Self {
self.user_id = user_id;
self
}
// If a non-401 code is expected.
// (eg: a 404 for a hidden resource, or 200 for a resource with hidden values deeper in)
pub fn with_failure_code(mut self, code: u16) -> Self {
self.expected_failure_code = code;
self
}
// Call the endpoint generated by req_gen twice, once with a PAT with the failure scopes, and once with the success scopes.
// success_scopes : the scopes that we are testing that should succeed
// returns a tuple of (failure_body, success_body)
// Should return a String error if on unexpected status code, allowing unwrapping in tests.
pub async fn test<T>(
&self,
req_gen: T,
success_scopes: Scopes,
) -> Result<(serde_json::Value, serde_json::Value), String>
where
T: Fn() -> TestRequest,
{
// First, create a PAT with failure scopes
let failure_scopes = self
.failure_scopes
.unwrap_or(Scopes::all() ^ success_scopes);
let access_token_all_others =
create_test_pat(failure_scopes, self.user_id, &self.test_env.db).await;
// Create a PAT with the success scopes
let access_token = create_test_pat(success_scopes, self.user_id, &self.test_env.db).await;
// Perform test twice, once with each PAT
// the first time, we expect a 401 (or known failure code)
let req = req_gen()
.append_header(("Authorization", access_token_all_others.as_str()))
.to_request();
let resp = self.test_env.call(req).await;
if resp.status().as_u16() != self.expected_failure_code {
return Err(format!(
"Expected failure code {}, got {}",
self.expected_failure_code,
resp.status().as_u16()
));
}
let failure_body = if resp.status() == 200
&& resp.headers().contains_key("Content-Type")
&& resp.headers().get("Content-Type").unwrap() == "application/json"
{
test::read_body_json(resp).await
} else {
serde_json::Value::Null
};
// The second time, we expect a success code
let req = req_gen()
.append_header(("Authorization", access_token.as_str()))
.to_request();
let resp = self.test_env.call(req).await;
if !(resp.status().is_success() || resp.status().is_redirection()) {
return Err(format!(
"Expected success code, got {}",
resp.status().as_u16()
));
}
let success_body = if resp.status() == 200
&& resp.headers().contains_key("Content-Type")
&& resp.headers().get("Content-Type").unwrap() == "application/json"
{
test::read_body_json(resp).await
} else {
serde_json::Value::Null
};
Ok((failure_body, success_body))
}
}