Commonized networking (#3310)

* Fix not being able to connect to local friends socket

* Start basic work on tunneling protocol and move some code into a common crate

* Commonize message serialization logic

* Serialize Base62Ids as u64 when human-readability is not required

* Move ActiveSockets tuple into struct

* Make CI run when rust-common is updated

CI is currently broken for labrinth, however

* Fix theseus-release.yml to reference itself correctly

* Implement Labrinth side of tunneling

* Implement non-friend part of theseus tunneling

* Implement client-side except for socket loop

* Implement the socket loop

Doesn't work though. Debugging time!

* Fix config.rs

* Fix deadlock in labrinth socket handling

* Update dockerfile

* switch to workspace prepare at root level

* Wait for connection before tunneling in playground

* Move rust-common into labrinth

* Remove rust-common references from Actions

* Revert "Update dockerfile"

This reverts commit 3caad59bb474ce425d0b8928d7cee7ae1a5011bd.

* Fix Docker build

* Rebuild Theseus if common code changes

* Allow multiple connections from the same user

* Fix test building

* Move FriendSocketListening and FriendSocketStoppedListening to non-panicking TODO for now

* Make message_serialization macro take varargs for binary messages

* Improve syntax of message_serialization macro

* Remove the ability to connect to a virtual socket, and disable the ability to listen on one

* Allow the app to compile without running labrinth

* Clippy fix

* Update Rust and Clippy fix again

---------

Co-authored-by: Jai A <jaiagr+gpg@pm.me>

apps/labrinth/src/routes/v3/analytics_get.rs

@@ -1,4 +1,5 @@
 use super::ApiError;
+use crate::common::ids::base62_impl::to_base62;
 use crate::database;
 use crate::database::redis::RedisPool;
 use crate::models::teams::ProjectPermissions;
@@ -6,7 +7,7 @@ use crate::{
     auth::get_user_from_headers,
     database::models::user_item,
     models::{
-        ids::{base62_impl::to_base62, ProjectId, VersionId},
+        ids::{ProjectId, VersionId},
         pats::Scopes,
     },
     queue::session::AuthQueue,

apps/labrinth/src/routes/v3/collections.rs

@@ -1,12 +1,12 @@
 use crate::auth::checks::is_visible_collection;
 use crate::auth::{filter_visible_collections, get_user_from_headers};
+use crate::common::ids::base62_impl::parse_base62;
 use crate::database::models::{
     collection_item, generate_collection_id, project_item,
 };
 use crate::database::redis::RedisPool;
 use crate::file_hosting::FileHost;
 use crate::models::collections::{Collection, CollectionStatus};
-use crate::models::ids::base62_impl::parse_base62;
 use crate::models::ids::{CollectionId, ProjectId};
 use crate::models::pats::Scopes;
 use crate::queue::session::AuthQueue;

apps/labrinth/src/routes/v3/friends.rs

@@ -1,11 +1,12 @@
 use crate::auth::get_user_from_headers;
+use crate::common::networking::message::ServerToClientMessage;
 use crate::database::models::UserId;
 use crate::database::redis::RedisPool;
 use crate::models::pats::Scopes;
 use crate::models::users::UserFriend;
 use crate::queue::session::AuthQueue;
 use crate::queue::socket::ActiveSockets;
-use crate::routes::internal::statuses::{close_socket, ServerToClientMessage};
+use crate::routes::internal::statuses::send_message_to_user;
 use crate::routes::ApiError;
 use actix_web::{delete, get, post, web, HttpRequest, HttpResponse};
 use chrono::Utc;
@@ -76,22 +77,16 @@ pub async fn add_friend(
                 friend_id: UserId,
                 sockets: &ActiveSockets,
             ) -> Result<(), ApiError> {
-                if let Some(pair) = sockets.auth_sockets.get(&user_id.into()) {
-                    let (friend_status, _) = pair.value();
-                    if let Some(socket) =
-                        sockets.auth_sockets.get(&friend_id.into())
-                    {
-                        let (_, socket) = socket.value();
-
-                        let _ = socket
-                            .clone()
-                            .text(serde_json::to_string(
-                                &ServerToClientMessage::StatusUpdate {
-                                    status: friend_status.clone(),
-                                },
-                            )?)
-                            .await;
-                    }
+                if let Some(friend_status) = sockets.get_status(user_id.into())
+                {
+                    send_message_to_user(
+                        sockets,
+                        friend_id.into(),
+                        &ServerToClientMessage::StatusUpdate {
+                            status: friend_status.clone(),
+                        },
+                    )
+                    .await?;
                 }
 
                 Ok(())
@@ -121,20 +116,12 @@ pub async fn add_friend(
             .insert(&mut transaction)
             .await?;
 
-            if let Some(socket) = db.auth_sockets.get(&friend.id.into()) {
-                let (_, socket) = socket.value();
-
-                if socket
-                    .clone()
-                    .text(serde_json::to_string(
-                        &ServerToClientMessage::FriendRequest { from: user.id },
-                    )?)
-                    .await
-                    .is_err()
-                {
-                    close_socket(user.id, &pool, &db).await?;
-                }
-            }
+            send_message_to_user(
+                &db,
+                friend.id.into(),
+                &ServerToClientMessage::FriendRequest { from: user.id },
+            )
+            .await?;
         }
 
         transaction.commit().await?;
@@ -178,18 +165,12 @@ pub async fn remove_friend(
         )
         .await?;
 
-        if let Some(socket) = db.auth_sockets.get(&friend.id.into()) {
-            let (_, socket) = socket.value();
-
-            let _ = socket
-                .clone()
-                .text(serde_json::to_string(
-                    &ServerToClientMessage::FriendRequestRejected {
-                        from: user.id,
-                    },
-                )?)
-                .await;
-        }
+        send_message_to_user(
+            &db,
+            friend.id.into(),
+            &ServerToClientMessage::FriendRequestRejected { from: user.id },
+        )
+        .await?;
 
         transaction.commit().await?;
 

apps/labrinth/src/routes/v3/oauth_clients.rs

@@ -1,19 +1,7 @@
 use std::{collections::HashSet, fmt::Display, sync::Arc};
 
-use actix_web::{
-    delete, get, patch, post,
-    web::{self, scope},
-    HttpRequest, HttpResponse,
-};
-use chrono::Utc;
-use itertools::Itertools;
-use rand::{distributions::Alphanumeric, Rng, SeedableRng};
-use rand_chacha::ChaCha20Rng;
-use serde::{Deserialize, Serialize};
-use sqlx::PgPool;
-use validator::Validate;
-
 use super::ApiError;
+use crate::common::ids::base62_impl::parse_base62;
 use crate::{
     auth::{checks::ValidateAuthorized, get_user_from_headers},
     database::{
@@ -35,13 +23,21 @@ use crate::{
     util::validate::validation_errors_to_string,
 };
 use crate::{
-    file_hosting::FileHost,
-    models::{
-        ids::base62_impl::parse_base62,
-        oauth_clients::DeleteOAuthClientQueryParam,
-    },
+    file_hosting::FileHost, models::oauth_clients::DeleteOAuthClientQueryParam,
     util::routes::read_from_payload,
 };
+use actix_web::{
+    delete, get, patch, post,
+    web::{self, scope},
+    HttpRequest, HttpResponse,
+};
+use chrono::Utc;
+use itertools::Itertools;
+use rand::{distributions::Alphanumeric, Rng, SeedableRng};
+use rand_chacha::ChaCha20Rng;
+use serde::{Deserialize, Serialize};
+use sqlx::PgPool;
+use validator::Validate;
 
 use crate::database::models::oauth_client_item::OAuthClient as DBOAuthClient;
 use crate::models::ids::OAuthClientId as ApiOAuthClientId;

apps/labrinth/src/routes/v3/organizations.rs

@@ -3,13 +3,13 @@ use std::sync::Arc;
 
 use super::ApiError;
 use crate::auth::{filter_visible_projects, get_user_from_headers};
+use crate::common::ids::base62_impl::parse_base62;
 use crate::database::models::team_item::TeamMember;
 use crate::database::models::{
     generate_organization_id, team_item, Organization,
 };
 use crate::database::redis::RedisPool;
 use crate::file_hosting::FileHost;
-use crate::models::ids::base62_impl::parse_base62;
 use crate::models::ids::UserId;
 use crate::models::organizations::OrganizationId;
 use crate::models::pats::Scopes;
@@ -786,7 +786,7 @@ pub async fn organization_projects_add(
 
         let organization_owner_user_id = sqlx::query!(
             "
-            SELECT u.id 
+            SELECT u.id
             FROM team_members
             INNER JOIN users u ON u.id = team_members.user_id
             WHERE team_id = $1 AND is_owner = TRUE
@@ -969,7 +969,7 @@ pub async fn organization_projects_remove(
         sqlx::query!(
             "
             UPDATE team_members
-            SET 
+            SET
                 is_owner = TRUE,
                 accepted = TRUE,
                 permissions = $2,

apps/labrinth/src/routes/v3/project_creation.rs

@@ -1,5 +1,6 @@
 use super::version_creation::{try_create_version_fields, InitialVersionData};
 use crate::auth::{get_user_from_headers, AuthenticationError};
+use crate::common::ids::base62_impl::to_base62;
 use crate::database::models::loader_fields::{
     Loader, LoaderField, LoaderFieldEnumValue,
 };
@@ -8,7 +9,6 @@ use crate::database::models::{self, image_item, User};
 use crate::database::redis::RedisPool;
 use crate::file_hosting::{FileHost, FileHostingError};
 use crate::models::error::ApiError;
-use crate::models::ids::base62_impl::to_base62;
 use crate::models::ids::{ImageId, OrganizationId};
 use crate::models::images::{Image, ImageContext};
 use crate::models::pats::Scopes;

apps/labrinth/src/routes/v3/projects.rs

@@ -3,6 +3,7 @@ use std::sync::Arc;
 
 use crate::auth::checks::{filter_visible_versions, is_visible_project};
 use crate::auth::{filter_visible_projects, get_user_from_headers};
+use crate::common::ids::base62_impl::parse_base62;
 use crate::database::models::notification_item::NotificationBuilder;
 use crate::database::models::project_item::{GalleryItem, ModCategory};
 use crate::database::models::thread_item::ThreadMessageBuilder;
@@ -11,7 +12,6 @@ use crate::database::redis::RedisPool;
 use crate::database::{self, models as db_models};
 use crate::file_hosting::FileHost;
 use crate::models;
-use crate::models::ids::base62_impl::parse_base62;
 use crate::models::images::ImageContext;
 use crate::models::notifications::NotificationBody;
 use crate::models::pats::Scopes;

apps/labrinth/src/routes/v3/reports.rs

@@ -1,4 +1,5 @@
 use crate::auth::{check_is_moderator_from_headers, get_user_from_headers};
+use crate::common::ids::base62_impl::parse_base62;
 use crate::database;
 use crate::database::models::image_item;
 use crate::database::models::thread_item::{
@@ -6,9 +7,7 @@ use crate::database::models::thread_item::{
 };
 use crate::database::redis::RedisPool;
 use crate::models::ids::ImageId;
-use crate::models::ids::{
-    base62_impl::parse_base62, ProjectId, UserId, VersionId,
-};
+use crate::models::ids::{ProjectId, UserId, VersionId};
 use crate::models::images::{Image, ImageContext};
 use crate::models::pats::Scopes;
 use crate::models::reports::{ItemType, Report};

apps/labrinth/src/routes/v3/versions.rs

@@ -5,6 +5,7 @@ use crate::auth::checks::{
     filter_visible_versions, is_visible_project, is_visible_version,
 };
 use crate::auth::get_user_from_headers;
+use crate::common::ids::base62_impl::parse_base62;
 use crate::database;
 use crate::database::models::loader_fields::{
     self, LoaderField, LoaderFieldEnumValue, VersionField,
@@ -13,7 +14,6 @@ use crate::database::models::version_item::{DependencyBuilder, LoaderVersion};
 use crate::database::models::{image_item, Organization};
 use crate::database::redis::RedisPool;
 use crate::models;
-use crate::models::ids::base62_impl::parse_base62;
 use crate::models::ids::VersionId;
 use crate::models::images::ImageContext;
 use crate::models::pats::Scopes;
@@ -444,7 +444,7 @@ pub async fn version_edit_helper(
                     .collect::<Vec<i32>>();
                 sqlx::query!(
                     "
-                    DELETE FROM version_fields 
+                    DELETE FROM version_fields
                     WHERE version_id = $1
                     AND field_id = ANY($2)
                     ",