Testing search prod (#791)

* testing push

* lowers it

* removed unwrap

* reduced to 500

* Really took down time

* reorders chunking

* rebuild docker

* reverted most changes

* cargo fmt

* reduced meilisearch limit

* added logs, removed deletion of index

* one client creation

* changes

* reverted gallery cahnge

* testing re-splitting again

* Remove chunking + index deletion

* Bring back chunking

* Update chunk size

---------

Co-authored-by: Jai A <jaiagr+gpg@pm.me>
Co-authored-by: Geometrically <18202329+Geometrically@users.noreply.github.com>

src/routes/v2/projects.rs

@@ -67,16 +67,15 @@ pub async fn project_search(
             .into_iter()
             .map(|facets| {
                 facets
-                        .into_iter()
-                        .map(|facet| {
-                            if facet.is_array() {
-                                serde_json::from_value::<Vec<String>>(facet).unwrap_or_default()
-                            } else {
-                                vec![serde_json::from_value::<String>(facet.clone())
-                                    .unwrap_or_default()]
-                            }
-                        })
-                        .collect_vec()
+                    .into_iter()
+                    .map(|facet| {
+                        if facet.is_array() {
+                            serde_json::from_value::<Vec<String>>(facet).unwrap_or_default()
+                        } else {
+                            vec![serde_json::from_value::<String>(facet).unwrap_or_default()]
+                        }
+                    })
+                    .collect_vec()
             })
             .collect_vec();
 

src/routes/v2/users.rs

@@ -178,38 +178,17 @@ pub async fn user_icon_edit(
     .or_else(v2_reroute::flatten_404_error)
 }
 
-#[derive(Deserialize)]
-pub struct RemovalType {
-    #[serde(default = "default_removal")]
-    removal_type: String,
-}
-
-fn default_removal() -> String {
-    "partial".into()
-}
-
 #[delete("{id}")]
 pub async fn user_delete(
     req: HttpRequest,
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
-    removal_type: web::Query<RemovalType>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
-    let removal_type = removal_type.into_inner();
-    v3::users::user_delete(
-        req,
-        info,
-        pool,
-        web::Query(v3::users::RemovalType {
-            removal_type: removal_type.removal_type,
-        }),
-        redis,
-        session_queue,
-    )
-    .await
-    .or_else(v2_reroute::flatten_404_error)
+    v3::users::user_delete(req, info, pool, redis, session_queue)
+        .await
+        .or_else(v2_reroute::flatten_404_error)
 }
 
 #[get("{id}/follows")]

src/routes/v2/versions.rs

@@ -8,6 +8,7 @@ use crate::models::projects::{Dependency, FileType, Version, VersionStatus, Vers
 use crate::models::v2::projects::LegacyVersion;
 use crate::queue::session::AuthQueue;
 use crate::routes::{v2_reroute, v3};
+use crate::search::SearchConfig;
 use actix_web::{delete, get, patch, post, web, HttpRequest, HttpResponse};
 use chrono::{DateTime, Utc};
 use serde::{Deserialize, Serialize};
@@ -263,12 +264,13 @@ pub async fn version_edit(
 #[delete("{version_id}")]
 pub async fn version_delete(
     req: HttpRequest,
-    info: web::Path<(models::ids::VersionId,)>,
+    info: web::Path<(VersionId,)>,
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
+    search_config: web::Data<SearchConfig>,
 ) -> Result<HttpResponse, ApiError> {
-    v3::versions::version_delete(req, info, pool, redis, session_queue)
+    v3::versions::version_delete(req, info, pool, redis, session_queue, search_config)
         .await
         .or_else(v2_reroute::flatten_404_error)
 }

src/routes/v3/projects.rs

@@ -21,6 +21,7 @@ use crate::models::teams::ProjectPermissions;
 use crate::models::threads::MessageBody;
 use crate::queue::session::AuthQueue;
 use crate::routes::ApiError;
+use crate::search::indexing::remove_documents;
 use crate::search::{search_for_project, SearchConfig, SearchError};
 use crate::util::img;
 use crate::util::routes::read_from_payload;
@@ -28,7 +29,6 @@ use crate::util::validate::validation_errors_to_string;
 use actix_web::{web, HttpRequest, HttpResponse};
 use chrono::{DateTime, Utc};
 use futures::TryStreamExt;
-use meilisearch_sdk::indexes::IndexesResults;
 use serde::{Deserialize, Serialize};
 use serde_json::json;
 use sqlx::PgPool;
@@ -231,7 +231,7 @@ pub async fn project_edit(
     req: HttpRequest,
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
-    config: web::Data<SearchConfig>,
+    search_config: web::Data<SearchConfig>,
     new_project: web::Json<EditProject>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
@@ -472,7 +472,15 @@ pub async fn project_edit(
                 .await?;
 
                 if project_item.inner.status.is_searchable() && !status.is_searchable() {
-                    delete_from_index(id.into(), config).await?;
+                    remove_documents(
+                        &project_item
+                            .versions
+                            .into_iter()
+                            .map(|x| x.into())
+                            .collect::<Vec<_>>(),
+                        &search_config,
+                    )
+                    .await?;
                 }
             }
 
@@ -910,21 +918,6 @@ pub async fn project_search(
     Ok(HttpResponse::Ok().json(results))
 }
 
-pub async fn delete_from_index(
-    id: ProjectId,
-    config: web::Data<SearchConfig>,
-) -> Result<(), meilisearch_sdk::errors::Error> {
-    let client = meilisearch_sdk::client::Client::new(&*config.address, &*config.key);
-
-    let indexes: IndexesResults = client.get_indexes().await?;
-
-    for index in indexes.results {
-        index.delete_document(id.to_string()).await?;
-    }
-
-    Ok(())
-}
-
 //checks the validity of a project id or slug
 pub async fn project_get_check(
     info: web::Path<(String,)>,
@@ -2045,7 +2038,7 @@ pub async fn project_delete(
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
-    config: web::Data<SearchConfig>,
+    search_config: web::Data<SearchConfig>,
     session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
     let user = get_user_from_headers(
@@ -2118,7 +2111,15 @@ pub async fn project_delete(
 
     transaction.commit().await?;
 
-    delete_from_index(project.inner.id.into(), config).await?;
+    remove_documents(
+        &project
+            .versions
+            .into_iter()
+            .map(|x| x.into())
+            .collect::<Vec<_>>(),
+        &search_config,
+    )
+    .await?;
 
     if result.is_some() {
         Ok(HttpResponse::NoContent().body(""))

src/routes/v3/users.rs

@@ -546,21 +546,10 @@ pub async fn user_icon_edit(
     }
 }
 
-#[derive(Deserialize)]
-pub struct RemovalType {
-    #[serde(default = "default_removal")]
-    pub removal_type: String,
-}
-
-fn default_removal() -> String {
-    "partial".into()
-}
-
 pub async fn user_delete(
     req: HttpRequest,
     info: web::Path<(String,)>,
     pool: web::Data<PgPool>,
-    removal_type: web::Query<RemovalType>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
 ) -> Result<HttpResponse, ApiError> {
@@ -584,13 +573,7 @@ pub async fn user_delete(
 
         let mut transaction = pool.begin().await?;
 
-        let result = User::remove(
-            id,
-            removal_type.removal_type == "full",
-            &mut transaction,
-            &redis,
-        )
-        .await?;
+        let result = User::remove(id, &mut transaction, &redis).await?;
 
         transaction.commit().await?;
 

src/routes/v3/versions.rs

@@ -20,6 +20,8 @@ use crate::models::projects::{skip_nulls, Loader};
 use crate::models::projects::{Dependency, FileType, VersionStatus, VersionType};
 use crate::models::teams::ProjectPermissions;
 use crate::queue::session::AuthQueue;
+use crate::search::indexing::remove_documents;
+use crate::search::SearchConfig;
 use crate::util::img;
 use crate::util::validate::validation_errors_to_string;
 use actix_web::{web, HttpRequest, HttpResponse};
@@ -934,6 +936,7 @@ pub async fn version_delete(
     pool: web::Data<PgPool>,
     redis: web::Data<RedisPool>,
     session_queue: web::Data<AuthQueue>,
+    search_config: web::Data<SearchConfig>,
 ) -> Result<HttpResponse, ApiError> {
     let user = get_user_from_headers(
         &req,
@@ -1001,7 +1004,7 @@ pub async fn version_delete(
 
     let result =
         database::models::Version::remove_full(version.inner.id, &redis, &mut transaction).await?;
-
+    remove_documents(&[version.inner.id.into()], &search_config).await?;
     database::models::Project::clear_cache(version.inner.project_id, None, Some(true), &redis)
         .await?;