You've already forked AstralRinth
forked from didirus/AstralRinth
Fix indexing, upgrade MeiliSearch sdk (#100)
* Hotfixes for indexing * Handles missing INDEX_CACHE_PATH environment variable * Exits on startup if environment variables are missing. The flag --allow-missing-vars disables this, but that is generally a bad idea, since most environment variables are required (and the ones that aren't should be marked as such). * Disables the query loggers * Upgrade meilisearch-sdk to 0.4.0 for MeiliSearch 0.16 support * Fix swap of Forge and Fabric labeling
This commit is contained in:
@@ -119,21 +119,23 @@ lazy_static::lazy_static! {
|
||||
pub async fn index_curseforge(
|
||||
start_index: u32,
|
||||
end_index: u32,
|
||||
cache_path: &std::path::Path,
|
||||
cache_path: Option<&std::path::Path>,
|
||||
) -> Result<Vec<UploadSearchMod>, IndexingError> {
|
||||
info!("Indexing curseforge mods!");
|
||||
let start = std::time::Instant::now();
|
||||
|
||||
let mut docs_to_add: Vec<UploadSearchMod> = vec![];
|
||||
|
||||
let cache = std::fs::File::open(cache_path)
|
||||
let cache = cache_path
|
||||
.map(std::fs::File::open)
|
||||
.and_then(Result::ok)
|
||||
.map(std::io::BufReader::new)
|
||||
.map(serde_json::from_reader::<_, Vec<u32>>);
|
||||
|
||||
let requested_ids;
|
||||
|
||||
// This caching system can't handle segmented indexing
|
||||
if let Ok(Ok(mut cache)) = cache {
|
||||
if let Some(Ok(mut cache)) = cache {
|
||||
let end = cache.last().copied().unwrap_or(start_index);
|
||||
cache.extend(end..end_index);
|
||||
requested_ids = serde_json::to_string(&cache)?;
|
||||
@@ -167,11 +169,13 @@ pub async fn index_curseforge(
|
||||
// Only write to the cache if this doesn't skip mods at the start
|
||||
// The caching system iterates through all ids normally past the last
|
||||
// id in the cache, so the end_index shouldn't matter.
|
||||
if start_index <= 1 {
|
||||
let mut ids = curseforge_mods.iter().map(|m| m.id).collect::<Vec<_>>();
|
||||
ids.sort_unstable();
|
||||
if let Err(e) = std::fs::write(cache_path, serde_json::to_string(&ids)?) {
|
||||
log::warn!("Error writing to index id cache: {}", e);
|
||||
if let Some(path) = cache_path {
|
||||
if start_index <= 1 {
|
||||
let mut ids = curseforge_mods.iter().map(|m| m.id).collect::<Vec<_>>();
|
||||
ids.sort_unstable();
|
||||
if let Err(e) = std::fs::write(path, serde_json::to_string(&ids)?) {
|
||||
log::warn!("Error writing to index id cache: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -192,8 +196,8 @@ pub async fn index_curseforge(
|
||||
for file in curseforge_mod.latest_files {
|
||||
for version in file.game_version {
|
||||
match &*version {
|
||||
"Fabric" => loaders.forge = true,
|
||||
"Forge" => loaders.fabric = true,
|
||||
"Fabric" => loaders.fabric = true,
|
||||
"Forge" => loaders.forge = true,
|
||||
"Rift" => loaders.rift = true,
|
||||
_ => (),
|
||||
}
|
||||
@@ -309,7 +313,6 @@ pub async fn index_curseforge(
|
||||
modified_timestamp: curseforge_mod.date_modified.timestamp(),
|
||||
latest_version,
|
||||
host: Cow::Borrowed("curseforge"),
|
||||
empty: Cow::Borrowed("{}{}{}"),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -112,7 +112,6 @@ pub async fn index_local(pool: PgPool) -> Result<Vec<UploadSearchMod>, IndexingE
|
||||
modified_timestamp: mod_data.updated.timestamp(),
|
||||
latest_version,
|
||||
host: Cow::Borrowed("modrinth"),
|
||||
empty: Cow::Borrowed("{}{}{}"),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -225,6 +224,5 @@ pub async fn query_one(
|
||||
modified_timestamp: mod_data.updated.timestamp(),
|
||||
latest_version,
|
||||
host: Cow::Borrowed("modrinth"),
|
||||
empty: Cow::Borrowed("{}{}{}"),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -63,7 +63,7 @@ pub async fn index_mods(
|
||||
) -> Result<(), IndexingError> {
|
||||
let mut docs_to_add: Vec<UploadSearchMod> = vec![];
|
||||
|
||||
let cache_path = std::path::PathBuf::from(std::env::var_os("INDEX_CACHE_PATH").unwrap());
|
||||
let cache_path = std::env::var_os("INDEX_CACHE_PATH").map(std::path::PathBuf::from);
|
||||
|
||||
if settings.index_local {
|
||||
docs_to_add.append(&mut index_local(pool.clone()).await?);
|
||||
@@ -74,7 +74,7 @@ pub async fn index_mods(
|
||||
.map(|i| i.parse().unwrap())
|
||||
.unwrap_or(450_000);
|
||||
|
||||
docs_to_add.append(&mut index_curseforge(1, end_index, &cache_path).await?);
|
||||
docs_to_add.append(&mut index_curseforge(1, end_index, cache_path.as_deref()).await?);
|
||||
}
|
||||
|
||||
// Write Indices
|
||||
@@ -270,7 +270,6 @@ fn default_settings() -> Settings {
|
||||
"categories".to_string(),
|
||||
"versions".to_string(),
|
||||
"author".to_string(),
|
||||
"empty".to_string(),
|
||||
];
|
||||
|
||||
Settings::new()
|
||||
|
||||
Reference in New Issue
Block a user