Optimize and fix some bugs in indexing (#98)

* Improve curseforge and local indexing

This should make curseforge indexing more efficient, and reuses
some of the normal local indexing for the queued indexing of
recently created mods.

* Unify impls for single and multiple routes for mods and versions

This uses the same backend for the single and multiple query
routes so that they no longer return inconsistent information.

* Cache valid curseforge mod ids to reduce request load

This caches the ids of minecraft mods and reuses them on indexing
to reduce the amount of unused addons that are returned.
This commit is contained in:
Aeledfyr
2020-11-03 18:55:50 -06:00
committed by GitHub
parent da79386cc3
commit d477874535
15 changed files with 745 additions and 343 deletions

View File

@@ -384,6 +384,74 @@
]
}
},
"225597042db9c2d95296ea6bbeda4e99ffc9ddfab3991c8637ac3f4749ece6f3": {
"query": "\n SELECT m.id, m.title, m.description, m.downloads, m.icon_url, m.body_url, m.published, m.updated, m.team_id\n FROM mods m\n WHERE id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "title",
"type_info": "Varchar"
},
{
"ordinal": 2,
"name": "description",
"type_info": "Varchar"
},
{
"ordinal": 3,
"name": "downloads",
"type_info": "Int4"
},
{
"ordinal": 4,
"name": "icon_url",
"type_info": "Varchar"
},
{
"ordinal": 5,
"name": "body_url",
"type_info": "Varchar"
},
{
"ordinal": 6,
"name": "published",
"type_info": "Timestamptz"
},
{
"ordinal": 7,
"name": "updated",
"type_info": "Timestamptz"
},
{
"ordinal": 8,
"name": "team_id",
"type_info": "Int8"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false,
false,
false,
false,
true,
false,
false,
false,
false
]
}
},
"25131559cb73a088000ab6379a769233440ade6c7511542da410065190d203fc": {
"query": "\n SELECT id FROM loaders\n WHERE loader = $1\n ",
"describe": {
@@ -664,6 +732,26 @@
"nullable": []
}
},
"621c3c5e5b3ac00c291b5f9cae2134420ef3e23f1f236267c4132222299c87a2": {
"query": "\n SELECT gv.version FROM versions\n INNER JOIN game_versions_versions gvv ON gvv.joining_version_id=versions.id\n INNER JOIN game_versions gv ON gvv.game_version_id=gv.id\n WHERE versions.mod_id = $1\n ORDER BY gv.created ASC\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "version",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false
]
}
},
"6562c876826ad3091a14eb50fa1f961a971c1d1bb158fc3dcb55d469a73facc6": {
"query": "\n SELECT v.mod_id, v.author_id, v.name, v.version_number,\n v.changelog_url, v.date_published, v.downloads,\n v.release_channel\n FROM versions v\n WHERE v.id = $1\n ",
"describe": {
@@ -770,6 +858,26 @@
]
}
},
"72c1e6de8f2c8d89be030454eeab6d5c9695164af2ebfb8d7e94b2deee4f130d": {
"query": "\n SELECT c.category\n FROM mods_categories mc\n INNER JOIN categories c ON mc.joining_category_id=c.id\n WHERE mc.joining_mod_id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "category",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false
]
}
},
"72c75313688dfd88a659c5250c71b9899abd6186ab32a067a7d4b8a0846ebd18": {
"query": "\n INSERT INTO game_versions (version, type, created)\n VALUES ($1, COALESCE($2, 'other'), COALESCE($3, timezone('utc', now())))\n ON CONFLICT (version) DO UPDATE\n SET type = COALESCE($2, game_versions.type),\n created = COALESCE($3, game_versions.created)\n RETURNING id\n ",
"describe": {
@@ -986,6 +1094,33 @@
]
}
},
"9d95d136d0e6eedee57e6aa524232c02609b89e4e26032e07403aabb69bea0d8": {
"query": "\n SELECT u.id, u.username FROM users u\n INNER JOIN team_members tm ON tm.user_id = u.id\n WHERE tm.team_id = $2 AND tm.role = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "id",
"type_info": "Int8"
},
{
"ordinal": 1,
"name": "username",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Text",
"Int8"
]
},
"nullable": [
false,
false
]
}
},
"a2a99a640468a9fb8f0718e5aea6740cf5b33dafd5e038c154d6a13674fa999b": {
"query": "\n INSERT INTO mods (\n id, team_id, title, description, body_url,\n published, downloads, icon_url, issues_url,\n source_url, wiki_url, status\n )\n VALUES (\n $1, $2, $3, $4, $5,\n $6, $7, $8, $9,\n $10, $11, $12\n )\n ",
"describe": {
@@ -1139,6 +1274,26 @@
"nullable": []
}
},
"b34577335d30ffe30327cdd5b3c029a187a1cae27bea99ff0bcf062f87468fe7": {
"query": "\n SELECT loaders.loader FROM versions\n INNER JOIN loaders_versions lv ON lv.version_id = versions.id\n INNER JOIN loaders ON loaders.id = lv.loader_id\n WHERE versions.mod_id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "loader",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false
]
}
},
"b903ac4e686ef85ba28d698c668da07860e7f276b261d8f2cebb74e73b094970": {
"query": "\n DELETE FROM hashes\n WHERE EXISTS(\n SELECT 1 FROM files WHERE\n (files.version_id = $1) AND\n (hashes.file_id = files.id)\n )\n ",
"describe": {
@@ -1248,26 +1403,6 @@
]
}
},
"c0899dcff4d7bc1ba3e953e5099210316bff2f98e6ab77ba84bc612eac4bce0a": {
"query": "\n SELECT gv.version FROM versions\n INNER JOIN game_versions_versions gvv ON gvv.joining_version_id=versions.id\n INNER JOIN game_versions gv ON gvv.game_version_id=gv.id\n WHERE versions.mod_id = $1\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "version",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false
]
}
},
"c1fddbf97350871b79cb0c235b1f7488c6616b7c1dfbde76a712fd57e91ba158": {
"query": "\n SELECT id FROM game_versions\n WHERE version = $1\n ",
"describe": {
@@ -1288,6 +1423,26 @@
]
}
},
"c59de96d66ebf26c0497674308550da125e3ce2314a8ae5b2f95d892f4205f90": {
"query": "\n SELECT gv.version FROM versions\n INNER JOIN game_versions_versions gvv ON gvv.joining_version_id=versions.id\n INNER JOIN game_versions gv ON gvv.game_version_id=gv.id\n WHERE versions.mod_id = $1\n ORDER BY gv.created ASC\n ",
"describe": {
"columns": [
{
"ordinal": 0,
"name": "version",
"type_info": "Varchar"
}
],
"parameters": {
"Left": [
"Int8"
]
},
"nullable": [
false
]
}
},
"c64c487b56a25b252ff070fe03a7416e84260df8a6f938a018cc768598e9435b": {
"query": "\n SELECT category FROM categories\n WHERE id = $1\n ",
"describe": {