diff --git a/.env b/.env index 0bbda3b4..16e15920 100644 --- a/.env +++ b/.env @@ -10,11 +10,18 @@ MEILISEARCH_ADDR=http://localhost:7700 BIND_ADDR=127.0.0.1:8000 MOCK_FILE_PATH=/tmp/modrinth -BACKBLAZE_ENABLED=false +STORAGE_BACKEND=local + BACKBLAZE_KEY_ID=none BACKBLAZE_KEY=none BACKBLAZE_BUCKET_ID=none +S3_ACCESS_TOKEN=none +S3_SECRET=none +S3_URL=none +S3_REGION=none +S3_BUCKET_NAME=none + INDEX_CURSEFORGE=false MAX_CURSEFORGE_ID=450000 # 1 hour diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 6cfc40fa..3757ded0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -45,4 +45,9 @@ jobs: BACKBLAZE_BUCKET_ID: ${{ secrets.BACKBLAZE_BUCKET_ID }} BACKBLAZE_KEY: ${{ secrets.BACKBLAZE_KEY }} BACKBLAZE_KEY_ID: ${{ secrets.BACKBLAZE_KEY_ID }} - SQLX_OFFLINE: true + S3_ACCESS_TOKEN: ${{ secrets.S3_ACCESS_TOKEN }} + S3_SECRET: ${{ secrets.S3_SECRET }} + S3_URL: ${{ secrets.S3_URL }} + S3_REGION: ${{ secrets.S3_REGION }} + S3_BUCKET_NAME: ${{ secrets.S3_BUCKET_NAME }} + SQLX_OFFLINE: true \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 17196725..37b54280 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -326,6 +326,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" +[[package]] +name = "ahash" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217" + [[package]] name = "ahash" version = "0.5.3" @@ -351,6 +357,118 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4d25d88fd6b8041580a654f9d0c581a047baee2b3efee13275f2fc392fc75034" +[[package]] +name = "arrayref" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4c527152e37cf757a3f78aae5a06fbeefdb07ccc535c980a3208ee3060dd544" + +[[package]] +name = "arrayvec" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" + +[[package]] +name = "async-channel" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59740d83946db6a5af71ae25ddf9562c2b176b2ca42cf99a455f09f4a220d6b9" +dependencies = [ + "concurrent-queue", + "event-listener", + "futures-core", +] + +[[package]] +name = "async-executor" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d373d78ded7d0b3fa8039375718cde0aace493f2e34fb60f51cbf567562ca801" +dependencies = [ + "async-task", + "concurrent-queue", + "fastrand", + "futures-lite", + "once_cell", + "vec-arena", +] + +[[package]] +name = "async-global-executor" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0659b83a146398616883aa5199cdd1b055ec088a83c9a338eab3534f33f0622e" +dependencies = [ + "async-executor", + "async-io", + "futures-lite", + "num_cpus", + "once_cell", +] + +[[package]] +name = "async-io" +version = "1.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54bc4c1c7292475efb2253227dbcfad8fe1ca4c02bc62c510cc2f3da5c4704e" +dependencies = [ + "concurrent-queue", + "fastrand", + "futures-lite", + "libc", + "log", + "nb-connect", + "once_cell", + "parking", + "polling", + "vec-arena", + "waker-fn", + "winapi 0.3.9", +] + +[[package]] +name = "async-mutex" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479db852db25d9dbf6204e6cb6253698f175c15726470f78af0d918e99d6156e" +dependencies = [ + "event-listener", +] + +[[package]] +name = "async-std" +version = "1.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9fa76751505e8df1c7a77762f60486f60c71bbd9b8557f4da6ad47d083732ed" +dependencies = [ + "async-global-executor", + "async-io", + "async-mutex", + "blocking", + "crossbeam-utils", + "futures-channel", + "futures-core", + "futures-io", + "futures-lite", + "gloo-timers", + "kv-log-macro", + "log", + "memchr", + "num_cpus", + "once_cell", + "pin-project-lite", + "pin-utils", + "slab", + "wasm-bindgen-futures", +] + +[[package]] +name = "async-task" +version = "4.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ab27c1aa62945039e44edaeee1dc23c74cc0c303dd5fe0fb462a184f1c3a518" + [[package]] name = "async-trait" version = "0.1.41" @@ -371,6 +489,27 @@ dependencies = [ "num-traits", ] +[[package]] +name = "atomic-waker" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" + +[[package]] +name = "attohttpc" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe174d1b67f7b2bafed829c09db039301eb5841f66e43be2cf60b326e7f8e2cc" +dependencies = [ + "http", + "log", + "native-tls", + "openssl", + "serde", + "serde_json", + "url", +] + [[package]] name = "atty" version = "0.2.14" @@ -411,6 +550,31 @@ dependencies = [ "serde_urlencoded", ] +[[package]] +name = "aws-creds" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad53a54cb2c99990e96eefacde6f143dc6d471ab70809d26d360292be421d490" +dependencies = [ + "attohttpc", + "dirs", + "rust-ini", + "serde", + "serde-xml-rs", + "serde_derive", + "simpl", + "url", +] + +[[package]] +name = "aws-region" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f610af4a396f07592014dc3410f6ad78fab931852a99bb6cfdc1ad04b9329b80" +dependencies = [ + "simpl", +] + [[package]] name = "backtrace" version = "0.3.53" @@ -449,6 +613,17 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "blake2b_simd" +version = "0.5.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a" +dependencies = [ + "arrayref", + "arrayvec", + "constant_time_eq", +] + [[package]] name = "block-buffer" version = "0.9.0" @@ -458,6 +633,20 @@ dependencies = [ "generic-array", ] +[[package]] +name = "blocking" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5e170dbede1f740736619b776d7251cb1b9095c435c34d8ca9f57fcd2f335e9" +dependencies = [ + "async-channel", + "async-task", + "atomic-waker", + "fastrand", + "futures-lite", + "once_cell", +] + [[package]] name = "brotli-sys" version = "0.3.2" @@ -520,6 +709,12 @@ dependencies = [ "bytes", ] +[[package]] +name = "cache-padded" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" + [[package]] name = "cc" version = "1.0.61" @@ -561,12 +756,27 @@ dependencies = [ "bitflags", ] +[[package]] +name = "concurrent-queue" +version = "1.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" +dependencies = [ + "cache-padded", +] + [[package]] name = "const_fn" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ce90df4c658c62f12d78f7508cf92f9173e5184a539c10bfe54a3107b3ffd0f2" +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "cookie" version = "0.14.2" @@ -686,12 +896,41 @@ dependencies = [ "generic-array", ] +[[package]] +name = "dirs" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142995ed02755914747cc6ca76fc7e4583cd18578746716d0508ea6ed558b9ff" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e93d7f5705de3e49895a2b5e0b8855a1c27f080192ae9c32a6432d50741a57a" +dependencies = [ + "libc", + "redox_users", + "winapi 0.3.9", +] + [[package]] name = "discard" version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" +[[package]] +name = "dlv-list" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b391911b9a786312a10cb9d2b3d0735adfd5a8113eb3648de26a75e91b0826c" +dependencies = [ + "rand", +] + [[package]] name = "dotenv" version = "0.15.0" @@ -747,6 +986,21 @@ dependencies = [ "termcolor", ] +[[package]] +name = "event-listener" +version = "2.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59" + +[[package]] +name = "fastrand" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca5faf057445ce5c9d4329e382b2ce7ca38550ef3b73a5348362d5f24e0c7fe3" +dependencies = [ + "instant", +] + [[package]] name = "flate2" version = "1.0.18" @@ -844,6 +1098,21 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fc94b64bb39543b4e432f1790b6bf18e3ee3b74653c5449f63310e9a74b123c" +[[package]] +name = "futures-lite" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "381a7ad57b1bad34693f63f6f377e1abded7a9c85c9d3eb6771e11c60aaadab9" +dependencies = [ + "fastrand", + "futures-core", + "futures-io", + "memchr", + "parking", + "pin-project-lite", + "waker-fn", +] + [[package]] name = "futures-macro" version = "0.3.6" @@ -944,6 +1213,19 @@ version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aaf91faf136cb47367fa430cd46e37a788775e7fa104f8b4bcb3861dc389b724" +[[package]] +name = "gloo-timers" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47204a46aaff920a1ea58b11d03dec6f704287d27561724a4631e450654a891f" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "gumdrop" version = "0.8.0" @@ -983,6 +1265,16 @@ dependencies = [ "tracing", ] +[[package]] +name = "hashbrown" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96282e96bfcd3da0d3aa9938bedf1e50df3269b6db08b4876d2da0bb1a0841cf" +dependencies = [ + "ahash 0.3.8", + "autocfg", +] + [[package]] name = "hashbrown" version = "0.9.1" @@ -1128,7 +1420,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.9.1", ] [[package]] @@ -1192,6 +1484,15 @@ dependencies = [ "winapi-build", ] +[[package]] +name = "kv-log-macro" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" +dependencies = [ + "log", +] + [[package]] name = "labrinth" version = "0.1.0" @@ -1213,6 +1514,7 @@ dependencies = [ "meilisearch-sdk", "rand", "reqwest", + "rust-s3", "serde", "serde_json", "sha1", @@ -1307,6 +1609,12 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "meilisearch-sdk" version = "0.3.0" @@ -1415,6 +1723,16 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nb-connect" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8123a81538e457d44b933a02faf885d3fe8408806b23fa700e8f01c6c3a98998" +dependencies = [ + "libc", + "winapi 0.3.9", +] + [[package]] name = "net2" version = "0.2.35" @@ -1516,6 +1834,22 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "ordered-multimap" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e88f947c6799d5eff50e6cf8a2365c17ac4aa8f8f43aceeedc29b616d872a358" +dependencies = [ + "dlv-list", + "hashbrown 0.7.2", +] + +[[package]] +name = "parking" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" + [[package]] name = "parking_lot" version = "0.11.0" @@ -1586,6 +1920,19 @@ version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" +[[package]] +name = "polling" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab773feb154f12c49ffcfd66ab8bdcf9a1843f950db48b0d8be9d4393783b058" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "log", + "wepoll-sys", + "winapi 0.3.9", +] + [[package]] name = "ppv-lite86" version = "0.2.9" @@ -1675,6 +2022,17 @@ version = "0.1.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" +[[package]] +name = "redox_users" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de0737333e7a9502c789a36d7c7fa6092a49895d4faa31ca5df163857ded2e9d" +dependencies = [ + "getrandom 0.1.15", + "redox_syscall", + "rust-argon2", +] + [[package]] name = "regex" version = "1.4.1" @@ -1748,6 +2106,57 @@ dependencies = [ "quick-error", ] +[[package]] +name = "rust-argon2" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dab61250775933275e84053ac235621dfb739556d5c54a2f2e9313b7cf43a19" +dependencies = [ + "base64 0.12.3", + "blake2b_simd", + "constant_time_eq", + "crossbeam-utils", +] + +[[package]] +name = "rust-ini" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a3679dd538c876a7b606f3bb951c8a20fc281a0ff7795f59f7cb490e3f979e1" +dependencies = [ + "cfg-if 0.1.10", + "ordered-multimap", +] + +[[package]] +name = "rust-s3" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d1bfc082a5a767bcf1a9ac9c4ae1435a76ab521e3737d2bcda5ea3c55bf444b" +dependencies = [ + "async-std", + "aws-creds", + "aws-region", + "base64 0.13.0", + "cfg-if 0.1.10", + "chrono", + "futures", + "hex", + "hmac", + "http", + "log", + "md5", + "percent-encoding", + "reqwest", + "serde", + "serde-xml-rs", + "serde_derive", + "sha2", + "simpl", + "tokio", + "url", +] + [[package]] name = "rustc-demangle" version = "0.1.17" @@ -1832,6 +2241,18 @@ dependencies = [ "serde_derive", ] +[[package]] +name = "serde-xml-rs" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efe415925cf3d0bbb2fc47d09b56ce03eef51c5d56846468a39bcc293c7a846c" +dependencies = [ + "log", + "serde", + "thiserror", + "xml-rs", +] + [[package]] name = "serde_derive" version = "1.0.117" @@ -1909,6 +2330,12 @@ dependencies = [ "libc", ] +[[package]] +name = "simpl" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a30f10c911c0355f80f1c2faa8096efc4a58cdf8590b954d5b395efa071c711" + [[package]] name = "slab" version = "0.4.2" @@ -1947,7 +2374,7 @@ dependencies = [ [[package]] name = "sqlx" version = "0.4.0-beta.1" -source = "git+https://github.com/launchbadge/sqlx/#228729e2eec12d2ce3bd4493379d2ff20cf90505" +source = "git+https://github.com/launchbadge/sqlx/?branch=master#228729e2eec12d2ce3bd4493379d2ff20cf90505" dependencies = [ "sqlx-core", "sqlx-macros", @@ -1956,9 +2383,9 @@ dependencies = [ [[package]] name = "sqlx-core" version = "0.4.0-beta.1" -source = "git+https://github.com/launchbadge/sqlx/#228729e2eec12d2ce3bd4493379d2ff20cf90505" +source = "git+https://github.com/launchbadge/sqlx/?branch=master#228729e2eec12d2ce3bd4493379d2ff20cf90505" dependencies = [ - "ahash", + "ahash 0.5.3", "atoi", "base64 0.13.0", "bitflags", @@ -2000,7 +2427,7 @@ dependencies = [ [[package]] name = "sqlx-macros" version = "0.4.0-beta.1" -source = "git+https://github.com/launchbadge/sqlx/#228729e2eec12d2ce3bd4493379d2ff20cf90505" +source = "git+https://github.com/launchbadge/sqlx/?branch=master#228729e2eec12d2ce3bd4493379d2ff20cf90505" dependencies = [ "dotenv", "either", @@ -2021,7 +2448,7 @@ dependencies = [ [[package]] name = "sqlx-rt" version = "0.1.1" -source = "git+https://github.com/launchbadge/sqlx/#228729e2eec12d2ce3bd4493379d2ff20cf90505" +source = "git+https://github.com/launchbadge/sqlx/?branch=master#228729e2eec12d2ce3bd4493379d2ff20cf90505" dependencies = [ "actix-rt", "actix-threadpool", @@ -2504,6 +2931,12 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c" +[[package]] +name = "vec-arena" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eafc1b9b2dfc6f5529177b62cf806484db55b32dc7c9658a118e11bbeb33061d" + [[package]] name = "version_check" version = "0.1.5" @@ -2516,6 +2949,12 @@ version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b5a972e5669d67ba988ce3dc826706fb0a8b01471c088cb0b6110b805cc36aed" +[[package]] +name = "waker-fn" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" + [[package]] name = "want" version = "0.3.0" @@ -2616,6 +3055,15 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "wepoll-sys" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "142bc2cba3fe88be1a8fcb55c727fa4cd5b0cf2d7438722792e22f26f04bc1e0" +dependencies = [ + "cc", +] + [[package]] name = "whoami" version = "0.9.0" @@ -2698,3 +3146,9 @@ dependencies = [ "winapi 0.2.8", "winapi-build", ] + +[[package]] +name = "xml-rs" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b07db065a5cf61a7e4ba64f29e67db906fb1787316516c4e6e5ff0fea1efcd8a" diff --git a/Cargo.toml b/Cargo.toml index f7e3bb95..b33eac78 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ thiserror = "1.0.21" futures = "0.3.6" futures-timer = "3.0.2" +rust-s3 = "0.26.1" async-trait = "0.1.41" [dependencies.sqlx] diff --git a/src/file_hosting/mod.rs b/src/file_hosting/mod.rs index 21b41659..f40436ea 100644 --- a/src/file_hosting/mod.rs +++ b/src/file_hosting/mod.rs @@ -3,9 +3,13 @@ use thiserror::Error; mod backblaze; mod mock; +mod s3_host; pub use backblaze::BackblazeHost; pub use mock::MockHost; +use s3::creds::AwsCredsError; +use s3::S3Error; +pub use s3_host::S3Host; #[derive(Error, Debug)] pub enum FileHostingError { @@ -13,6 +17,10 @@ pub enum FileHostingError { HttpError(#[from] reqwest::Error), #[error("Backblaze error: {0}")] BackblazeError(serde_json::Value), + #[error("S3 error: {0}")] + S3Error(#[from] S3Error), + #[error("S3 Authentication error: {0}")] + S3CredentialsError(#[from] AwsCredsError), #[error("File system error in file hosting: {0}")] FileSystemError(#[from] std::io::Error), #[error("Invalid Filename")] diff --git a/src/file_hosting/s3_host.rs b/src/file_hosting/s3_host.rs new file mode 100644 index 00000000..427482c5 --- /dev/null +++ b/src/file_hosting/s3_host.rs @@ -0,0 +1,104 @@ +use crate::file_hosting::{DeleteFileData, FileHost, FileHostingError, UploadFileData}; +use async_trait::async_trait; +use s3::bucket::Bucket; +use s3::creds::Credentials; +use s3::region::Region; + +pub struct S3Host { + bucket: Bucket, +} + +impl S3Host { + pub fn new( + bucket_name: &str, + bucket_region: &str, + url: &str, + access_token: &str, + secret: &str, + ) -> Result { + let mut bucket = Bucket::new( + bucket_name, + Region::Custom { + region: bucket_region.to_string(), + endpoint: url.to_string(), + }, + Credentials::new(Some(access_token), Some(secret), None, None, None)?, + )?; + + bucket.add_header("x-amz-acl", "public-read"); + + Ok(S3Host { bucket }) + } +} + +#[async_trait] +impl FileHost for S3Host { + async fn upload_file( + &self, + content_type: &str, + file_name: &str, + file_bytes: Vec, + ) -> Result { + let content_sha1 = sha1::Sha1::from(&file_bytes).hexdigest(); + + self.bucket + .put_object_with_content_type( + format!("/{}", file_name), + file_bytes.as_slice(), + content_type, + ) + .await?; + + Ok(UploadFileData { + file_id: file_name.to_string(), + file_name: file_name.to_string(), + content_length: file_bytes.len() as u32, + content_sha1, + content_md5: None, + content_type: content_type.to_string(), + upload_timestamp: chrono::Utc::now().timestamp_millis() as u64, + }) + } + + async fn delete_file_version( + &self, + file_id: &str, + file_name: &str, + ) -> Result { + self.bucket.delete_object(format!("/{}", file_name)).await?; + + Ok(DeleteFileData { + file_id: file_id.to_string(), + file_name: file_name.to_string(), + }) + } +} + +#[cfg(test)] +mod tests { + use crate::file_hosting::s3_host::S3Host; + use crate::file_hosting::FileHost; + + #[actix_rt::test] + async fn test_file_management() { + let s3_host = S3Host::new( + &*dotenv::var("S3_BUCKET_NAME").unwrap(), + &*dotenv::var("S3_REGION").unwrap(), + &*dotenv::var("S3_URL").unwrap(), + &*dotenv::var("S3_ACCESS_TOKEN").unwrap(), + &*dotenv::var("S3_SECRET").unwrap(), + ) + .unwrap(); + + s3_host + .upload_file( + "text/plain", + "test.txt", + "test file".to_string().into_bytes(), + ) + .await + .unwrap(); + + s3_host.delete_file_version("", "test.txt").await.unwrap(); + } +} diff --git a/src/main.rs b/src/main.rs index fd56f528..e365fca3 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,4 @@ +use crate::file_hosting::S3Host; use actix_cors::Cors; use actix_web::middleware::Logger; use actix_web::{http, web, App, HttpServer}; @@ -64,12 +65,10 @@ async fn main() -> std::io::Result<()> { .await .expect("Database connection failed"); - let backblaze_enabled = dotenv::var("BACKBLAZE_ENABLED") - .ok() - .and_then(|s| s.parse::().ok()) - .unwrap_or(false); + let storage_backend = dotenv::var("STORAGE_BACKEND").unwrap_or_else(|_| "local".to_string()); - let file_host: Arc = if backblaze_enabled { + let file_host: Arc = if storage_backend == "backblaze" + { Arc::new( file_hosting::BackblazeHost::new( &dotenv::var("BACKBLAZE_KEY_ID").unwrap(), @@ -78,8 +77,21 @@ async fn main() -> std::io::Result<()> { ) .await, ) - } else { + } else if storage_backend == "s3" { + Arc::new( + S3Host::new( + &*dotenv::var("S3_BUCKET_NAME").unwrap(), + &*dotenv::var("S3_REGION").unwrap(), + &*dotenv::var("S3_URL").unwrap(), + &*dotenv::var("S3_ACCESS_TOKEN").unwrap(), + &*dotenv::var("S3_SECRET").unwrap(), + ) + .unwrap(), + ) + } else if storage_backend == "local" { Arc::new(file_hosting::MockHost::new()) + } else { + panic!("Invalid storage backend specified. Aborting startup!") }; let mut scheduler = scheduler::Scheduler::new(); @@ -243,16 +255,24 @@ fn check_env_vars() { check_var::("MEILISEARCH_ADDR"); check_var::("BIND_ADDR"); - if dotenv::var("BACKBLAZE_ENABLED") - .ok() - .and_then(|s| s.parse::().ok()) - .unwrap_or(false) - { + check_var::("STORAGE_BACKEND"); + + let storage_backend = dotenv::var("STORAGE_BACKEND").ok(); + + if storage_backend.as_deref() == Some("backblaze") { check_var::("BACKBLAZE_KEY_ID"); check_var::("BACKBLAZE_KEY"); check_var::("BACKBLAZE_BUCKET_ID"); - } else { + } else if storage_backend.as_deref() == Some("s3") { + check_var::("S3_ACCESS_TOKEN"); + check_var::("S3_SECRET"); + check_var::("S3_URL"); + check_var::("S3_REGION"); + check_var::("S3_BUCKET_NAME"); + } else if storage_backend.as_deref() == Some("local") { check_var::("MOCK_FILE_PATH"); + } else if let Some(backend) = storage_backend { + warn!("Variable `STORAGE_BACKEND` contains an invalid value: {}. Expected \"backblaze\", \"s3\", or \"local\".", backend); } check_var::("INDEX_CURSEFORGE");