From f53b5a5c254e30718b04c2dece1a6f08d427ef52 Mon Sep 17 00:00:00 2001 From: lambda-0x <0xlambda@protonmail.com> Date: Wed, 16 Oct 2024 15:44:31 +0530 Subject: [PATCH] feat(torii): fetch and process erc721 metadata and image commit-id:274aaa3a --- Cargo.lock | 490 +++++++++++++++++- Cargo.toml | 3 + bin/torii/src/main.rs | 34 +- crates/torii/core/Cargo.toml | 4 + crates/torii/core/src/engine.rs | 7 +- crates/torii/core/src/executor.rs | 34 +- crates/torii/core/src/sql/erc.rs | 411 ++++++++++++--- crates/torii/core/src/sql/mod.rs | 4 + ...5532_add_image_url_and_metadata_fields.sql | 2 + crates/torii/server/Cargo.toml | 10 +- crates/torii/server/src/artifacts.rs | 17 + crates/torii/server/src/lib.rs | 1 + crates/torii/server/src/proxy.rs | 37 +- 13 files changed, 962 insertions(+), 92 deletions(-) create mode 100644 crates/torii/migrations/20241014085532_add_image_url_and_metadata_fields.sql create mode 100644 crates/torii/server/src/artifacts.rs diff --git a/Cargo.lock b/Cargo.lock index 14e5553b0a..c9111be50b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -68,6 +68,12 @@ dependencies = [ "gimli", ] +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "adler2" version = "2.0.0" @@ -142,6 +148,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aligned-vec" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4aa90d7ce82d4be67b64039a3d588d38dbcc6736577de4a847025ce5b0c468d1" + [[package]] name = "alloc-no-stdlib" version = "2.0.4" @@ -960,12 +972,29 @@ version = "1.0.89" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "86fdf8605db99b54d3cd748a44c6d04df638eb5dafb219b135d0149bd0db01f6" +[[package]] +name = "arbitrary" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" + [[package]] name = "arc-swap" version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "ark-ec" version = "0.4.2" @@ -1631,6 +1660,29 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "av1-grain" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6678909d8c5d46a42abcf571271e15fdbc0a225e3646cf23762cd415046c78bf" +dependencies = [ + "anyhow", + "arrayvec", + "log", + "nom", + "num-rational", + "v_frame", +] + +[[package]] +name = "avif-serialize" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "876c75a42f6364451a033496a14c44bffe41f5f4a8236f697391f11024e596d2" +dependencies = [ + "arrayvec", +] + [[package]] name = "aws-lc-rs" version = "1.9.0" @@ -1767,7 +1819,7 @@ dependencies = [ "addr2line", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.8.0", "object", "rustc-demangle", "windows-targets 0.52.6", @@ -1935,6 +1987,12 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +[[package]] +name = "bit_field" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc827186963e592360843fb5ba4b973e145841266c1357f7180c43526f2e5b61" + [[package]] name = "bitflags" version = "1.3.2" @@ -1950,6 +2008,12 @@ dependencies = [ "serde", ] +[[package]] +name = "bitstream-io" +version = "2.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b81e1519b0d82120d2fd469d5bfb2919a9361c48b02d82d04befc1cdd2002452" + [[package]] name = "bitvec" version = "1.0.1" @@ -2076,6 +2140,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "borrow-or-share" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eeab4423108c5d7c744f4d234de88d18d636100093ae04caf4825134b9c3a32" + [[package]] name = "borsh" version = "1.5.1" @@ -2141,6 +2211,12 @@ dependencies = [ "serde", ] +[[package]] +name = "built" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "236e6289eda5a812bc6b53c3b024039382a2895fbbeef2d748b2931546d392c4" + [[package]] name = "bumpalo" version = "3.16.0" @@ -2198,6 +2274,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.7.2" @@ -3322,7 +3404,7 @@ checksum = "9a95746c5221a74d7b913a415fdbb9e7c90e1b4d818dbbff59bddc034cfce2ec" dependencies = [ "bytes", "flex-error", - "num-derive", + "num-derive 0.3.3", "num-traits 0.2.19", "prost 0.12.6", "prost-types 0.12.6", @@ -3376,6 +3458,16 @@ dependencies = [ "nom", ] +[[package]] +name = "cfg-expr" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d067ad48b8650848b989a59a86c6c36a995d02d2bf778d45c3c5d57bc2718f02" +dependencies = [ + "smallvec", + "target-lexicon", +] + [[package]] name = "cfg-if" version = "1.0.0" @@ -3568,6 +3660,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15" +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + [[package]] name = "colorchoice" version = "1.0.2" @@ -4235,6 +4333,12 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "data-url" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c297a1c74b71ae29df00c3e22dd9534821d60eb9af5a0192823fa2acea70c2a" + [[package]] name = "debugid" version = "0.8.0" @@ -5056,6 +5160,22 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "exr" +version = "1.72.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "887d93f60543e9a9362ef8a21beedd0a833c5d9610e18c67abe15a5963dcb1a4" +dependencies = [ + "bit_field", + "flume", + "half 2.4.1", + "lebe", + "miniz_oxide 0.7.4", + "rayon-core", + "smallvec", + "zune-inflate", +] + [[package]] name = "eyre" version = "0.6.12" @@ -5107,6 +5227,15 @@ dependencies = [ "bytes", ] +[[package]] +name = "fdeflate" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8090f921a24b04994d9929e204f50b498a33ea6ba559ffaa05e04f7ee7fb5ab" +dependencies = [ + "simd-adler32", +] + [[package]] name = "ff" version = "0.13.0" @@ -5172,7 +5301,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "324a1be68054ef05ad64b861cc9eaf1d623d2d8cb25b4bf2cb9cdd902b4bf253" dependencies = [ "crc32fast", - "miniz_oxide", + "miniz_oxide 0.8.0", ] [[package]] @@ -5185,6 +5314,16 @@ dependencies = [ "paste", ] +[[package]] +name = "fluent-uri" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1918b65d96df47d3591bed19c5cca17e3fa5d0707318e4b5ef2eae01764df7e5" +dependencies = [ + "borrow-or-share", + "ref-cast", +] + [[package]] name = "flume" version = "0.11.0" @@ -5498,6 +5637,16 @@ dependencies = [ "polyval", ] +[[package]] +name = "gif" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb2d69b19215e18bb912fa30f7ce15846e301408695e44e0ef719f1da9e19f2" +dependencies = [ + "color_quant", + "weezl", +] + [[package]] name = "gimli" version = "0.31.0" @@ -7213,6 +7362,39 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "image" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99314c8a2152b8ddb211f924cdae532d8c5e4c8bb54728e12fff1b0cd5963a10" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "exr", + "gif", + "image-webp", + "num-traits 0.2.19", + "png", + "qoi", + "ravif", + "rayon", + "rgb", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f79afb8cbee2ef20f59ccd477a218c12a93943d075b492015ecb1bb81f8ee904" +dependencies = [ + "byteorder-lite", + "quick-error 2.0.1", +] + [[package]] name = "imara-diff" version = "0.1.7" @@ -7223,6 +7405,12 @@ dependencies = [ "hashbrown 0.14.5", ] +[[package]] +name = "imgref" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44feda355f4159a7c757171a77de25daf6411e217b4cabd03bd6650690468126" + [[package]] name = "impl-codec" version = "0.6.0" @@ -7415,6 +7603,17 @@ dependencies = [ "webrtc-util 0.8.1", ] +[[package]] +name = "interpolate_name" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "io-close" version = "0.3.7" @@ -7665,6 +7864,12 @@ dependencies = [ "libc", ] +[[package]] +name = "jpeg-decoder" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f5d4a7da358eff58addd2877a45865158f0d78c911d43a5784ceb7bbf52833b0" + [[package]] name = "js-sys" version = "0.3.70" @@ -8516,6 +8721,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "lebe" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8" + [[package]] name = "leopard-codec" version = "0.1.0" @@ -8533,6 +8744,17 @@ version = "0.2.158" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" +[[package]] +name = "libfuzzer-sys" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a96cfd5557eb82f2b83fed4955246c988d331975a002961b07c81584d107e7f7" +dependencies = [ + "arbitrary", + "cc", + "once_cell", +] + [[package]] name = "libloading" version = "0.8.5" @@ -9141,6 +9363,15 @@ dependencies = [ "value-bag", ] +[[package]] +name = "loop9" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" +dependencies = [ + "imgref", +] + [[package]] name = "lru" version = "0.12.4" @@ -9220,6 +9451,15 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", +] + [[package]] name = "md-5" version = "0.10.6" @@ -9373,6 +9613,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" +[[package]] +name = "miniz_oxide" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08" +dependencies = [ + "adler", +] + [[package]] name = "miniz_oxide" version = "0.8.0" @@ -9380,6 +9629,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -9726,6 +9976,12 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + [[package]] name = "normalize-line-endings" version = "0.3.0" @@ -9817,6 +10073,17 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "num-format" version = "0.4.4" @@ -10502,6 +10769,19 @@ dependencies = [ "plotters-backend", ] +[[package]] +name = "png" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f9d46a34a05a6a57566bc2bfae066ef07585a6e3fa30fbbdff5936380623f0" +dependencies = [ + "bitflags 1.3.2", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide 0.8.0", +] + [[package]] name = "polling" version = "2.8.0" @@ -10826,6 +11106,25 @@ dependencies = [ "human_format", ] +[[package]] +name = "profiling" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afbdc74edc00b6f6a218ca6a5364d6226a259d4b8ea1af4a0ea063f27e179f4d" +dependencies = [ + "profiling-procmacros", +] + +[[package]] +name = "profiling-procmacros" +version = "1.0.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a65f2e60fbf1063868558d69c6beacf412dc755f9fc020f514b7955fc914fe30" +dependencies = [ + "quote", + "syn 2.0.77", +] + [[package]] name = "prometheus-client" version = "0.22.3" @@ -11034,6 +11333,15 @@ dependencies = [ "psl-types", ] +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + [[package]] name = "quanta" version = "0.12.3" @@ -11055,6 +11363,12 @@ version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quick-protobuf" version = "0.8.1" @@ -11236,6 +11550,55 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "rav1e" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd87ce80a7665b1cce111f8a16c1f3929f6547ce91ade6addf4ec86a8dda5ce9" +dependencies = [ + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av1-grain", + "bitstream-io", + "built", + "cfg-if", + "interpolate_name", + "itertools 0.12.1", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "new_debug_unreachable", + "noop_proc_macro", + "num-derive 0.4.2", + "num-traits 0.2.19", + "once_cell", + "paste", + "profiling", + "rand 0.8.5", + "rand_chacha", + "simd_helpers", + "system-deps", + "thiserror", + "v_frame", + "wasm-bindgen", +] + +[[package]] +name = "ravif" +version = "0.11.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f0bfd976333248de2078d350bfdf182ff96e168a24d23d2436cef320dd4bdd" +dependencies = [ + "avif-serialize", + "imgref", + "loop9", + "quick-error 2.0.1", + "rav1e", + "rgb", +] + [[package]] name = "raw-cpuid" version = "11.1.0" @@ -11337,6 +11700,26 @@ dependencies = [ "thiserror", ] +[[package]] +name = "ref-cast" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ccf0a6f84d5f1d581da8b41b47ec8600871962f2a528115b542b362d4b744931" +dependencies = [ + "ref-cast-impl", +] + +[[package]] +name = "ref-cast-impl" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcc303e793d3734489387d205e9b186fac9c6cfacedd98cbb2e8a5943595f3e6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.77", +] + [[package]] name = "regex" version = "1.10.6" @@ -11523,7 +11906,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" dependencies = [ "hostname", - "quick-error", + "quick-error 1.2.3", ] [[package]] @@ -12122,7 +12505,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" dependencies = [ "fnv", - "quick-error", + "quick-error 1.2.3", "tempfile", "wait-timeout", ] @@ -12985,6 +13368,21 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote", +] + [[package]] name = "simdutf8" version = "0.1.5" @@ -14237,6 +14635,19 @@ dependencies = [ "libc", ] +[[package]] +name = "system-deps" +version = "6.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3e535eb8dded36d55ec13eddacd30dec501792ff23a0b1682c38601b8cf2349" +dependencies = [ + "cfg-expr", + "heck 0.5.0", + "pkg-config", + "toml 0.8.19", + "version-compare", +] + [[package]] name = "tap" version = "1.0.1" @@ -14254,6 +14665,12 @@ dependencies = [ "xattr", ] +[[package]] +name = "target-lexicon" +version = "0.12.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1" + [[package]] name = "tempdir" version = "0.3.7" @@ -14375,6 +14792,17 @@ dependencies = [ "num_cpus", ] +[[package]] +name = "tiff" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba1310fcea54c6a9a4fd1aad794ecc02c31682f6bfbecdf460bf19533eed1e3e" +dependencies = [ + "flate2", + "jpeg-decoder", + "weezl", +] + [[package]] name = "time" version = "0.3.36" @@ -14858,13 +15286,16 @@ dependencies = [ "camino", "chrono", "crypto-bigint", + "data-url", "dojo-test-utils", "dojo-types", "dojo-utils", "dojo-world", + "fluent-uri", "futures-channel", "futures-util", "hashlink 0.9.1", + "image", "katana-runner", "num-traits 0.2.19", "once_cell", @@ -15007,6 +15438,7 @@ name = "torii-server" version = "1.0.0-alpha.16" dependencies = [ "base64 0.21.7", + "camino", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.30", @@ -15020,6 +15452,7 @@ dependencies = [ "tower 0.4.13", "tower-http 0.4.4", "tracing", + "warp", ] [[package]] @@ -15582,6 +16015,17 @@ dependencies = [ "getrandom", ] +[[package]] +name = "v_frame" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f32aaa24bacd11e488aa9ba66369c7cd514885742c9fe08cfe85884db3e92b" +dependencies = [ + "aligned-vec", + "num-traits 0.2.19", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.0" @@ -15613,6 +16057,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "version-compare" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852e951cb7832cb45cb1169900d19760cfa39b82bc0ea9c0e5a14ae88411c98b" + [[package]] name = "version_check" version = "0.9.5" @@ -16143,6 +16593,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "weezl" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53a85b86a771b1c87058196170769dd264f66c0782acf1ae6cc51bfd64b39082" + [[package]] name = "which" version = "4.4.2" @@ -16835,3 +17291,27 @@ dependencies = [ "cc", "pkg-config", ] + +[[package]] +name = "zune-core" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "zune-jpeg" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16099418600b4d8f028622f73ff6e3deaabdff330fb9a2a131dea781ee8b0768" +dependencies = [ + "zune-core", +] diff --git a/Cargo.toml b/Cargo.toml index 9757979d72..667b0b91c1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -157,13 +157,16 @@ clap_complete = "4.3" console = "0.15.7" convert_case = "0.6.0" crypto-bigint = { version = "0.5.3", features = [ "serde" ] } +data-url = "0.3" derive_more = "0.99.17" flate2 = "1.0.24" +fluent-uri = "0.3" futures = "0.3.30" futures-util = "0.3.30" hashlink = "0.9.1" hex = "0.4.3" http = "0.2.9" +image = "0.25.2" indexmap = "2.2.5" indoc = "1.0.7" itertools = "0.12.1" diff --git a/bin/torii/src/main.rs b/bin/torii/src/main.rs index 9e776f3bfc..c20da751e5 100644 --- a/bin/torii/src/main.rs +++ b/bin/torii/src/main.rs @@ -19,6 +19,7 @@ use std::sync::Arc; use std::time::Duration; use anyhow::Context; +use camino::Utf8PathBuf; use clap::{ArgAction, Parser}; use dojo_metrics::{metrics_process, prometheus_exporter}; use dojo_utils::parse::{parse_socket_address, parse_url}; @@ -146,6 +147,10 @@ struct Args { /// Configuration file #[arg(long)] config: Option, + + /// Path to a directory to store ERC artifacts + #[arg(long)] + artifacts_path: Utf8PathBuf, } #[tokio::main] @@ -218,7 +223,8 @@ async fn main() -> anyhow::Result<()> { executor.run().await.unwrap(); }); - let db = Sql::new(pool.clone(), sender.clone(), &contracts).await?; + let db = + Sql::new(pool.clone(), sender.clone(), &contracts, args.artifacts_path.clone()).await?; let processors = Processors { transaction: vec![Box::new(StoreTransactionProcessor)], @@ -254,10 +260,19 @@ async fn main() -> anyhow::Result<()> { Arc::new(contracts), ); - let shutdown_rx = shutdown_tx.subscribe(); - let (grpc_addr, grpc_server) = - torii_grpc::server::new(shutdown_rx, &pool, block_rx, world_address, Arc::clone(&provider)) - .await?; + let (grpc_addr, grpc_server) = torii_grpc::server::new( + shutdown_tx.subscribe(), + &pool, + block_rx, + world_address, + Arc::clone(&provider), + ) + .await?; + + tokio::fs::create_dir_all(args.artifacts_path.clone()).await?; + let absolute_path = args.artifacts_path.canonicalize()?; + let (artifacts_addr, artifacts_server) = + torii_server::artifacts::new(shutdown_tx.subscribe(), absolute_path).await?; let mut libp2p_relay_server = torii_relay::server::Relay::new( db, @@ -270,7 +285,13 @@ async fn main() -> anyhow::Result<()> { ) .expect("Failed to start libp2p relay server"); - let proxy_server = Arc::new(Proxy::new(args.addr, args.allowed_origins, Some(grpc_addr), None)); + let proxy_server = Arc::new(Proxy::new( + args.addr, + args.allowed_origins, + Some(grpc_addr), + None, + Some(artifacts_addr), + )); let graphql_server = spawn_rebuilding_graphql_server( shutdown_tx.clone(), @@ -314,6 +335,7 @@ async fn main() -> anyhow::Result<()> { _ = graphql_server => {}, _ = grpc_server => {}, _ = libp2p_relay_server.run() => {}, + _ = artifacts_server => {}, _ = dojo_utils::signal::wait_signals() => {}, }; diff --git a/crates/torii/core/Cargo.toml b/crates/torii/core/Cargo.toml index 30040d528b..22cebd2642 100644 --- a/crates/torii/core/Cargo.toml +++ b/crates/torii/core/Cargo.toml @@ -14,13 +14,17 @@ async-trait.workspace = true base64.workspace = true bitflags = "2.6.0" cainome.workspace = true +camino.workspace = true chrono.workspace = true crypto-bigint.workspace = true +data-url.workspace = true dojo-types.workspace = true dojo-world = { workspace = true, features = [ "contracts", "manifest" ] } +fluent-uri.workspace = true futures-channel = "0.3.0" futures-util.workspace = true hashlink.workspace = true +image.workspace = true num-traits.workspace = true once_cell.workspace = true reqwest.workspace = true diff --git a/crates/torii/core/src/engine.rs b/crates/torii/core/src/engine.rs index ed51840ae6..9352b48cb1 100644 --- a/crates/torii/core/src/engine.rs +++ b/crates/torii/core/src/engine.rs @@ -264,11 +264,7 @@ impl Engine

{ }, Err(e) => { error!(target: LOG_TARGET, error = %e, "Processing fetched data."); - erroring_out = true; - sleep(backoff_delay).await; - if backoff_delay < max_backoff_delay { - backoff_delay *= 2; - } + return Err(e); } } debug!(target: LOG_TARGET, duration = ?instant.elapsed(), "Processed fetched data."); @@ -853,6 +849,7 @@ impl Engine

{ .await { error!(target: LOG_TARGET, event_name = processor.event_key(), error = ?e, "Processing event."); + return Err(e); } } else { warn!(target: LOG_TARGET, event_name = processor.event_key(), "Event not validated."); diff --git a/crates/torii/core/src/executor.rs b/crates/torii/core/src/executor.rs index 04d64676b9..7b3e1af35c 100644 --- a/crates/torii/core/src/executor.rs +++ b/crates/torii/core/src/executor.rs @@ -12,7 +12,8 @@ use tokio::sync::broadcast::{Receiver, Sender}; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; use tokio::sync::oneshot; use tokio::time::Instant; -use tracing::{debug, error}; +use tracing::{debug, error}; /* Added import for data URI parsing // Uncommented to reuse + * HTTP client */ use crate::simple_broker::SimpleBroker; use crate::sql::utils::{felt_to_sql_string, sql_string_to_u256, u256_to_sql_string, I256}; @@ -81,6 +82,16 @@ pub struct UpdateCursorsQuery { pub pending_block_timestamp: u64, } +#[derive(Debug, Clone)] +pub struct RegisterErc721TokenQuery { + pub token_id: String, + pub contract_address: Felt, + pub name: String, + pub symbol: String, + pub metadata: String, + pub image_path: String, +} + #[derive(Debug, Clone)] pub enum QueryType { SetHead(SetHeadQuery), @@ -90,6 +101,7 @@ pub enum QueryType { DeleteEntity(DeleteEntityQuery), EventMessage(Ty), ApplyBalanceDiff(ApplyBalanceDiffQuery), + RegisterErc721Token(RegisterErc721TokenQuery), RegisterModel, StoreEvent, Execute, @@ -471,6 +483,26 @@ impl<'c> Executor<'c> { self.apply_balance_diff(apply_balance_diff).await?; debug!(target: LOG_TARGET, duration = ?instant.elapsed(), "Applied balance diff."); } + QueryType::RegisterErc721Token(register_erc721_token) => { + let query = sqlx::query( + "INSERT INTO tokens (id, contract_address, name, symbol, decimals, metadata, \ + image_url) VALUES (?, ?, ?, ?, ?, ?, ?)", + ) + .bind(®ister_erc721_token.token_id) + .bind(felt_to_sql_string(®ister_erc721_token.contract_address)) + .bind(®ister_erc721_token.name) + .bind(®ister_erc721_token.symbol) + .bind(0) + .bind(®ister_erc721_token.metadata) + .bind(®ister_erc721_token.image_path); + + query.execute(&mut **tx).await.with_context(|| { + format!( + "Failed to execute RegisterErc721Token query: {:?}", + register_erc721_token + ) + })?; + } QueryType::Execute => { debug!(target: LOG_TARGET, "Executing query."); let instant = Instant::now(); diff --git a/crates/torii/core/src/sql/erc.rs b/crates/torii/core/src/sql/erc.rs index f82eacb1a2..21df976b66 100644 --- a/crates/torii/core/src/sql/erc.rs +++ b/crates/torii/core/src/sql/erc.rs @@ -1,16 +1,28 @@ use std::collections::HashMap; +use std::io::Cursor; use std::mem; +use std::str::FromStr; use anyhow::{Context, Result}; use cainome::cairo_serde::{ByteArray, CairoSerde}; +use camino::Utf8PathBuf; +use data_url::mime::Mime; +use data_url::DataUrl; +use fluent_uri::Uri; +use image::{DynamicImage, ImageFormat}; +use reqwest::Client; use starknet::core::types::{BlockId, BlockTag, Felt, FunctionCall, U256}; use starknet::core::utils::{get_selector_from_name, parse_cairo_short_string}; use starknet::providers::Provider; +use tokio::fs; +use tokio::io::AsyncWriteExt; use tracing::debug; use super::utils::{u256_to_sql_string, I256}; use super::{Sql, FELT_DELIMITER}; -use crate::executor::{ApplyBalanceDiffQuery, Argument, QueryMessage, QueryType}; +use crate::executor::{ + ApplyBalanceDiffQuery, Argument, QueryMessage, QueryType, RegisterErc721TokenQuery, +}; use crate::sql::utils::{felt_and_u256_to_sql_string, felt_to_sql_string, felts_to_sql_string}; use crate::types::ContractType; use crate::utils::utc_dt_string_from_timestamp; @@ -84,11 +96,18 @@ impl Sql { event_id: &str, ) -> Result<()> { // contract_address:id + let actual_token_id = token_id; let token_id = felt_and_u256_to_sql_string(&contract_address, &token_id); let token_exists: bool = self.local_cache.contains_token_id(&token_id); if !token_exists { - self.register_erc721_token_metadata(contract_address, &token_id, provider).await?; + self.register_erc721_token_metadata( + contract_address, + &token_id, + actual_token_id, + provider, + ) + .await?; self.execute().await?; } @@ -216,10 +235,11 @@ impl Sql { &mut self, contract_address: Felt, token_id: &str, + actual_token_id: U256, provider: &P, ) -> Result<()> { - let res = sqlx::query_as::<_, (String, String, u8)>( - "SELECT name, symbol, decimals FROM tokens WHERE contract_address = ?", + let res = sqlx::query_as::<_, (String, String)>( + "SELECT name, symbol FROM tokens WHERE contract_address = ?", ) .bind(felt_to_sql_string(&contract_address)) .fetch_one(&self.pool) @@ -227,83 +247,110 @@ impl Sql { // If we find a token already registered for this contract_address we dont need to refetch // the data since its same for all ERC721 tokens - if let Ok((name, symbol, decimals)) = res { - debug!( - contract_address = %felt_to_sql_string(&contract_address), - "Token already registered for contract_address, so reusing fetched data", - ); - self.executor.send(QueryMessage::other( - "INSERT INTO tokens (id, contract_address, name, symbol, decimals) VALUES (?, ?, \ - ?, ?, ?)" - .to_string(), - vec![ - Argument::String(token_id.to_string()), - Argument::FieldElement(contract_address), - Argument::String(name), - Argument::String(symbol), - Argument::Int(decimals.into()), - ], - ))?; - self.local_cache.register_token_id(token_id.to_string()); - return Ok(()); - } + let (name, symbol) = match res { + Ok((name, symbol)) => { + debug!( + contract_address = %felt_to_sql_string(&contract_address), + "Token already registered for contract_address, so reusing fetched data", + ); + (name, symbol) + } + Err(_) => { + // Fetch token information from the chain + let name = provider + .call( + FunctionCall { + contract_address, + entry_point_selector: get_selector_from_name("name").unwrap(), + calldata: vec![], + }, + BlockId::Tag(BlockTag::Pending), + ) + .await?; + + // len = 1 => return value felt (i.e. legacy erc721 token) + // len > 1 => return value ByteArray (i.e. new erc721 token) + let name = if name.len() == 1 { + parse_cairo_short_string(&name[0]).unwrap() + } else { + ByteArray::cairo_deserialize(&name, 0) + .expect("Return value not ByteArray") + .to_string() + .expect("Return value not String") + }; + + let symbol = provider + .call( + FunctionCall { + contract_address, + entry_point_selector: get_selector_from_name("symbol").unwrap(), + calldata: vec![], + }, + BlockId::Tag(BlockTag::Pending), + ) + .await?; + let symbol = if symbol.len() == 1 { + parse_cairo_short_string(&symbol[0]).unwrap() + } else { + ByteArray::cairo_deserialize(&symbol, 0) + .expect("Return value not ByteArray") + .to_string() + .expect("Return value not String") + }; + + (name, symbol) + } + }; - // Fetch token information from the chain - let name = provider + let token_uri = provider .call( FunctionCall { contract_address, - entry_point_selector: get_selector_from_name("name").unwrap(), - calldata: vec![], + entry_point_selector: get_selector_from_name("token_uri").unwrap(), + calldata: vec![actual_token_id.low().into(), actual_token_id.high().into()], }, BlockId::Tag(BlockTag::Pending), ) .await?; - // len = 1 => return value felt (i.e. legacy erc721 token) - // len > 1 => return value ByteArray (i.e. new erc721 token) - let name = if name.len() == 1 { - parse_cairo_short_string(&name[0]).unwrap() + let token_uri = if let Ok(byte_array) = ByteArray::cairo_deserialize(&token_uri, 0) { + byte_array.to_string().expect("Return value not String") + } else if let Ok(felt_array) = Vec::::cairo_deserialize(&token_uri, 0) { + felt_array + .iter() + .map(parse_cairo_short_string) + .collect::, _>>() + .map(|strings| strings.join("")) + .map_err(|_| anyhow::anyhow!("Failed parsing Array to String"))? } else { - ByteArray::cairo_deserialize(&name, 0) - .expect("Return value not ByteArray") - .to_string() - .expect("Return value not String") + return Err(anyhow::anyhow!("token_uri is neither ByteArray nor Array").into()); }; - let symbol = provider - .call( - FunctionCall { - contract_address, - entry_point_selector: get_selector_from_name("symbol").unwrap(), - calldata: vec![], - }, - BlockId::Tag(BlockTag::Pending), - ) - .await?; - let symbol = if symbol.len() == 1 { - parse_cairo_short_string(&symbol[0]).unwrap() - } else { - ByteArray::cairo_deserialize(&symbol, 0) - .expect("Return value not ByteArray") - .to_string() - .expect("Return value not String") - }; - - let decimals = 0; - - // Insert the token into the tokens table - self.executor.send(QueryMessage::other( - "INSERT INTO tokens (id, contract_address, name, symbol, decimals) VALUES (?, ?, ?, \ - ?, ?)" - .to_string(), - vec![ - Argument::String(token_id.to_string()), - Argument::FieldElement(contract_address), - Argument::String(name), - Argument::String(symbol), - Argument::Int(decimals.into()), - ], + let metadata = Self::fetch_metadata(&token_uri).await?; + let image_url = metadata + .get("image") + .with_context(|| "Image URL not found in metadata")? + .as_str() + .with_context(|| "Image field not a string")? + .to_string(); + + let image_path = + Self::fetch_and_process_image(&image_url, &self.artifacts_path, token_id).await?; + + // serialized metadata as json string + let metadata = serde_json::to_string(&metadata).context("Failed to serialize metadata")?; + + self.executor.send(QueryMessage::new( + "".to_string(), + vec![], + QueryType::RegisterErc721Token(RegisterErc721TokenQuery { + token_id: token_id.to_string(), + contract_address, + name, + symbol, + metadata, + image_path, + }), ))?; self.local_cache.register_token_id(token_id.to_string()); @@ -357,4 +404,230 @@ impl Sql { } Ok(()) } + + // given a uri which can be either http/https url or data uri, fetch the metadata erc721 + // metadata json schema + async fn fetch_metadata(token_uri: &str) -> Result { + // Parse the token_uri + let uri = Uri::parse(token_uri).context("Invalid token URI")?; + + match uri.scheme().as_str() { + "http" | "https" => { + // Fetch metadata from HTTP/HTTPS URL + let response = + reqwest::get(token_uri).await.context("Failed to fetch metadata from URL")?; + + let json: serde_json::Value = + response.json().await.context("Failed to parse metadata JSON")?; + + Ok(json) + } + "ipfs" => { + let cid = token_uri.strip_prefix("ipfs://").unwrap(); + let gateway_url = format!("https://ipfs.io/ipfs/{}", cid); + let client = Client::new(); + let response = client + .get(&gateway_url) + .send() + .await + .context("Failed to fetch metadata from IPFS")?; + + let json: serde_json::Value = + response.json().await.context("Failed to parse metadata JSON from IPFS")?; + + Ok(json) + } + "data" => { + // Parse and decode data URI + let data_url = DataUrl::process(token_uri).context("Failed to parse data URI")?; + + // Ensure the MIME type is JSON + if data_url.mime_type() != &Mime::from_str("application/json").unwrap() { + return Err(anyhow::anyhow!("Data URI is not of JSON type")); + } + + let decoded = data_url.decode_to_vec().context("Failed to decode data URI")?; + + let json: serde_json::Value = serde_json::from_slice(&decoded.0) + .context("Failed to parse metadata JSON from data URI")?; + + Ok(json) + } + _ => Err(anyhow::anyhow!("Unsupported URI scheme found in token URI: {}", uri)), + } + } + + async fn fetch_and_process_image( + image_uri: &str, + artifacts_path: &Utf8PathBuf, + token_id: &str, + ) -> Result { + // Determine the URI scheme + let uri = Uri::parse(image_uri).context("Invalid image URI")?; + let image_type = match uri.scheme().as_str() { + "http" | "https" => { + // Fetch image from HTTP/HTTPS URL + let client = Client::new(); + let response = client + .get(image_uri) + .send() + .await + .context("Failed to fetch image from URL")? + .bytes() + .await + .context("Failed to read image bytes from response")?; + + // svg files typically start with { + let cid = image_uri.strip_prefix("ipfs://").unwrap(); + let gateway_url = format!("https://ipfs.io/ipfs/{}", cid); + let client = Client::new(); + let response = client + .get(&gateway_url) + .send() + .await + .context("Failed to fetch image from IPFS")? + .bytes() + .await + .context("Failed to read image bytes from IPFS response")?; + + if response.starts_with(b" { + // Parse and decode data URI + let data_url = DataUrl::process(image_uri).context("Failed to parse data URI")?; + + // Check if it's an SVG + if data_url.mime_type() == &Mime::from_str("image/svg+xml").unwrap() { + let decoded = data_url.decode_to_vec().context("Failed to decode data URI")?; + ErcImageType::Svg(decoded.0) + } else { + let decoded = data_url.decode_to_vec().context("Failed to decode data URI")?; + let format = + image::guess_format(&decoded.0).with_context(|| "Unknown file format")?; + ErcImageType::DynamicImage(( + image::load_from_memory(&decoded.0) + .context("Failed to load image from bytes")?, + format, + )) + } + } + _ => { + return Err(anyhow::anyhow!("Unsupported URI scheme: {}", uri.scheme())); + } + }; + + // Extract contract_address and token_id from token_id + let parts: Vec<&str> = token_id.split(':').collect(); + if parts.len() != 2 { + return Err(anyhow::anyhow!("token_id must be in format contract_address:token_id")); + } + let contract_address = parts[0]; + let token_id_part = parts[1]; + + // Define directory path + let dir_path = artifacts_path.join(contract_address).join(token_id_part); + + // Create directories if they don't exist + fs::create_dir_all(&dir_path) + .await + .context("Failed to create directories for image storage")?; + + // Define base image name + let base_image_name = "image"; + + let relative_path = Utf8PathBuf::new().join(contract_address).join(token_id_part); + + match image_type { + ErcImageType::DynamicImage((img, format)) => { + let format_ext = format.extensions_str()[0]; + + let scales = [1.0, 0.5, 0.25]; // 1x, 0.5x, 0.25x + + for &scale in &scales { + let resized_image = Self::resize_image(&img, scale)?; + let file_name = if scale == 1.0 { + format!("{}.{}", base_image_name, format_ext) + } else { + format!( + "{}@{}x.{}", + base_image_name, + if scale == 0.5 { "0_5" } else { "0_25" }, + format_ext + ) + }; + let file_path = dir_path.join(&file_name); + + // Save the resized image + let mut file = fs::File::create(&file_path) + .await + .with_context(|| format!("Failed to create file: {:?}", file_path))?; + let encoded_image = Self::encode_image_to_vec(&resized_image, format) + .context("Failed to encode image")?; + file.write_all(&encoded_image).await.with_context(|| { + format!("Failed to write image to file: {:?}", file_path) + })?; + } + + Ok(format!("{}/{}.{}", relative_path, base_image_name, format_ext)) + } + ErcImageType::Svg(svg_data) => { + let file_name = format!("{}.svg", base_image_name); + let file_path = dir_path.join(&file_name); + + // Save the SVG file + let mut file = fs::File::create(&file_path) + .await + .with_context(|| format!("Failed to create file: {:?}", file_path))?; + file.write_all(&svg_data) + .await + .with_context(|| format!("Failed to write SVG to file: {:?}", file_path))?; + + Ok(format!("{}/{}", relative_path, file_name)) + } + } + } + + // Helper function to resize image based on scale + fn resize_image(image: &DynamicImage, scale: f32) -> Result { + let width = (image.width() as f32 * scale) as u32; + let height = (image.height() as f32 * scale) as u32; + Ok(image.resize(width, height, image::imageops::FilterType::Lanczos3)) + } + + // Helper function to encode image to bytes + fn encode_image_to_vec(image: &DynamicImage, format: ImageFormat) -> Result> { + let mut buf = Vec::new(); + image + .write_to(&mut Cursor::new(&mut buf), format) + .context("Failed to write image to buffer")?; + Ok(buf) + } +} + +pub enum ErcImageType { + DynamicImage((DynamicImage, ImageFormat)), + Svg(Vec), } diff --git a/crates/torii/core/src/sql/mod.rs b/crates/torii/core/src/sql/mod.rs index ad00c34ca6..131269da80 100644 --- a/crates/torii/core/src/sql/mod.rs +++ b/crates/torii/core/src/sql/mod.rs @@ -4,6 +4,7 @@ use std::str::FromStr; use std::sync::Arc; use anyhow::{anyhow, Context, Result}; +use camino::Utf8PathBuf; use dojo_types::primitive::Primitive; use dojo_types::schema::{EnumOption, Member, Struct, Ty}; use dojo_world::contracts::abi::model::Layout; @@ -46,6 +47,7 @@ pub struct Sql { model_cache: Arc, // when SQL struct is cloned a empty local_cache is created local_cache: LocalCache, + artifacts_path: Utf8PathBuf, } #[derive(Debug, Clone)] @@ -60,6 +62,7 @@ impl Sql { pool: Pool, executor: UnboundedSender, contracts: &HashMap, + artifacts_path: Utf8PathBuf, ) -> Result { for contract in contracts { executor.send(QueryMessage::other( @@ -80,6 +83,7 @@ impl Sql { executor, model_cache: Arc::new(ModelCache::new(pool.clone())), local_cache, + artifacts_path, }; db.execute().await?; diff --git a/crates/torii/migrations/20241014085532_add_image_url_and_metadata_fields.sql b/crates/torii/migrations/20241014085532_add_image_url_and_metadata_fields.sql new file mode 100644 index 0000000000..33f6369476 --- /dev/null +++ b/crates/torii/migrations/20241014085532_add_image_url_and_metadata_fields.sql @@ -0,0 +1,2 @@ +ALTER TABLE tokens ADD COLUMN image_url TEXT; +ALTER TABLE tokens ADD COLUMN metadata TEXT; \ No newline at end of file diff --git a/crates/torii/server/Cargo.toml b/crates/torii/server/Cargo.toml index 1ca82c911c..ae7363e849 100644 --- a/crates/torii/server/Cargo.toml +++ b/crates/torii/server/Cargo.toml @@ -7,16 +7,18 @@ version.workspace = true [dependencies] base64.workspace = true -http.workspace = true +camino.workspace = true http-body = "0.4.5" -hyper.workspace = true +http.workspace = true hyper-reverse-proxy = { git = "https://github.com/tarrencev/hyper-reverse-proxy" } +hyper.workspace = true indexmap.workspace = true lazy_static.workspace = true serde.workspace = true serde_json.workspace = true -tokio.workspace = true tokio-util = "0.7.7" -tower.workspace = true +tokio.workspace = true tower-http.workspace = true +tower.workspace = true tracing.workspace = true +warp.workspace = true diff --git a/crates/torii/server/src/artifacts.rs b/crates/torii/server/src/artifacts.rs new file mode 100644 index 0000000000..ec683ba46b --- /dev/null +++ b/crates/torii/server/src/artifacts.rs @@ -0,0 +1,17 @@ +use std::future::Future; +use std::net::SocketAddr; +use std::path::PathBuf; + +use tokio::sync::broadcast::Receiver; +use warp::Filter; + +pub async fn new( + mut shutdown_rx: Receiver<()>, + static_dir: PathBuf, +) -> Result<(SocketAddr, impl Future + 'static), std::io::Error> { + let routes = warp::path("static").and(warp::fs::dir(static_dir)); + + Ok(warp::serve(routes).bind_with_graceful_shutdown(([127, 0, 0, 1], 0), async move { + shutdown_rx.recv().await.ok(); + })) +} diff --git a/crates/torii/server/src/lib.rs b/crates/torii/server/src/lib.rs index 44dcc92d61..621f66d155 100644 --- a/crates/torii/server/src/lib.rs +++ b/crates/torii/server/src/lib.rs @@ -1 +1,2 @@ +pub mod artifacts; pub mod proxy; diff --git a/crates/torii/server/src/proxy.rs b/crates/torii/server/src/proxy.rs index 30ee956f79..a7aa298fb8 100644 --- a/crates/torii/server/src/proxy.rs +++ b/crates/torii/server/src/proxy.rs @@ -56,6 +56,7 @@ pub struct Proxy { addr: SocketAddr, allowed_origins: Option>, grpc_addr: Option, + artifacts_addr: Option, graphql_addr: Arc>>, } @@ -65,8 +66,15 @@ impl Proxy { allowed_origins: Option>, grpc_addr: Option, graphql_addr: Option, + artifacts_addr: Option, ) -> Self { - Self { addr, allowed_origins, grpc_addr, graphql_addr: Arc::new(RwLock::new(graphql_addr)) } + Self { + addr, + allowed_origins, + grpc_addr, + graphql_addr: Arc::new(RwLock::new(graphql_addr)), + artifacts_addr, + } } pub async fn set_graphql_addr(&self, addr: SocketAddr) { @@ -82,6 +90,7 @@ impl Proxy { let allowed_origins = self.allowed_origins.clone(); let grpc_addr = self.grpc_addr; let graphql_addr = self.graphql_addr.clone(); + let artifacts_addr = self.artifacts_addr.clone(); let make_svc = make_service_fn(move |conn: &AddrStream| { let remote_addr = conn.remote_addr().ip(); @@ -123,7 +132,7 @@ impl Proxy { let graphql_addr = graphql_addr_clone.clone(); async move { let graphql_addr = graphql_addr.read().await; - handle(remote_addr, grpc_addr, *graphql_addr, req).await + handle(remote_addr, grpc_addr, artifacts_addr, *graphql_addr, req).await } }); @@ -143,9 +152,33 @@ impl Proxy { async fn handle( client_ip: IpAddr, grpc_addr: Option, + artifacts_addr: Option, graphql_addr: Option, req: Request, ) -> Result, Infallible> { + if req.uri().path().starts_with("/static") { + if let Some(artifacts_addr) = artifacts_addr { + let artifacts_addr = format!("http://{}{}", artifacts_addr, req.uri().path()); + + dbg!(&artifacts_addr); + return match GRAPHQL_PROXY_CLIENT.call(client_ip, &artifacts_addr, req).await { + Ok(response) => Ok(response), + Err(_error) => { + error!("{:?}", _error); + Ok(Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(Body::empty()) + .unwrap()) + } + }; + } else { + return Ok(Response::builder() + .status(StatusCode::NOT_FOUND) + .body(Body::empty()) + .unwrap()); + } + } + if req.uri().path().starts_with("/graphql") { if let Some(graphql_addr) = graphql_addr { let graphql_addr = format!("http://{}", graphql_addr);