From 4decead016c05df0549e1ac5500866c065e9f8a2 Mon Sep 17 00:00:00 2001 From: Domas Kalinauskas Date: Wed, 1 Nov 2023 14:01:25 +0200 Subject: [PATCH 1/4] support wasm build --- Cargo.lock | 115 +++++++++++++++++++------------------- arkworks/Cargo.toml | 30 ++++++---- arkworks/src/eip_4844.rs | 5 +- arkworks/src/kzg_types.rs | 8 ++- 4 files changed, 90 insertions(+), 68 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5acc2c5d0..085f18615 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,13 +4,14 @@ version = 3 [[package]] name = "ahash" -version = "0.8.3" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c99f64d1e06488f620f932677e24bc6e2897582980441ae90a671415bd7ec2f" +checksum = "91429305e9f0a25f6205c5b8e0d2db09e0708a7a6df0f42212bb56c32c8ac97a" dependencies = [ "cfg-if", "once_cell", "version_check", + "zerocopy", ] [[package]] @@ -151,7 +152,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94893f1e0c6eeab764ade8dc4c0db24caf4fe7cbbaafc0eba0a9030f447b5185" dependencies = [ - "colored", "num-traits", "rand", "rayon", @@ -180,9 +180,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "bitflags" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" [[package]] name = "bitvec" @@ -221,7 +221,7 @@ dependencies = [ [[package]] name = "blst" version = "0.3.11" -source = "git+https://github.com/supranational/blst.git#1514337fde866d3e4ce5c1447a6e1f17bcfd7d65" +source = "git+https://github.com/supranational/blst.git#badb7f9ab4b8f14ee491b30ce33ab3867154ec89" dependencies = [ "cc", "glob", @@ -297,18 +297,18 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.6" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d04704f56c2cde07f43e8e2c154b43f216dc5c92fc98ada720177362f953b956" +checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.4.6" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e231faeaca65ebd1ea3c737966bf858971cd38c3849107aa3ea7de90a804e45" +checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663" dependencies = [ "anstyle", "clap_lex", @@ -316,26 +316,15 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" - -[[package]] -name = "colored" -version = "2.0.4" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2674ec482fbc38012cf31e6c42ba0177b431a0cb6f15fe40efa5aab1bda516f6" -dependencies = [ - "is-terminal", - "lazy_static", - "windows-sys", -] +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "cpufeatures" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a17b76ff3a4162b0b27f354a0c87015ddad39d35f9c0c36607a3bdd175dde1f1" +checksum = "ce420fe07aecd3e67c5f910618fe65e94158f6dcc0adf44e00d69ce2bdfe0fd0" dependencies = [ "libc", ] @@ -558,9 +547,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dfda62a12f55daeae5015f81b0baea145391cb4520f86c248fc615d72640d12" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" [[package]] name = "hermit-abi" @@ -596,12 +585,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.0.2" +version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" dependencies = [ "equivalent", - "hashbrown 0.14.1", + "hashbrown 0.14.2", ] [[package]] @@ -664,12 +653,6 @@ dependencies = [ "serde_yaml", ] -[[package]] -name = "lazy_static" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" - [[package]] name = "libc" version = "0.2.149" @@ -937,9 +920,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.0" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", @@ -949,9 +932,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.1" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465c6fc0621e4abc4187a2bda0937bfd4f722c2730b29562e19689ea796c9a4b" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", @@ -960,9 +943,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3cbb081b9784b07cceb8824c8583f86db4814d172ab043f3c23f7dc600bf83d" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "rust-kzg-arkworks" @@ -1064,9 +1047,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.18" +version = "0.38.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a74ee2d7c2581cd139b42447d7d9389b889bdaad3a73f1ebb16f2a3237bb19c" +checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" dependencies = [ "bitflags", "errno", @@ -1104,18 +1087,18 @@ checksum = "836fa6a3e1e547f9a2c4040802ec865b5d85f4014efe00555d7090a3dcaa1090" [[package]] name = "serde" -version = "1.0.188" +version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" +checksum = "91d3c334ca1ee894a2c6f6ad698fe8c435b76d504b13d436f0685d648d6d96f7" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.188" +version = "1.0.190" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" +checksum = "67c5609f394e5c2bd7fc51efda478004ea80ef42fee983d5c67a65e34f32c0e3" dependencies = [ "proc-macro2", "quote", @@ -1124,9 +1107,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.107" +version = "1.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b420ce6e3d8bd882e9b243c6eed35dbc9a6110c9769e74b584e0d68d1f20c65" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" dependencies = [ "itoa", "ryu", @@ -1135,9 +1118,9 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.25" +version = "0.9.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a49e178e4452f45cb61d0cd8cebc1b0fafd3e41929e996cef79aa3aca91f574" +checksum = "3cc7a1570e38322cfe4154732e5110f887ea57e22b76f4bfd32b5bdd3368666c" dependencies = [ "indexmap", "itoa", @@ -1224,9 +1207,9 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.3" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cda73e2f1397b1262d6dfdcef8aafae14d1de7748d66822d3bfeeb6d03e5e4b" +checksum = "3550f4e9685620ac18a50ed434eb3aec30db8ba93b0287467bca5826ea25baf1" [[package]] name = "toml_edit" @@ -1454,9 +1437,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.5.16" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037711d82167854aff2018dfd193aa0fef5370f456732f0d5a0c59b0f1b4b907" +checksum = "176b6138793677221d420fd2f0aeeced263f197688b36484660da767bca2fa32" dependencies = [ "memchr", ] @@ -1470,6 +1453,26 @@ dependencies = [ "tap", ] +[[package]] +name = "zerocopy" +version = "0.7.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "686b7e407015242119c33dab17b8f61ba6843534de936d94368856528eae4dcc" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "020f3dfe25dfc38dfea49ce62d5d45ecdd7f0d8a724fa63eb36b6eba4ec76806" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.38", +] + [[package]] name = "zeroize" version = "1.6.0" diff --git a/arkworks/Cargo.toml b/arkworks/Cargo.toml index 0f84f290f..fc8bc441a 100644 --- a/arkworks/Cargo.toml +++ b/arkworks/Cargo.toml @@ -4,32 +4,42 @@ version = "0.1.0" edition = "2021" [dependencies] -kzg = { path = "../kzg" } -kzg-bench = { path = "../kzg-bench" } blst = {'git' = 'https://github.com/supranational/blst.git'} +kzg = { path = "../kzg", default-features = false } ark-std = { version = "^0.4.0", default-features = false } ark-ff = { version = "^0.4.2", default-features = false } ark-ec = { version = "^0.4.2", default-features = false } ark-poly = { version = "^0.4.2", default-features = false } ark-bls12-381 = { version = "^0.4.0", default-features = false, features = [ "curve" ] } ark-serialize = { version = "^0.4.2", default-features = false } -rand = "0.8" -hex = "0.4.2" -# Needed for ckzg extern C interface +hex = "0.4.3" +rand = { version = "0.8.5", optional = true } libc = { version = "0.2.148", default-features = false } -rayon = { version = "1.6.0", optional = true } +rayon = { version = "1.8.0", optional = true } [dev-dependencies] criterion = "0.5.1" +kzg-bench = { path = "../kzg-bench" } +rand = { version = "0.8.5" } [features] -default = ["std"] -std = ["ark-ff/std", "ark-ec/std", "ark-poly/std", "ark-std/std"] -print-trace = ["ark-std/print-trace"] +default = [ + "std", + "rand" +] +std = [ + "ark-ff/std", "ark-ec/std", "ark-poly/std", "ark-std/std", + "kzg/std", + "libc/std" +] parallel = [ -"std", "dep:rayon", "kzg/parallel", +"dep:rayon", "kzg/parallel", "ark-std/parallel", "ark-ff/parallel", "ark-ec/parallel", "ark-poly/parallel" ] +rand = [ + "dep:rand", + "kzg/rand", +] [[bench]] name = "fft" diff --git a/arkworks/src/eip_4844.rs b/arkworks/src/eip_4844.rs index 7d6630fb5..e0fcd6564 100644 --- a/arkworks/src/eip_4844.rs +++ b/arkworks/src/eip_4844.rs @@ -6,7 +6,7 @@ use blst::{blst_fr, blst_p1, blst_p2}; use kzg::common_utils::reverse_bit_order; use kzg::eip_4844::{ blob_to_kzg_commitment_rust, compute_blob_kzg_proof_rust, compute_kzg_proof_rust, - load_trusted_setup_rust, load_trusted_setup_string, verify_blob_kzg_proof_batch_rust, + load_trusted_setup_rust, verify_blob_kzg_proof_batch_rust, verify_blob_kzg_proof_rust, verify_kzg_proof_rust, Blob, Bytes32, Bytes48, CKZGSettings, KZGCommitment, KZGProof, BYTES_PER_FIELD_ELEMENT, BYTES_PER_G1, BYTES_PER_G2, C_KZG_RET, C_KZG_RET_BADARGS, C_KZG_RET_OK, FIELD_ELEMENTS_PER_BLOB, TRUSTED_SETUP_NUM_G1_POINTS, @@ -25,6 +25,9 @@ use std::io::Read; #[cfg(feature = "parallel")] use rayon::prelude::*; +#[cfg(feature = "std")] +use kzg::eip_4844::load_trusted_setup_string; + #[cfg(feature = "std")] pub fn load_trusted_setup_filename_rust(filepath: &str) -> Result { let mut file = File::open(filepath).map_err(|_| "Unable to open file".to_string())?; diff --git a/arkworks/src/kzg_types.rs b/arkworks/src/kzg_types.rs index 89812cd14..b5fc29db1 100644 --- a/arkworks/src/kzg_types.rs +++ b/arkworks/src/kzg_types.rs @@ -17,7 +17,11 @@ use ark_bls12_381::{g1, g2, Fr, G1Affine, G2Affine}; use ark_ec::{models::short_weierstrass::Projective, AffineRepr, Group}; use ark_ff::{biginteger::BigInteger256, BigInteger, Field}; use ark_serialize::{CanonicalDeserialize, CanonicalSerialize}; -use ark_std::{One, UniformRand, Zero}; +use ark_std::{One, Zero}; + +#[cfg(feature = "rand")] +use ark_std::UniformRand; + use blst::{blst_fr, blst_p1}; use kzg::common_utils::reverse_bit_order; use kzg::eip_4844::{BYTES_PER_FIELD_ELEMENT, BYTES_PER_G1, BYTES_PER_G2}; @@ -78,6 +82,7 @@ impl KzgFr for ArkFr { Self::from_u64(1) } + #[cfg(feature = "rand")] fn rand() -> Self { let mut rng = rand::thread_rng(); Self { @@ -265,6 +270,7 @@ impl G1 for ArkG1 { G1_NEGATIVE_GENERATOR } + #[cfg(feature = "rand")] fn rand() -> Self { let mut rng = rand::thread_rng(); Self { From 1276be74504d8ff6cbec5e8dd96c3116c15a3bee Mon Sep 17 00:00:00 2001 From: Saulius Grigaitis Date: Tue, 10 Oct 2023 10:52:04 +0300 Subject: [PATCH 2/4] Do not build releases with 'force adx' --- .github/workflows/release.yml | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e9cf30a5a..e072a4312 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -24,25 +24,17 @@ jobs: run: | # Linux mkdir -p staging/linux/non-parallel - mkdir -p staging/linux/non-parallel-force-adx mkdir -p staging/linux/parallel - mkdir -p staging/linux/parallel-force-adx # Windows mkdir -p staging/windows/non-parallel - mkdir -p staging/windows/non-parallel-force-adx mkdir -p staging/windows/parallel - mkdir -p staging/windows/parallel-force-adx - name: "[blst] Build Linux" run: | - cargo rustc --manifest-path blst/Cargo.toml --release --target x86_64-unknown-linux-gnu --crate-type=staticlib --features=blst-portable + cargo rustc --manifest-path blst/Cargo.toml --release --target x86_64-unknown-linux-gnu --crate-type=staticlib mv target/x86_64-unknown-linux-gnu/release/librust_kzg_blst.a staging/linux/non-parallel/rust_kzg_blst.a - cargo rustc --manifest-path blst/Cargo.toml --release --target x86_64-unknown-linux-gnu --crate-type=staticlib --features=blst-force-adx - mv target/x86_64-unknown-linux-gnu/release/librust_kzg_blst.a staging/linux/non-parallel-force-adx/rust_kzg_blst.a - cargo rustc --manifest-path blst/Cargo.toml --release --target x86_64-unknown-linux-gnu --crate-type=staticlib --features=parallel,blst-portable + cargo rustc --manifest-path blst/Cargo.toml --release --target x86_64-unknown-linux-gnu --crate-type=staticlib --features=parallel mv target/x86_64-unknown-linux-gnu/release/librust_kzg_blst.a staging/linux/parallel/rust_kzg_blst.a - cargo rustc --manifest-path blst/Cargo.toml --release --target x86_64-unknown-linux-gnu --crate-type=staticlib --features=parallel,blst-force-adx - mv target/x86_64-unknown-linux-gnu/release/librust_kzg_blst.a staging/linux/parallel-force-adx/rust_kzg_blst.a - name: "[blst] Compress Linux artifacts" run: | @@ -53,14 +45,10 @@ jobs: - name: "[blst] Build Windows" run: | - cargo rustc --manifest-path blst/Cargo.toml --release --target x86_64-pc-windows-gnu --crate-type=staticlib --features=blst-portable + cargo rustc --manifest-path blst/Cargo.toml --release --target x86_64-pc-windows-gnu --crate-type=staticlib mv target/x86_64-pc-windows-gnu/release/librust_kzg_blst.a staging/windows/non-parallel/rust_kzg_blst.a - cargo rustc --manifest-path blst/Cargo.toml --release --target x86_64-pc-windows-gnu --crate-type=staticlib --features=blst-force-adx - mv target/x86_64-pc-windows-gnu/release/librust_kzg_blst.a staging/windows/non-parallel-force-adx/rust_kzg_blst.a - cargo rustc --manifest-path blst/Cargo.toml --release --target x86_64-pc-windows-gnu --crate-type=staticlib --features=parallel,blst-portable + cargo rustc --manifest-path blst/Cargo.toml --release --target x86_64-pc-windows-gnu --crate-type=staticlib --features=parallel mv target/x86_64-pc-windows-gnu/release/librust_kzg_blst.a staging/windows/parallel/rust_kzg_blst.a - cargo rustc --manifest-path blst/Cargo.toml --release --target x86_64-pc-windows-gnu --crate-type=staticlib --features=parallel,blst-force-adx - mv target/x86_64-pc-windows-gnu/release/librust_kzg_blst.a staging/windows/parallel-force-adx/rust_kzg_blst.a - name: "[blst] Compress Windows artifacts" run: | From 1a3e6f37dda9a0e613796b5b0319175d32d20210 Mon Sep 17 00:00:00 2001 From: Domas Kalinauskas Date: Wed, 1 Nov 2023 16:14:27 +0200 Subject: [PATCH 3/4] refactor release for easier backend addition --- .github/workflows/release.yml | 96 +++++++++++++++++++---------------- arkworks/src/eip_4844.rs | 8 +-- 2 files changed, 57 insertions(+), 47 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e072a4312..fb844fd81 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,62 +7,72 @@ on: workflow_dispatch: env: - C_KZG_4844_GIT_HASH: '5703f6f3536b7692616bc289ac3f3867ab8db9d8' + C_KZG_4844_GIT_HASH: '624aa60d01da524827123506975431aa5454c80d' jobs: - release: + release-build: runs-on: ubuntu-latest - + strategy: + fail-fast: false + matrix: + backend: [blst, arkworks] + target: [windows, linux] + include: + # Set target-name for target builds + - target: linux + target-name: x86_64-unknown-linux-gnu + - target: windows + target-name: x86_64-pc-windows-gnu + env: + matrix-name: "[${{ matrix.backend }}, ${{ matrix.target }}]" + staging-dir: staging/${{ matrix.backend }}/${{ matrix.target }} + archive-name: rust-kzg-${{ matrix.backend }}-${{ matrix.target }}.zip steps: - uses: actions/checkout@v2 - - name: Install packages + - if: matrix.target == 'windows' + name: Install MinGW run: | sudo apt install -y mingw-w64 - - name: "[blst] Prepare artifact folders" - run: | - # Linux - mkdir -p staging/linux/non-parallel - mkdir -p staging/linux/parallel - # Windows - mkdir -p staging/windows/non-parallel - mkdir -p staging/windows/parallel - - - name: "[blst] Build Linux" - run: | - cargo rustc --manifest-path blst/Cargo.toml --release --target x86_64-unknown-linux-gnu --crate-type=staticlib - mv target/x86_64-unknown-linux-gnu/release/librust_kzg_blst.a staging/linux/non-parallel/rust_kzg_blst.a - cargo rustc --manifest-path blst/Cargo.toml --release --target x86_64-unknown-linux-gnu --crate-type=staticlib --features=parallel - mv target/x86_64-unknown-linux-gnu/release/librust_kzg_blst.a staging/linux/parallel/rust_kzg_blst.a - - - name: "[blst] Compress Linux artifacts" + - name: "${{ env.matrix-name }} Prepare artifact folders" run: | - cp LICENSE staging/linux/ - cp blst/*.patch staging/linux/ - cd staging/linux/ - zip -rz rust-kzg-blst-linux.zip * <<< "Compatible with c-kzg-4844 git hash: $C_KZG_4844_GIT_HASH." + mkdir -p ${{ env.staging-dir }}/parallel + mkdir -p ${{ env.staging-dir }}/non-parallel - - name: "[blst] Build Windows" + - name: "${{ env.matrix-name }} Build" run: | - cargo rustc --manifest-path blst/Cargo.toml --release --target x86_64-pc-windows-gnu --crate-type=staticlib - mv target/x86_64-pc-windows-gnu/release/librust_kzg_blst.a staging/windows/non-parallel/rust_kzg_blst.a - cargo rustc --manifest-path blst/Cargo.toml --release --target x86_64-pc-windows-gnu --crate-type=staticlib --features=parallel - mv target/x86_64-pc-windows-gnu/release/librust_kzg_blst.a staging/windows/parallel/rust_kzg_blst.a + # Non-parallel build + cargo rustc --manifest-path ${{ matrix.backend }}/Cargo.toml --release --target ${{ matrix.target-name }} --crate-type=staticlib + mv target/${{ matrix.target-name }}/release/librust_kzg_${{ matrix.backend}}.a ${{ env.staging-dir }}/non-parallel/rust_kzg_${{ matrix.backend}}.a + # Parallel build + cargo rustc --manifest-path ${{ matrix.backend }}/Cargo.toml --release --target ${{ matrix.target-name }} --crate-type=staticlib --features=parallel + mv target/${{ matrix.target-name }}/release/librust_kzg_${{ matrix.backend}}.a ${{ env.staging-dir }}/parallel/rust_kzg_${{ matrix.backend}}.a - - name: "[blst] Compress Windows artifacts" + - name: "${{ env.matrix-name }} Compress artifacts" run: | - cp LICENSE staging/windows/ - cp blst/*.patch staging/windows/ - cd staging/windows/ - zip -rz rust-kzg-blst-windows.zip * <<< "Compatible with c-kzg-4844 git hash: $C_KZG_4844_GIT_HASH." + cp LICENSE ${{ env.staging-dir }}/ + cp ${{ matrix.backend }}/*.patch ${{ env.staging-dir }}/ + cd ${{ env.staging-dir }} + zip -rz ${{ env.archive-name }} * <<< "Compatible with c-kzg-4844 git hash: $C_KZG_4844_GIT_HASH." - - uses: "marvinpinto/action-automatic-releases@latest" + - name: "${{ env.matrix-name }} Upload" + uses: actions/upload-artifact@v3 with: - repo_token: "${{ secrets.GITHUB_TOKEN }}" - automatic_release_tag: "latest" - prerelease: false - title: "Automatic Release" - files: | - staging/linux/rust-kzg-blst-linux.zip - staging/windows/rust-kzg-blst-windows.zip + name: ${{ env.archive-name }} + path: ${{ env.staging-dir }}/${{ env.archive-name }} + + release-publish: + runs-on: ubuntu-latest + needs: release-build + steps: + - name: Download build artifacts + uses: actions/download-artifact@v3 + - uses: "marvinpinto/action-automatic-releases@latest" + with: + repo_token: "${{ secrets.GITHUB_TOKEN }}" + automatic_release_tag: "latest" + prerelease: false + title: "Automatic Release" + files: | + ./rust-kzg-*.zip/rust-kzg-*.zip diff --git a/arkworks/src/eip_4844.rs b/arkworks/src/eip_4844.rs index e0fcd6564..3b8a5b385 100644 --- a/arkworks/src/eip_4844.rs +++ b/arkworks/src/eip_4844.rs @@ -6,10 +6,10 @@ use blst::{blst_fr, blst_p1, blst_p2}; use kzg::common_utils::reverse_bit_order; use kzg::eip_4844::{ blob_to_kzg_commitment_rust, compute_blob_kzg_proof_rust, compute_kzg_proof_rust, - load_trusted_setup_rust, verify_blob_kzg_proof_batch_rust, - verify_blob_kzg_proof_rust, verify_kzg_proof_rust, Blob, Bytes32, Bytes48, CKZGSettings, - KZGCommitment, KZGProof, BYTES_PER_FIELD_ELEMENT, BYTES_PER_G1, BYTES_PER_G2, C_KZG_RET, - C_KZG_RET_BADARGS, C_KZG_RET_OK, FIELD_ELEMENTS_PER_BLOB, TRUSTED_SETUP_NUM_G1_POINTS, + load_trusted_setup_rust, verify_blob_kzg_proof_batch_rust, verify_blob_kzg_proof_rust, + verify_kzg_proof_rust, Blob, Bytes32, Bytes48, CKZGSettings, KZGCommitment, KZGProof, + BYTES_PER_FIELD_ELEMENT, BYTES_PER_G1, BYTES_PER_G2, C_KZG_RET, C_KZG_RET_BADARGS, + C_KZG_RET_OK, FIELD_ELEMENTS_PER_BLOB, TRUSTED_SETUP_NUM_G1_POINTS, TRUSTED_SETUP_NUM_G2_POINTS, }; use kzg::{cfg_into_iter, Fr, G1}; From e8c6366fd902e58f2e085d3060902f7a9bb41b41 Mon Sep 17 00:00:00 2001 From: Domas Kalinauskas Date: Wed, 1 Nov 2023 19:05:19 +0200 Subject: [PATCH 4/4] consolidate backend tests & benchmark workflows --- .github/workflows/arkworks-benchmarks.yml | 47 ------ .github/workflows/arkworks-tests.yml | 74 --------- .github/workflows/backend-benchmarks.yml | 56 +++++++ .github/workflows/backend-tests.yml | 140 ++++++++++++++++ .github/workflows/blst-benchmarks.yml | 47 ------ .github/workflows/blst-tests.yml | 90 ----------- .github/workflows/mcl-benchmarks.yml | 50 ------ .github/workflows/mcl-tests.yml | 90 ----------- .github/workflows/zkcrypto-benchmarks.yml | 25 --- .github/workflows/zkcrypto-tests.yml | 48 ------ arkworks/run-c-kzg-4844-tests.sh | 151 ------------------ blst/run-c-kzg-4844-benches.sh | 83 ---------- mcl/kzg/run-c-kzg-4844-benches.sh | 90 ----------- mcl/kzg/run-c-kzg-4844-tests.sh | 141 ---------------- ...44-benches.sh => run-c-kzg-4844-benches.sh | 22 ++- ...g-4844-tests.sh => run-c-kzg-4844-tests.sh | 22 ++- 16 files changed, 225 insertions(+), 951 deletions(-) delete mode 100644 .github/workflows/arkworks-benchmarks.yml delete mode 100644 .github/workflows/arkworks-tests.yml create mode 100644 .github/workflows/backend-benchmarks.yml create mode 100644 .github/workflows/backend-tests.yml delete mode 100644 .github/workflows/blst-benchmarks.yml delete mode 100644 .github/workflows/blst-tests.yml delete mode 100644 .github/workflows/mcl-benchmarks.yml delete mode 100644 .github/workflows/mcl-tests.yml delete mode 100644 .github/workflows/zkcrypto-benchmarks.yml delete mode 100644 .github/workflows/zkcrypto-tests.yml delete mode 100644 arkworks/run-c-kzg-4844-tests.sh delete mode 100644 blst/run-c-kzg-4844-benches.sh delete mode 100644 mcl/kzg/run-c-kzg-4844-benches.sh delete mode 100755 mcl/kzg/run-c-kzg-4844-tests.sh rename arkworks/run-c-kzg-4844-benches.sh => run-c-kzg-4844-benches.sh (78%) mode change 100644 => 100755 rename blst/run-c-kzg-4844-tests.sh => run-c-kzg-4844-tests.sh (87%) mode change 100644 => 100755 diff --git a/.github/workflows/arkworks-benchmarks.yml b/.github/workflows/arkworks-benchmarks.yml deleted file mode 100644 index 8df82afdf..000000000 --- a/.github/workflows/arkworks-benchmarks.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: arkworks benchmarks - -on: [push, pull_request, workflow_dispatch] - -env: - C_KZG_4844_GIT_HASH: '624aa60d01da524827123506975431aa5454c80d' - -jobs: - benchmarks: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macos-11] - - steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-java@v3 - with: - distribution: "temurin" - java-version: "11" - - - uses: actions/setup-go@v2 - with: - go-version: ^1.19 - - - name: "[arkworks] Benchmark" - uses: actions-rs/cargo@v1 - with: - command: bench - args: --manifest-path arkworks/Cargo.toml - - - name: "[arkworks] Benchmark (parallel)" - uses: actions-rs/cargo@v1 - with: - command: bench - args: --manifest-path arkworks/Cargo.toml --features parallel - - - name: "[arkworks] Benchmark (c-kzg-4844)" - if: matrix.os == 'ubuntu-latest' - run: | - cd arkworks && bash run-c-kzg-4844-benches.sh - - - name: "[arkworks] Benchmark (c-kzg-4844 parallel)" - if: matrix.os == 'ubuntu-latest' - run: | - cd arkworks && bash run-c-kzg-4844-benches.sh --parallel diff --git a/.github/workflows/arkworks-tests.yml b/.github/workflows/arkworks-tests.yml deleted file mode 100644 index bfb7fd11d..000000000 --- a/.github/workflows/arkworks-tests.yml +++ /dev/null @@ -1,74 +0,0 @@ -name: arkworks tests - -on: [push, pull_request, workflow_dispatch] - -env: - C_KZG_4844_GIT_HASH: '624aa60d01da524827123506975431aa5454c80d' - -jobs: - tests: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macos-11] - - steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-java@v3 - with: - distribution: "temurin" - java-version: "11" - - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - pip install py_ecc - pip install PyYAML - - - uses: actions/setup-go@v2 - with: - go-version: ^1.19 - - - name: "[arkworks] Tests" - uses: actions-rs/cargo@v1 - with: - command: test - args: --manifest-path arkworks/Cargo.toml --no-fail-fast - - - name: "[arkworks] Tests (parallel)" - uses: actions-rs/cargo@v1 - with: - command: test - args: --manifest-path arkworks/Cargo.toml --no-fail-fast --features parallel - - - name: "[arkworks] Tests (c-kzg-4844)" - if: matrix.os == 'ubuntu-latest' - run: | - cd arkworks && bash run-c-kzg-4844-tests.sh - - - name: "[arkworks] Tests (c-kzg-4844 parallel)" - if: matrix.os == 'ubuntu-latest' - run: | - cd arkworks && bash run-c-kzg-4844-tests.sh --parallel - - - name: "[arkworks] Clippy" - uses: actions-rs/cargo@v1 - with: - command: clippy - args: --manifest-path arkworks/Cargo.toml --all-targets --all-features -- -D warnings - - - name: "[arkworks] Formatting" - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --manifest-path arkworks/Cargo.toml -- --check - - - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --manifest-path kzg-bench/Cargo.toml -- --check diff --git a/.github/workflows/backend-benchmarks.yml b/.github/workflows/backend-benchmarks.yml new file mode 100644 index 000000000..2822ca63a --- /dev/null +++ b/.github/workflows/backend-benchmarks.yml @@ -0,0 +1,56 @@ +name: benchmarks +on: [push, pull_request, workflow_dispatch] +env: + C_KZG_4844_GIT_HASH: '624aa60d01da524827123506975431aa5454c80d' + +jobs: + benchmarks: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-11] + # Excluding mcl for now due to they have different project layout + backend: [blst, arkworks, zkcrypto] + include: + # Setup exec_once_per_backend flag + - os: ubuntu-latest + exec_once_per_backend: true + # Select backends which support wasm & which support ckzg drop-in + - backend: blst + support_ckzg: true + - backend: arkworks + support_ckzg: true + + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v3 + with: + distribution: "temurin" + java-version: "11" + + - uses: actions/setup-go@v2 + with: + go-version: ^1.19 + + - name: "${{ matrix.backend }} Benchmark" + uses: actions-rs/cargo@v1 + with: + command: bench + args: --manifest-path ${{ matrix.backend }}/Cargo.toml + + - name: "${{ matrix.backend }} Benchmark (parallel)" + uses: actions-rs/cargo@v1 + with: + command: bench + args: --manifest-path ${{ matrix.backend }}/Cargo.toml --features parallel + + - name: "${{ matrix.backend }} Benchmark (c-kzg-4844)" + if: matrix.exec_once_per_backend && matrix.support_ckzg + run: | + bash run-c-kzg-4844-benches.sh ${{ matrix.backend }} + + - name: "${{ matrix.backend }} Benchmark (c-kzg-4844 parallel)" + if: matrix.exec_once_per_backend && matrix.support_ckzg + run: | + bash run-c-kzg-4844-benches.sh --parallel ${{ matrix.backend }} diff --git a/.github/workflows/backend-tests.yml b/.github/workflows/backend-tests.yml new file mode 100644 index 000000000..85be45ba1 --- /dev/null +++ b/.github/workflows/backend-tests.yml @@ -0,0 +1,140 @@ +name: tests +on: [push, pull_request, workflow_dispatch] +env: + C_KZG_4844_GIT_HASH: '624aa60d01da524827123506975431aa5454c80d' + +jobs: + tests: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest, macos-11] + # Excluding mcl for now due to they have different project layout + backend: [blst, arkworks, zkcrypto] + include: + # Set default clippy flag to all-features + - clippy-flag: --all-features + # Setup exec_once_per_backend flag + - os: ubuntu-latest + exec_once_per_backend: true + # Setup exec_once_overall flag + - os: ubuntu-latest + backend: blst + exec_once_overall: true + # Select backends which support wasm & which support ckzg drop-in + - backend: blst + support_wasm: true + support_ckzg: true + # Override all-features flag for blst, due to incompatibility between portable & force-adx + clippy-flag: --features=default,std,rand,parallel + - backend: arkworks + support_wasm: true + support_ckzg: true + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-java@v3 + with: + distribution: "temurin" + java-version: "11" + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Install python dependencies + run: | + python -m pip install --upgrade pip + pip install py_ecc + pip install PyYAML + - uses: actions/setup-go@v2 + with: + go-version: ^1.19 + + # Check kzg clippy + - name: "kzg clippy" + if: matrix.exec_once_overall + uses: actions-rs/cargo@v1 + with: + command: clippy + args: --manifest-path kzg/Cargo.toml --all-targets --all-features -- -D warnings + + # Check kzg formatting + - name: "kzg format" + uses: actions-rs/cargo@v1 + if: matrix.exec_once_overall + with: + command: fmt + args: --manifest-path kzg/Cargo.toml -- --check + + # Check kzg-bench clippy + - name: "kzg-bench clippy" + if: matrix.exec_once_overall + uses: actions-rs/cargo@v1 + with: + command: clippy + args: --manifest-path kzg-bench/Cargo.toml --all-targets --all-features -- -D warnings + + # Check kzg-bench formatting + - name: "kzg-bench format" + uses: actions-rs/cargo@v1 + if: matrix.exec_once_overall + with: + command: fmt + args: --manifest-path kzg-bench/Cargo.toml -- --check + + # Check backend clippy + - name: "${{ matrix.backend }} clippy" + if: matrix.exec_once_per_backend + uses: actions-rs/cargo@v1 + with: + command: clippy + args: --manifest-path ${{ matrix.backend }}/Cargo.toml --all-targets ${{ matrix.clippy-flag }} -- -D warnings + + # Check backend formatting + - name: "${{ matrix.backend }} format" + if: matrix.exec_once_per_backend + uses: actions-rs/cargo@v1 + with: + command: fmt + args: --manifest-path ${{ matrix.backend }}/Cargo.toml -- --check + + # Check wasm32 target clippy for backend + - name: "[wasm32] ${{ matrix.backend }} clippy" + if: matrix.exec_once_per_backend && matrix.support_wasm + uses: actions-rs/cargo@v1 + with: + command: clippy + args: --manifest-path ${{ matrix.backend }}/Cargo.toml --target wasm32-unknown-unknown --no-default-features + + # Check wasm32 target backend build + - name: "[wasm32] ${{ matrix.backend }} build" + if: matrix.exec_once_per_backend && matrix.support_wasm + uses: actions-rs/cargo@v1 + with: + command: build + args: --manifest-path ${{ matrix.backend }}/Cargo.toml --target wasm32-unknown-unknown --no-default-features + + # Check non-parallel backend tests + - name: "${{ matrix.backend }} Tests" + uses: actions-rs/cargo@v1 + with: + command: test + args: --manifest-path ${{ matrix.backend }}/Cargo.toml --no-fail-fast --release + + # Check parallel backend tests + - name: "${{ matrix.backend }} Tests (parallel)" + uses: actions-rs/cargo@v1 + with: + command: test + args: --manifest-path ${{ matrix.backend }}/Cargo.toml --no-fail-fast --release --features parallel + + # Check ckzg backend tests + - name: "${{ matrix.backend }} Tests (c-kzg-4844)" + if: matrix.exec_once_per_backend && matrix.support_ckzg + run: | + bash run-c-kzg-4844-tests.sh ${{ matrix.backend }} + + # Check parallel ckzg backend tests + - name: "${{ matrix.backend }} Tests (c-kzg-4844 parallel)" + if: matrix.exec_once_per_backend && matrix.support_ckzg + run: | + bash run-c-kzg-4844-tests.sh --parallel ${{ matrix.backend }} diff --git a/.github/workflows/blst-benchmarks.yml b/.github/workflows/blst-benchmarks.yml deleted file mode 100644 index 07bbedfc6..000000000 --- a/.github/workflows/blst-benchmarks.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: blst benchmarks - -on: [push, pull_request, workflow_dispatch] - -env: - C_KZG_4844_GIT_HASH: '624aa60d01da524827123506975431aa5454c80d' - -jobs: - benchmarks: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macos-11] - - steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-java@v3 - with: - distribution: "temurin" - java-version: "11" - - - uses: actions/setup-go@v2 - with: - go-version: ^1.19 - - - name: "[blst] Benchmark" - uses: actions-rs/cargo@v1 - with: - command: bench - args: --manifest-path blst/Cargo.toml - - - name: "[blst] Benchmark (parallel)" - uses: actions-rs/cargo@v1 - with: - command: bench - args: --manifest-path blst/Cargo.toml --features parallel - - - name: "[blst] Benchmark (c-kzg-4844)" - if: matrix.os == 'ubuntu-latest' - run: | - cd blst && bash run-c-kzg-4844-benches.sh - - - name: "[blst] Benchmark (c-kzg-4844 parallel)" - if: matrix.os == 'ubuntu-latest' - run: | - cd blst && bash run-c-kzg-4844-benches.sh --parallel diff --git a/.github/workflows/blst-tests.yml b/.github/workflows/blst-tests.yml deleted file mode 100644 index a51360a06..000000000 --- a/.github/workflows/blst-tests.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: blst tests - -on: [push, pull_request, workflow_dispatch] - -env: - C_KZG_4844_GIT_HASH: '624aa60d01da524827123506975431aa5454c80d' - -jobs: - tests: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macos-11] - - steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-java@v3 - with: - distribution: "temurin" - java-version: "11" - - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - pip install py_ecc - pip install PyYAML - - - uses: actions/setup-go@v2 - with: - go-version: ^1.19 - - - name: "[blst][wasm32] Clippy" - # Apple's Clang doesn't support WASM, and we are lazy to install it - if: runner.os != 'macOS' - uses: actions-rs/cargo@v1 - with: - command: clippy - args: --manifest-path blst/Cargo.toml --target wasm32-unknown-unknown --no-default-features - - - name: "[blst][wasm32] Build" - # Apple's Clang doesn't support WASM, and we are lazy to install it - if: runner.os != 'macOS' - uses: actions-rs/cargo@v1 - with: - command: build - args: --manifest-path blst/Cargo.toml --target wasm32-unknown-unknown --no-default-features - - - name: "[blst] Tests" - uses: actions-rs/cargo@v1 - with: - command: test - args: --manifest-path blst/Cargo.toml --no-fail-fast - - - name: "[blst] Tests (parallel)" - uses: actions-rs/cargo@v1 - with: - command: test - args: --manifest-path blst/Cargo.toml --no-fail-fast --features parallel - - - name: "[blst] Tests (c-kzg-4844)" - if: matrix.os == 'ubuntu-latest' - run: | - cd blst && bash run-c-kzg-4844-tests.sh - - - name: "[blst] Tests (c-kzg-4844 parallel)" - if: matrix.os == 'ubuntu-latest' - run: | - cd blst && bash run-c-kzg-4844-tests.sh --parallel - - - name: "[blst] Clippy" - uses: actions-rs/cargo@v1 - with: - command: clippy - args: --manifest-path blst/Cargo.toml --all-targets --features=default,std,rand,parallel -- -D warnings - - - name: "[blst] Formatting" - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --manifest-path blst/Cargo.toml -- --check - - - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --manifest-path kzg-bench/Cargo.toml -- --check diff --git a/.github/workflows/mcl-benchmarks.yml b/.github/workflows/mcl-benchmarks.yml deleted file mode 100644 index acebd4d3f..000000000 --- a/.github/workflows/mcl-benchmarks.yml +++ /dev/null @@ -1,50 +0,0 @@ -name: mcl benchmarks - -on: [push, pull_request, workflow_dispatch] - -env: - C_KZG_4844_GIT_HASH: '5703f6f3536b7692616bc289ac3f3867ab8db9d8' - -jobs: - benchmarks: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - - steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-java@v3 - with: - distribution: "temurin" - java-version: "11" - - - uses: actions/setup-go@v2 - with: - go-version: ^1.19 - - - name: "[mcl] Build native libs for Linux" - if: matrix.os == 'ubuntu-latest' - run: | - cd mcl/kzg && bash build.sh - - - name: "[mcl] Benchmark" - if: matrix.os == 'ubuntu-latest' - run: | - cargo bench --manifest-path mcl/kzg-bench/Cargo.toml - - - name: "[mcl] Benchmark (parallel)" - if: matrix.os == 'ubuntu-latest' - run: | - cargo bench --manifest-path mcl/kzg-bench/Cargo.toml --features rust-kzg-mcl/parallel - - #- name: "[mcl] Benchmark (c-kzg-4844)" - # if: matrix.os == 'ubuntu-latest' - # run: | - # cd mcl/kzg && bash run-c-kzg-4844-benches.sh - - #- name: "[mcl] Benchmark (c-kzg-4844 parallel)" - # if: matrix.os == 'ubuntu-latest' - # run: | - # cd mcl/kzg && bash run-c-kzg-4844-benches.sh --parallel diff --git a/.github/workflows/mcl-tests.yml b/.github/workflows/mcl-tests.yml deleted file mode 100644 index 831b90ed1..000000000 --- a/.github/workflows/mcl-tests.yml +++ /dev/null @@ -1,90 +0,0 @@ -name: mcl tests - -on: [push, pull_request, workflow_dispatch] - -env: - C_KZG_4844_GIT_HASH: '5703f6f3536b7692616bc289ac3f3867ab8db9d8' - -jobs: - tests: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - - steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-java@v3 - with: - distribution: "temurin" - java-version: "11" - - - uses: actions/setup-python@v4 - with: - python-version: '3.10' - - - name: Install python dependencies - run: | - python -m pip install --upgrade pip - pip install py_ecc - pip install PyYAML - - - uses: actions/setup-go@v2 - with: - go-version: ^1.19 - - - name: "[mcl] Build native libs for Linux" - if: matrix.os == 'ubuntu-latest' - run: | - cd mcl/kzg && bash build.sh - - - name: "[mcl] Tests (minimal preset)" - if: matrix.os == 'ubuntu-latest' - uses: actions-rs/cargo@v1 - with: - command: test - args: --package rust-kzg-mcl-bench --lib shared_tests::eip_4844::tests --no-fail-fast --features minimal-spec -- --test-threads 1 - - - name: "[mcl] Tests (mainnet preset)" - if: matrix.os == 'ubuntu-latest' - uses: actions-rs/cargo@v1 - with: - command: test - args: shared_tests --manifest-path mcl/kzg-bench/Cargo.toml --no-fail-fast -- --test-threads 1 - - - name: "[mcl] Tests (mainnet preset, parallel)" - if: matrix.os == 'ubuntu-latest' - uses: actions-rs/cargo@v1 - with: - command: test - args: shared_tests --manifest-path mcl/kzg-bench/Cargo.toml --no-fail-fast --features rust-kzg-mcl/parallel -- --test-threads 1 - - #- name: "[mcl] Tests (c-kzg-4844)" - # if: matrix.os == 'ubuntu-latest' - # run: | - # cd mcl/kzg && bash run-c-kzg-4844-tests.sh - - #- name: "[mcl] Tests (c-kzg-4844 parallel)" - # if: matrix.os == 'ubuntu-latest' - # run: | - # cd mcl/kzg && bash run-c-kzg-4844-tests.sh --parallel - - - name: "[mcl] Clippy" - if: matrix.os == 'ubuntu-latest' - uses: actions-rs/cargo@v1 - with: - command: clippy - args: --manifest-path mcl/kzg/Cargo.toml --all-targets --all-features -- -D warnings - - - name: "[mcl] Formatting" - if: matrix.os == 'ubuntu-latest' - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --manifest-path mcl/kzg/Cargo.toml -- --check - - - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --manifest-path kzg-bench/Cargo.toml -- --check diff --git a/.github/workflows/zkcrypto-benchmarks.yml b/.github/workflows/zkcrypto-benchmarks.yml deleted file mode 100644 index 07679c084..000000000 --- a/.github/workflows/zkcrypto-benchmarks.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: zkcrypto benchmarks - -on: [push, pull_request, workflow_dispatch] - -jobs: - benchmarks: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macos-11] - - steps: - - uses: actions/checkout@v2 - - - name: "[zkcrypto] Benchmark" - uses: actions-rs/cargo@v1 - with: - command: bench - args: --manifest-path zkcrypto/Cargo.toml - - - name: "[zkcrypto] Benchmark (parallel)" - uses: actions-rs/cargo@v1 - with: - command: bench - args: --manifest-path zkcrypto/Cargo.toml --features parallel diff --git a/.github/workflows/zkcrypto-tests.yml b/.github/workflows/zkcrypto-tests.yml deleted file mode 100644 index 5c380781e..000000000 --- a/.github/workflows/zkcrypto-tests.yml +++ /dev/null @@ -1,48 +0,0 @@ -name: zkcrypto tests - -on: [push, pull_request, workflow_dispatch] - -jobs: - tests: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest, windows-latest, macos-11] - - steps: - - uses: actions/checkout@v2 - - - name: "[zkcrypto] Tests (minimal preset)" - uses: actions-rs/cargo@v1 - with: - command: test - args: --manifest-path zkcrypto/Cargo.toml --no-fail-fast --test eip_4844 tests --features minimal-spec - - - name: "[zkcrypto] Tests (mainnet preset)" - uses: actions-rs/cargo@v1 - with: - command: test - args: --manifest-path zkcrypto/Cargo.toml --no-fail-fast - - - name: "[zkcrypto] Tests (mainnet preset, parallel)" - uses: actions-rs/cargo@v1 - with: - command: test - args: --manifest-path zkcrypto/Cargo.toml --no-fail-fast --features parallel - - - name: "[zkcrypto] Clippy" - uses: actions-rs/cargo@v1 - with: - command: clippy - args: --manifest-path zkcrypto/Cargo.toml --all-targets --all-features -- -D warnings - - - name: "[zkcrypto] Formatting" - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --manifest-path zkcrypto/Cargo.toml -- --check - - - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --manifest-path kzg-bench/Cargo.toml -- --check diff --git a/arkworks/run-c-kzg-4844-tests.sh b/arkworks/run-c-kzg-4844-tests.sh deleted file mode 100644 index f80b323c9..000000000 --- a/arkworks/run-c-kzg-4844-tests.sh +++ /dev/null @@ -1,151 +0,0 @@ -#!/bin/bash - -set -e - -print_msg () { - echo "[*]" "$1" -} - -###################### parallel configuration ###################### - -parallel=false - -while [[ -n $# ]]; do - case $1 in - -p|--parallel) - parallel=true - ;; - -*) - echo "Unknown parameter: $1" - exit 1 - ;; - *) - break - ;; - esac; - shift -done - -###################### building static libs ###################### - -print_msg "Compiling rust-kzg-arkworks" -if [[ "$parallel" = true ]]; then - print_msg "Using parallel version" - cargo rustc --release --crate-type=staticlib --features=parallel -else - print_msg "Using non-parallel version" - cargo rustc --release --crate-type=staticlib -fi - -mv ../target/release/librust_kzg_arkworks.a ../target/release/rust_kzg_arkworks.a - -###################### cloning c-kzg-4844 ###################### - -print_msg "Removing existing c-kzg-4844" -rm -rf c-kzg-4844 - -print_msg "Cloning c-kzg-4844" -git clone https://github.com/ethereum/c-kzg-4844.git -cd c-kzg-4844 || exit -git -c advice.detachedHead=false checkout "$C_KZG_4844_GIT_HASH" -git submodule update --init - -print_msg "Applying patches and building blst" -cd src -export CFLAGS="-Ofast -fno-builtin-memcpy -fPIC -Wall -Wextra -Werror" -make blst -unset CFLAGS -cd .. - -###################### detecting os ###################### - -case $(uname -s) in - "Linux") - CSHARP_PLATFORM=linux-x64 - CLANG_PLATFORM=x86_64-linux - ;; - "Darwin") - CSHARP_PLATFORM=osx-x64 - CLANG_PLATFORM=x86_64-darwin - ;; - *) - echo "FAIL: unsupported OS" - exit 1 - ;; -esac - -###################### dotnet tests ###################### - -print_msg "Patching dotnet binding" -git apply < ../csharp.patch -cd bindings/csharp || exit - -print_msg "Building dotnet" -make -B ckzg CSHARP_PLATFORM=$CSHARP_PLATFORM CLANG_PLATFORM=$CLANG_PLATFORM -dotnet restore - -print_msg "Running dotnet tests" -dotnet test -c release --no-restore -cd ../.. - -###################### rust tests ###################### - -print_msg "Patching rust binding" -git apply < ../rust.patch -cd bindings/rust || exit - -print_msg "Running rust tests" -cargo test --release -cd ../.. - -print_msg "Rebuilding blst" -cd src -export CFLAGS="-Ofast -fno-builtin-memcpy -fPIC -Wall -Wextra -Werror" -make blst -unset CFLAGS -cd .. - -###################### python tests ###################### - -print_msg "Patching python binding" -git apply < ../python.patch -cd bindings/python || exit - -print_msg "Running python tests" -make -cd ../.. - -###################### java tests ###################### - -print_msg "Patching java binding" -git apply < ../java.patch -cd bindings/java || exit - -print_msg "Running java tests" -make build test -cd ../.. - -###################### nodejs tests ###################### - -print_msg "Patching nodejs binding" -git apply < ../nodejs.patch -cd bindings/node.js || exit - -print_msg "Running nodejs tests" -make -cd ../.. - -###################### go tests ###################### - -print_msg "Patching go binding" -git apply < ../go.patch -cd bindings/go || exit - -print_msg "Running go tests" -CGO_CFLAGS="-O2 -D__BLST_PORTABLE__" go test -cd ../../.. - -###################### cleaning up ###################### - -print_msg "Cleaning up" -rm -rf c-kzg-4844 diff --git a/blst/run-c-kzg-4844-benches.sh b/blst/run-c-kzg-4844-benches.sh deleted file mode 100644 index 7b6d1c2bc..000000000 --- a/blst/run-c-kzg-4844-benches.sh +++ /dev/null @@ -1,83 +0,0 @@ -#!/bin/bash - -set -e - -print_msg () { - echo "[*]" "$1" -} - -###################### parallel configuration ###################### - -parallel=false - -while [[ -n $# ]]; do - case $1 in - -p|--parallel) - parallel=true - ;; - -*) - echo "Unknown parameter: $1" - exit 1 - ;; - *) - break - ;; - esac; - shift -done - -###################### building static libs ###################### - -print_msg "Compiling rust-kzg-blst" -if [[ "$parallel" = true ]]; then - print_msg "Using parallel version" - cargo rustc --release --crate-type=staticlib --features=parallel -else - print_msg "Using non-parallel version" - cargo rustc --release --crate-type=staticlib -fi - -mv ../target/release/librust_kzg_blst.a ../target/release/rust_kzg_blst.a - -###################### cloning c-kzg-4844 ###################### - -print_msg "Cloning c-kzg-4844" -git clone https://github.com/ethereum/c-kzg-4844.git -cd c-kzg-4844 || exit -git -c advice.detachedHead=false checkout "$C_KZG_4844_GIT_HASH" -git submodule update --init - -###################### rust benchmarks ###################### - -print_msg "Patching rust binding" -git apply < ../rust.patch -cd bindings/rust || exit - -print_msg "Running rust benchmarks" -cargo bench -cd ../.. - -###################### java benchmarks ###################### - -print_msg "Patching java binding" -git apply < ../java.patch -cd bindings/java || exit - -print_msg "Running java benchmarks" -make build benchmark -cd ../.. - -###################### go benchmarks ###################### - -print_msg "Patching go binding" -git apply < ../go.patch -cd bindings/go || exit - -print_msg "Running go benchmarks" -CGO_CFLAGS="-O2 -D__BLST_PORTABLE__" go test -run ^$ -bench . -cd ../../.. - -###################### cleaning up ###################### - -print_msg "Cleaning up" -rm -rf c-kzg-4844 diff --git a/mcl/kzg/run-c-kzg-4844-benches.sh b/mcl/kzg/run-c-kzg-4844-benches.sh deleted file mode 100644 index 7ee84d9f2..000000000 --- a/mcl/kzg/run-c-kzg-4844-benches.sh +++ /dev/null @@ -1,90 +0,0 @@ -#!/bin/bash - -set -e - -print_msg () { - echo "[*]" "$1" -} - -###################### parallel configuration ###################### - -parallel=false - -while [[ -n $# ]]; do - case $1 in - -p|--parallel) - parallel=true - ;; - -*) - echo "Unknown parameter: $1" - exit 1 - ;; - *) - break - ;; - esac; - shift -done - -###################### building static libs ###################### - -print_msg "Compiling rust-kzg-mcl" -if [[ "$parallel" = true ]]; then - print_msg "Using parallel version" - cargo rustc --release --crate-type=staticlib --features=parallel -else - print_msg "Using non-parallel version" - cargo rustc --release --crate-type=staticlib -fi - -mv ../../target/release/librust_kzg_mcl.a ../../target/release/rust_kzg_mcl.a - -###################### cloning c-kzg-4844 ###################### - -print_msg "Cloning c-kzg-4844" -git clone https://github.com/ethereum/c-kzg-4844.git -cd c-kzg-4844 || exit -git -c advice.detachedHead=false checkout "$C_KZG_4844_GIT_HASH" -git submodule update --init - -print_msg "Applying patches and building blst" -cd src -export CFLAGS="-Ofast -fno-builtin-memcpy -fPIC -Wall -Wextra -Werror" -make blst -unset CFLAGS -cd .. - -###################### rust benchmarks ###################### - -#print_msg "Patching rust binding" -#git apply < ../rust.patch -#cd bindings/rust || exit - -#print_msg "Running rust benchmarks" -#cargo bench -#cd ../.. - -###################### java benchmarks ###################### - -print_msg "Patching java binding" -git apply < ../java.patch -cd bindings/java || exit - -print_msg "Running java benchmarks" -make CC_FLAGS=-lstdc++ build benchmark -cd ../.. - -###################### go benchmarks ###################### - -print_msg "Patching go binding" -git apply < ../go.patch -cd bindings/go || exit - -print_msg "Running go benchmarks" -CGO_CFLAGS="-O2 -D__BLST_PORTABLE__" go test -run ^$ -bench . -cd ../../.. - -###################### cleaning up ###################### - -print_msg "Cleaning up" -rm -rf c-kzg-4844 diff --git a/mcl/kzg/run-c-kzg-4844-tests.sh b/mcl/kzg/run-c-kzg-4844-tests.sh deleted file mode 100755 index 71cb3b74f..000000000 --- a/mcl/kzg/run-c-kzg-4844-tests.sh +++ /dev/null @@ -1,141 +0,0 @@ -#!/bin/bash - -set -e - -print_msg () { - echo "[*]" "$1" -} - -###################### parallel configuration ###################### - -parallel=false - -while [[ -n $# ]]; do - case $1 in - -p|--parallel) - parallel=true - ;; - -*) - echo "Unknown parameter: $1" - exit 1 - ;; - *) - break - ;; - esac; - shift -done - -###################### building static libs ###################### - -print_msg "Compiling rust-kzg-mcl" -if [[ "$parallel" = true ]]; then - print_msg "Using parallel version" - cargo rustc --release --crate-type=staticlib --features=parallel -else - print_msg "Using non-parallel version" - cargo rustc --release --crate-type=staticlib -fi - -mv ../../target/release/librust_kzg_mcl.a ../../target/release/rust_kzg_mcl.a - -###################### cloning c-kzg-4844 ###################### - -print_msg "Cloning c-kzg-4844" -git clone https://github.com/ethereum/c-kzg-4844.git -cd c-kzg-4844 || exit -git -c advice.detachedHead=false checkout "$C_KZG_4844_GIT_HASH" -git submodule update --init - -print_msg "Applying patches and building blst" -cd src -export CFLAGS="-Ofast -fno-builtin-memcpy -fPIC -Wall -Wextra -Werror" -make blst -unset CFLAGS -cd .. - -###################### detecting os ###################### - -case $(uname -s) in - "Linux") - CSHARP_PLATFORM=linux-x64 - CLANG_PLATFORM=x86_64-linux - ;; - "Darwin") - CSHARP_PLATFORM=osx-x64 - CLANG_PLATFORM=x86_64-darwin - ;; - *) - echo "FAIL: unsupported OS" - exit 1 - ;; -esac - -###################### dotnet tests ###################### - -print_msg "Patching dotnet binding" -git apply < ../csharp.patch -cd bindings/csharp || exit - -print_msg "Building dotnet" -make -B ckzg CSHARP_PLATFORM=$CSHARP_PLATFORM CLANG_PLATFORM=$CLANG_PLATFORM -dotnet restore - -print_msg "Running dotnet tests" -dotnet test -c release --no-restore -cd ../.. - -###################### rust tests ###################### - -#print_msg "Patching rust binding" -#git apply < ../rust.patch -#cd bindings/rust || exit - -#print_msg "Running rust tests" -#cargo test --release -#cd ../.. - -###################### python tests ###################### - -print_msg "Patching python binding" -git apply < ../python.patch -cd bindings/python || exit - -print_msg "Running python tests" -make -cd ../.. - -###################### java tests ###################### - -print_msg "Patching java binding" -git apply < ../java.patch -cd bindings/java || exit - -print_msg "Running java tests" -make CC_FLAGS=-lstdc++ build test -cd ../.. - -###################### nodejs tests ###################### - -print_msg "Patching nodejs binding" -git apply < ../nodejs.patch -cd bindings/node.js || exit - -print_msg "Running nodejs tests" -make -cd ../../.. - -###################### go tests ###################### - -#print_msg "Patching go binding" -#git apply < ../go.patch -#cd bindings/go || exit - -#print_msg "Running go tests" -#CGO_CFLAGS="-O2 -D__BLST_PORTABLE__" go test -#cd ../../.. - -###################### cleaning up ###################### - -print_msg "Cleaning up" -rm -rf c-kzg-4844 diff --git a/arkworks/run-c-kzg-4844-benches.sh b/run-c-kzg-4844-benches.sh old mode 100644 new mode 100755 similarity index 78% rename from arkworks/run-c-kzg-4844-benches.sh rename to run-c-kzg-4844-benches.sh index b46fd3552..6aacec196 --- a/arkworks/run-c-kzg-4844-benches.sh +++ b/run-c-kzg-4844-benches.sh @@ -6,29 +6,37 @@ print_msg () { echo "[*]" "$1" } -###################### parallel configuration ###################### +###################### parallel & backend configuration ###################### parallel=false +backend="unknown" while [[ -n $# ]]; do case $1 in -p|--parallel) parallel=true ;; - -*) - echo "Unknown parameter: $1" - exit 1 + blst|arkworks|mcl|zkcrypto) + backend="$1" ;; *) break ;; - esac; + esac shift done +if [ "$backend" == "unknown" ]; then + echo "Unknown backend: $backend" + exit 1 +fi + ###################### building static libs ###################### +print_msg "Selected backend: $backend" + +print_msg "Compiling rust-kzg-$backend" +cd $backend -print_msg "Compiling rust-kzg-arkworks" if [[ "$parallel" = true ]]; then print_msg "Using parallel version" cargo rustc --release --crate-type=staticlib --features=parallel @@ -37,7 +45,7 @@ else cargo rustc --release --crate-type=staticlib fi -mv ../target/release/librust_kzg_arkworks.a ../target/release/rust_kzg_arkworks.a +mv ../target/release/librust_kzg_$backend.a ../target/release/rust_kzg_$backend.a ###################### cloning c-kzg-4844 ###################### diff --git a/blst/run-c-kzg-4844-tests.sh b/run-c-kzg-4844-tests.sh old mode 100644 new mode 100755 similarity index 87% rename from blst/run-c-kzg-4844-tests.sh rename to run-c-kzg-4844-tests.sh index 0a862c580..9454dc32d --- a/blst/run-c-kzg-4844-tests.sh +++ b/run-c-kzg-4844-tests.sh @@ -1,4 +1,4 @@ -#!/bin/bash +#!/opt/homebrew/bin/bash set -e @@ -6,29 +6,35 @@ print_msg () { echo "[*]" "$1" } -###################### parallel configuration ###################### +###################### parallel & backend configuration ###################### parallel=false +backend="unknown" while [[ -n $# ]]; do case $1 in -p|--parallel) parallel=true ;; - -*) - echo "Unknown parameter: $1" - exit 1 + blst|arkworks|mcl|zkcrypto) + backend="$1" ;; *) break ;; - esac; + esac shift done +if [ "$backend" == "unknown" ]; then + echo "Unknown backend: $backend" + exit 1 +fi + ###################### building static libs ###################### +print_msg "Compiling rust-kzg-$backend" +cd $backend -print_msg "Compiling rust-kzg-blst" if [[ "$parallel" = true ]]; then print_msg "Using parallel version" cargo rustc --release --crate-type=staticlib --features=parallel @@ -37,7 +43,7 @@ else cargo rustc --release --crate-type=staticlib fi -mv ../target/release/librust_kzg_blst.a ../target/release/rust_kzg_blst.a +mv ../target/release/librust_kzg_$backend.a ../target/release/rust_kzg_$backend.a ###################### cloning c-kzg-4844 ######################