From e5c129d578ca65b8694002cf432cdf07236312a0 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 1 Oct 2024 19:28:26 +0200 Subject: [PATCH 01/47] Introduce checker for dhall (deps, dirtyWhen) --- buildkite/Makefile | 10 +- buildkite/scripts/dhall/checker.py | 219 ++++++++++++++++++++++++++++ buildkite/scripts/helm-ci.sh | 2 +- buildkite/src/Jobs/Lint/Dhall.dhall | 20 +++ buildkite/src/Monorepo.dhall | 4 +- buildkite/src/Prepare.dhall | 2 +- 6 files changed, 252 insertions(+), 5 deletions(-) create mode 100755 buildkite/scripts/dhall/checker.py diff --git a/buildkite/Makefile b/buildkite/Makefile index 33de08c0af8..4e44b848430 100644 --- a/buildkite/Makefile +++ b/buildkite/Makefile @@ -14,4 +14,12 @@ lint: find ./src/ -name "*.dhall" -print0 | xargs -I{} -0 -n1 bash -c 'echo "{}" && dhall --ascii lint --inplace {} || exit 255' format: - find ./src/ -name "*.dhall" -print0 | xargs -I{} -0 -n1 bash -c 'echo "{}" && dhall --ascii format --inplace {} || exit 255' \ No newline at end of file + find ./src/ -name "*.dhall" -print0 | xargs -I{} -0 -n1 bash -c 'echo "{}" && dhall --ascii format --inplace {} || exit 255' + +check_deps: + python3 scripts/dhall/checker.py --root ./src/Jobs deps + +check_dirty: + python3 scripts/dhall/checker.py --root ./src/Jobs dirty-when + +all: check_syntax lint format check_deps check_dirty \ No newline at end of file diff --git a/buildkite/scripts/dhall/checker.py b/buildkite/scripts/dhall/checker.py new file mode 100755 index 00000000000..ce6d6578d88 --- /dev/null +++ b/buildkite/scripts/dhall/checker.py @@ -0,0 +1,219 @@ +""" + Runs dhall checks like: + + - validate if all dependencies in jobs are covered + + python3 buildkite/scripts/dhall/checker.py --root ./buildkite/src/Jobs deps + + - all dirtyWhen entries relates to existing files + + python3 buildkite/scripts/dhall/checker.py --root ./buildkite/src/Jobs dirty-when + + - print commands for given job + + python3 buildkite/scripts/dhall/checker.py --root ./buildkite/src/Jobs print-cmd --job SingleNodeTest +""" + + +import argparse +import subprocess +import os +from glob import glob +import tempfile +from pathlib import Path +import yaml + + +class CmdColors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKCYAN = '\033[96m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + + +class PipelineInfoBuilder: + + def __init__(self, temp, file): + with open(f"{temp}/{file}") as stream: + try: + self.pipeline = yaml.safe_load(stream) + self.file = file + except yaml.YAMLError as exc: + print(f"cannot parse correctly {temp}/{file}, due to {exc}") + exit(1) + + def get_steps(self): + steps = [] + for step in self.pipeline["pipeline"]["steps"]: + key = step["key"] + deps = [] + if "depends_on" in step: + for dependsOn in step["depends_on"]: + deps.append(dependsOn["step"]) + commands = step["commands"] + steps.append(Step(key, deps, commands)) + return steps + + def get_dirty(self): + dirty = [] + for dirtyWhen in self.pipeline["spec"]["dirtyWhen"]: + path = dirtyWhen["dir"][0] if "dir" in dirtyWhen else "" + exts = dirtyWhen["exts"][0] if "exts" in dirtyWhen else "" + strictEnd = bool(dirtyWhen["strictEnd"]) if ( + not "strictEnd" in dirtyWhen) else False + strictStart = bool(dirtyWhen["strictStart"]) if ( + not "strictStart" in dirtyWhen) else False + dirty.append(DirtyWhen(path=path, strictStart=strictStart, + strictEnd=strictEnd, extension=exts)) + return dirty + + def build(self): + steps = self.get_steps() + dirty = self.get_dirty() + return PipelineInfo(self.file, self.pipeline, steps, dirty) + + +class DirtyWhen: + + def __init__(self, path, extension, strictStart, strictEnd): + self.path = path + self.extension = extension + self.strictStart = strictStart + self.strictEnd = strictEnd + + def calculate_path(self): + if self.extension and self.path: + return glob(f"{self.path}.{self.extension}") + if self.strictEnd and not self.strictStart: + if not self.extension: + return glob(f"**/*/{self.path}") + else: + return glob(f"*/{self.path}.{self.extension}") + if self.strictStart and self.strictEnd: + if not self.extension: + return glob(f"{self.path}*") + else: + return glob(f"{self.path}.{self.extension}") + if self.strictStart and not self.strictEnd: + return glob(self.path + '.*') + if not self.strictStart and not self.strictEnd: + if not self.extension: + if "." in self.path: + return glob(f"**/*/{self.path}", recursive=True) + else: + return glob(f"{self.path}*") + else: + return glob(f"*.{self.extension}") + raise RuntimeError("invalid state dirty when") + + def __str__(self): + return f"path: '{self.path}', exts: '{self.extension}', startStrict:{self.strictStart}, startEnd:{self.strictEnd}" + + +class Step: + + def __init__(self, key, deps, commands): + self.key = key + self.deps = deps + self.commands = commands + + +class PipelineInfo: + + def __init__(self, file, pipeline, steps, dirty): + self.file = file + self.pipeline = pipeline + self.steps = steps + self.dirty = dirty + + def keys(self): + return [step.key for step in self.steps] + + +parser = argparse.ArgumentParser(description='Executes mina benchmarks') +parser.add_argument("--root", required=True, + help="root folder where all dhall files resides") + +subparsers = parser.add_subparsers(dest="cmd") +subparsers.add_parser('dirty-when') +subparsers.add_parser('deps') +run = subparsers.add_parser('print-cmd') +run.add_argument("--job", required=True, help="job to run") +run.add_argument("--step", required=False, help="job to run") + + +args = parser.parse_args() +tmp = tempfile.mkdtemp() + +print(f"Artifacts are stored in {tmp}") + +for file in [y for x in os.walk(args.root) for y in glob(os.path.join(x[0], '*.dhall'))]: + name = Path(file).stem + with open(f"{tmp}/{name}.yml", "w") as outfile: + subprocess.run(["dhall-to-yaml", "--quoted", "--file", + file], stdout=outfile, check=True) + + +pipelinesInfo = [PipelineInfoBuilder(tmp, file).build() + for file in os.listdir(path=tmp)] + +if args.cmd == "deps": + + keys = [] + for pipeline in pipelinesInfo: + keys.extend(pipeline.keys()) + + failedSteps = [] + + for pipeline in pipelinesInfo: + for step in pipeline.steps: + for dep in step.deps: + if not dep in keys: + failedSteps.append((pipeline, step, dep)) + + if any(failedSteps): + print("Fatal: Missing dependency resolution found:") + for (pipeline, step, dep) in failedSteps: + file = str.replace(pipeline.file, ".yml", ".dhall") + print( + f"\t{CmdColors.FAIL}[FATAL] Unresolved dependency for step '{step.key}' in '{file}' depends on non existing job '{dep}'{CmdColors.ENDC}") + exit(1) + + +if args.cmd == "print-cmd": + pipeline = next(filter(lambda x: args.job in x.file, pipelinesInfo)) + + def get_steps(): + if args.step: + return [next(filter(lambda x: args.step in x.key, pipeline.steps))] + else: + return pipeline.steps + + steps = get_steps() + + for step in steps: + for command in step.commands: + if not command.startswith("echo"): + print(command) + +if args.cmd == "dirty-when": + + failedSteps = [] + + for pipeline in pipelinesInfo: + for dirty in pipeline.dirty: + if not bool(dirty.calculate_path()): + failedSteps.append((pipeline, dirty)) + + if any(failedSteps): + print("Fatal: Non existing dirtyWhen path detected:") + for (pipeline, dirty) in failedSteps: + file = str.replace(pipeline.file, ".yml", ".dhall") + print( + f"\t{CmdColors.FAIL}[FATAL] Unresolved dirtyWhen path in '{file}' ('{str(dirty)}'){CmdColors.ENDC}") + exit(1) diff --git a/buildkite/scripts/helm-ci.sh b/buildkite/scripts/helm-ci.sh index a4c2508e5d4..0a11adbe0f5 100755 --- a/buildkite/scripts/helm-ci.sh +++ b/buildkite/scripts/helm-ci.sh @@ -4,7 +4,7 @@ set -eou pipefail diff=$( - ./buildkite/scripts/generate-diff.sh + ./buildkite/scripts/git/generate-diff.sh ) echo "--- Generated change DIFF: ${diff}" diff --git a/buildkite/src/Jobs/Lint/Dhall.dhall b/buildkite/src/Jobs/Lint/Dhall.dhall index 0f7d3e081b4..73dd7e55419 100644 --- a/buildkite/src/Jobs/Lint/Dhall.dhall +++ b/buildkite/src/Jobs/Lint/Dhall.dhall @@ -54,5 +54,25 @@ in Pipeline.build , image = (../../Constants/ContainerImages.dhall).toolchainBase } } + , Command.build + Command.Config::{ + , commands = [ Cmd.run "cd buildkite && make check_deps" ] + , label = "Dhall: deps" + , key = "check-dhall-deps" + , target = Size.Multi + , docker = Some Docker::{ + , image = (../../Constants/ContainerImages.dhall).toolchainBase + } + } + , Command.build + Command.Config::{ + , commands = [ Cmd.run "cd buildkite && make check_dirty" ] + , label = "Dhall: dirtyWhen" + , key = "check-dhall-dirty" + , target = Size.Multi + , docker = Some Docker::{ + , image = (../../Constants/ContainerImages.dhall).toolchainBase + } + } ] } diff --git a/buildkite/src/Monorepo.dhall b/buildkite/src/Monorepo.dhall index 7106d8358e5..67867e53f7d 100644 --- a/buildkite/src/Monorepo.dhall +++ b/buildkite/src/Monorepo.dhall @@ -35,8 +35,8 @@ let jobs let prefixCommands = [ Cmd.run "git config --global http.sslCAInfo /etc/ssl/certs/ca-bundle.crt" - , Cmd.run "./buildkite/scripts/refresh_code.sh" - , Cmd.run "./buildkite/scripts/generate-diff.sh > _computed_diff.txt" + , Cmd.run "./buildkite/scripts/git/refresh_code.sh" + , Cmd.run "./buildkite/scripts/git/generate-diff.sh > _computed_diff.txt" ] let commands diff --git a/buildkite/src/Prepare.dhall b/buildkite/src/Prepare.dhall index 82cdb1c4df6..52a0761207e 100644 --- a/buildkite/src/Prepare.dhall +++ b/buildkite/src/Prepare.dhall @@ -33,7 +33,7 @@ let config [ Cmd.run "export BUILDKITE_PIPELINE_MODE=${mode}" , Cmd.run "export BUILDKITE_PIPELINE_FILTER=${filter}" , Cmd.run - "./buildkite/scripts/generate-jobs.sh > buildkite/src/gen/Jobs.dhall" + "./buildkite/scripts/dhall/generate-jobs.sh > buildkite/src/gen/Jobs.dhall" , Cmd.quietly "dhall-to-yaml --quoted <<< '(./buildkite/src/Monorepo.dhall) { mode=(./buildkite/src/Pipeline/Mode.dhall).Type.${mode}, filter=(./buildkite/src/Pipeline/Filter.dhall).Type.${filter} }' | buildkite-agent pipeline upload" ] From 300f2701b64c80d09e68338ac94a4d41d6c33f0a Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 1 Oct 2024 20:46:54 +0200 Subject: [PATCH 02/47] Fix usage of checker --- buildkite/Makefile | 2 +- buildkite/scripts/dhall/checker.py | 49 +++++++++++++----------------- 2 files changed, 22 insertions(+), 29 deletions(-) diff --git a/buildkite/Makefile b/buildkite/Makefile index 4e44b848430..d163c3bf273 100644 --- a/buildkite/Makefile +++ b/buildkite/Makefile @@ -20,6 +20,6 @@ check_deps: python3 scripts/dhall/checker.py --root ./src/Jobs deps check_dirty: - python3 scripts/dhall/checker.py --root ./src/Jobs dirty-when + python3 scripts/dhall/checker.py --root $(PWD)/src/Jobs dirty-when --repo "$(PWD)/../" all: check_syntax lint format check_deps check_dirty \ No newline at end of file diff --git a/buildkite/scripts/dhall/checker.py b/buildkite/scripts/dhall/checker.py index ce6d6578d88..53de3f872f0 100755 --- a/buildkite/scripts/dhall/checker.py +++ b/buildkite/scripts/dhall/checker.py @@ -65,9 +65,9 @@ def get_dirty(self): path = dirtyWhen["dir"][0] if "dir" in dirtyWhen else "" exts = dirtyWhen["exts"][0] if "exts" in dirtyWhen else "" strictEnd = bool(dirtyWhen["strictEnd"]) if ( - not "strictEnd" in dirtyWhen) else False + "strictEnd" in dirtyWhen) else False strictStart = bool(dirtyWhen["strictStart"]) if ( - not "strictStart" in dirtyWhen) else False + "strictStart" in dirtyWhen) else False dirty.append(DirtyWhen(path=path, strictStart=strictStart, strictEnd=strictEnd, extension=exts)) return dirty @@ -86,30 +86,20 @@ def __init__(self, path, extension, strictStart, strictEnd): self.strictStart = strictStart self.strictEnd = strictEnd - def calculate_path(self): - if self.extension and self.path: - return glob(f"{self.path}.{self.extension}") - if self.strictEnd and not self.strictStart: - if not self.extension: - return glob(f"**/*/{self.path}") - else: - return glob(f"*/{self.path}.{self.extension}") - if self.strictStart and self.strictEnd: - if not self.extension: - return glob(f"{self.path}*") - else: - return glob(f"{self.path}.{self.extension}") - if self.strictStart and not self.strictEnd: - return glob(self.path + '.*') - if not self.strictStart and not self.strictEnd: - if not self.extension: - if "." in self.path: - return glob(f"**/*/{self.path}", recursive=True) - else: - return glob(f"{self.path}*") - else: - return glob(f"*.{self.extension}") - raise RuntimeError("invalid state dirty when") + def calculate_path(self,repo): + if not self.path: + return glob(os.path.join(repo,f'**/*{self.extension}')) + if not self.extension: + if self.strictEnd and self.strictStart: + return glob(os.path.join(repo, f'{self.path}')) + if not self.strictEnd and self.strictStart: + return glob(os.path.join(repo, f'{self.path}*')) + if not self.strictStart and self.strictEnd: + return glob(os.path.join(repo, f'**/{self.path}'), recursive= True) + if not self.strictStart and not self.strictEnd: + return glob(os.path.join(repo, f'*{self.path}*')) + return glob(os.path.join(repo, f'{self.path}.{self.extension}')) + #raise RuntimeError("invalid state dirty when") def __str__(self): return f"path: '{self.path}', exts: '{self.extension}', startStrict:{self.strictStart}, startEnd:{self.strictEnd}" @@ -140,7 +130,10 @@ def keys(self): help="root folder where all dhall files resides") subparsers = parser.add_subparsers(dest="cmd") -subparsers.add_parser('dirty-when') +dirty_when = subparsers.add_parser('dirty-when') +dirty_when.add_argument("--repo", required=True, + help="root folder for mina repo") + subparsers.add_parser('deps') run = subparsers.add_parser('print-cmd') run.add_argument("--job", required=True, help="job to run") @@ -207,7 +200,7 @@ def get_steps(): for pipeline in pipelinesInfo: for dirty in pipeline.dirty: - if not bool(dirty.calculate_path()): + if not bool(dirty.calculate_path(args.repo)): failedSteps.append((pipeline, dirty)) if any(failedSteps): From 752a7c6b15e601cf0c3137d7daafb1785932b356 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 1 Oct 2024 20:47:07 +0200 Subject: [PATCH 03/47] Fix jobs (dirtyWhen & dependency) --- buildkite/src/Jobs/Lint/ValidationService.dhall | 5 +---- buildkite/src/Jobs/Lint/Xrefcheck.dhall | 2 +- buildkite/src/Jobs/Release/TestnetAlerts.dhall | 2 -- buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall | 2 +- buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall | 2 +- buildkite/src/Jobs/Test/TerraformNetworkTest.dhall | 4 ++-- buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall | 6 +++--- buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall | 2 +- 8 files changed, 10 insertions(+), 15 deletions(-) diff --git a/buildkite/src/Jobs/Lint/ValidationService.dhall b/buildkite/src/Jobs/Lint/ValidationService.dhall index c9b632ee054..109ec08afed 100644 --- a/buildkite/src/Jobs/Lint/ValidationService.dhall +++ b/buildkite/src/Jobs/Lint/ValidationService.dhall @@ -75,10 +75,7 @@ in Pipeline.build (S.contains "buildkite/src/Jobs/Lint/ValidationService") in JobSpec::{ - , dirtyWhen = - [ dirtyDhallDir - , S.strictlyStart (S.contains ValidationService.rootPath) - ] + , dirtyWhen = [ dirtyDhallDir ] , path = "Lint" , name = "ValidationService" , tags = [ PipelineTag.Type.Fast, PipelineTag.Type.Lint ] diff --git a/buildkite/src/Jobs/Lint/Xrefcheck.dhall b/buildkite/src/Jobs/Lint/Xrefcheck.dhall index 20092e9f4ee..d29e2b02a36 100644 --- a/buildkite/src/Jobs/Lint/Xrefcheck.dhall +++ b/buildkite/src/Jobs/Lint/Xrefcheck.dhall @@ -23,7 +23,7 @@ in Pipeline.build , spec = JobSpec::{ , dirtyWhen = [ SelectFiles.strictly SelectFiles::{ exts = Some [ "md" ] } - , SelectFiles.strictly (SelectFiles.contains ".xrefcheck.yml") + , SelectFiles.strictly (SelectFiles.contains ".xrefcheck.yaml") ] , path = "Lint" , name = "Xrefcheck" diff --git a/buildkite/src/Jobs/Release/TestnetAlerts.dhall b/buildkite/src/Jobs/Release/TestnetAlerts.dhall index 83d8b40b045..6bad658b1df 100644 --- a/buildkite/src/Jobs/Release/TestnetAlerts.dhall +++ b/buildkite/src/Jobs/Release/TestnetAlerts.dhall @@ -40,8 +40,6 @@ in Pipeline.build , label = "Deploy Testnet alert rules" , key = "deploy-testnet-alerts" , target = Size.Medium - , depends_on = - [ { name = "TestnetAlerts", key = "lint-testnet-alerts" } ] , docker = None Docker.Type , if = Some "build.env('DEPLOY_ALERTS') == 'true'" } diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 19dae98ec15..9c8e1b199de 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -64,7 +64,7 @@ in Pipeline.build Dockers.Type.Bullseye (Some Network.Type.Berkeley) Profiles.Type.Standard - Artifacts.Type.Rosetta + Artifacts.Type.Daemon } ] } diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall index 4f0430533d6..53a94bc327d 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall @@ -56,7 +56,7 @@ in Pipeline.build Dockers.Type.Bullseye (Some Network.Type.Berkeley) Profiles.Type.Standard - Artifacts.Type.Rosetta + Artifacts.Type.Daemon } ] } diff --git a/buildkite/src/Jobs/Test/TerraformNetworkTest.dhall b/buildkite/src/Jobs/Test/TerraformNetworkTest.dhall index 1a8db62038f..a0f1b8bb2a6 100644 --- a/buildkite/src/Jobs/Test/TerraformNetworkTest.dhall +++ b/buildkite/src/Jobs/Test/TerraformNetworkTest.dhall @@ -35,8 +35,8 @@ in Pipeline.build Pipeline.Config::{ , spec = let unitDirtyWhen = - [ S.strictlyStart (S.contains "src/automation/terraform") - , S.strictlyStart (S.contains "src/helm") + [ S.strictlyStart (S.contains "automation/terraform") + , S.strictlyStart (S.contains "helm") , S.strictlyStart (S.contains "buildkite/src/Jobs/Test/TerraformNetworkTest") , S.strictlyStart diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index d7acd176432..cae92fdbfd2 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -39,18 +39,18 @@ in Pipeline.build , S.strictlyStart (S.contains "buildkite/src/Jobs/Test/TestnetIntegrationTest") , S.strictlyStart - (S.contains "buildkite/src/Jobs/Command/TestExecutive") + (S.contains "buildkite/src/Command/TestExecutive") , S.strictlyStart (S.contains "automation/terraform/modules/o1-integration") , S.strictlyStart (S.contains "automation/terraform/modules/kubernetes/testnet") , S.strictlyStart ( S.contains - "automation/buildkite/script/run-test-executive-cloud" + "buildkite/scripts/run-test-executive-cloud" ) , S.strictlyStart ( S.contains - "automation/buildkite/script/run-test-executive-local" + "buildkite/scripts/run-test-executive-local" ) ] , path = "Test" diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall index 5cf7f25bdb3..22733be4e00 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall @@ -39,7 +39,7 @@ in Pipeline.build , S.strictlyStart (S.contains "buildkite/src/Jobs/Test/TestnetIntegrationTest") , S.strictlyStart - (S.contains "buildkite/src/Jobs/Command/TestExecutive") + (S.contains "buildkite/src/Command/TestExecutive") , S.strictlyStart (S.contains "automation/terraform/modules/o1-integration") , S.strictlyStart From 3f17bcbe47bf4443ab3dfe90a27222590b37f2e3 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 1 Oct 2024 21:22:12 +0200 Subject: [PATCH 04/47] Revert "Auxiliary commit to revert individual files from e5c129d578ca65b8694002cf432cdf07236312a0" This reverts commit 35f9de53f244c91656d101cb4b523deb6985367c. --- buildkite/src/Monorepo.dhall | 4 ++-- buildkite/src/Prepare.dhall | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/buildkite/src/Monorepo.dhall b/buildkite/src/Monorepo.dhall index 67867e53f7d..7106d8358e5 100644 --- a/buildkite/src/Monorepo.dhall +++ b/buildkite/src/Monorepo.dhall @@ -35,8 +35,8 @@ let jobs let prefixCommands = [ Cmd.run "git config --global http.sslCAInfo /etc/ssl/certs/ca-bundle.crt" - , Cmd.run "./buildkite/scripts/git/refresh_code.sh" - , Cmd.run "./buildkite/scripts/git/generate-diff.sh > _computed_diff.txt" + , Cmd.run "./buildkite/scripts/refresh_code.sh" + , Cmd.run "./buildkite/scripts/generate-diff.sh > _computed_diff.txt" ] let commands diff --git a/buildkite/src/Prepare.dhall b/buildkite/src/Prepare.dhall index 52a0761207e..82cdb1c4df6 100644 --- a/buildkite/src/Prepare.dhall +++ b/buildkite/src/Prepare.dhall @@ -33,7 +33,7 @@ let config [ Cmd.run "export BUILDKITE_PIPELINE_MODE=${mode}" , Cmd.run "export BUILDKITE_PIPELINE_FILTER=${filter}" , Cmd.run - "./buildkite/scripts/dhall/generate-jobs.sh > buildkite/src/gen/Jobs.dhall" + "./buildkite/scripts/generate-jobs.sh > buildkite/src/gen/Jobs.dhall" , Cmd.quietly "dhall-to-yaml --quoted <<< '(./buildkite/src/Monorepo.dhall) { mode=(./buildkite/src/Pipeline/Mode.dhall).Type.${mode}, filter=(./buildkite/src/Pipeline/Filter.dhall).Type.${filter} }' | buildkite-agent pipeline upload" ] From 2de0853f71015fbfd53d51cc579a0605de579e2d Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 1 Oct 2024 21:26:02 +0200 Subject: [PATCH 05/47] remove comment from checker.py --- buildkite/scripts/dhall/checker.py | 1 - 1 file changed, 1 deletion(-) diff --git a/buildkite/scripts/dhall/checker.py b/buildkite/scripts/dhall/checker.py index 53de3f872f0..4e7f49f3900 100755 --- a/buildkite/scripts/dhall/checker.py +++ b/buildkite/scripts/dhall/checker.py @@ -99,7 +99,6 @@ def calculate_path(self,repo): if not self.strictStart and not self.strictEnd: return glob(os.path.join(repo, f'*{self.path}*')) return glob(os.path.join(repo, f'{self.path}.{self.extension}')) - #raise RuntimeError("invalid state dirty when") def __str__(self): return f"path: '{self.path}', exts: '{self.extension}', startStrict:{self.strictStart}, startEnd:{self.strictEnd}" From 416becbaf87be1f33aacb3184128df1fd7056d09 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 1 Oct 2024 21:27:27 +0200 Subject: [PATCH 06/47] Revert "Auxiliary commit to revert individual files from e5c129d578ca65b8694002cf432cdf07236312a0" This reverts commit 561a5b889757092f8d9c4e21605b96d36dc53828. --- buildkite/scripts/helm-ci.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/helm-ci.sh b/buildkite/scripts/helm-ci.sh index 0a11adbe0f5..a4c2508e5d4 100755 --- a/buildkite/scripts/helm-ci.sh +++ b/buildkite/scripts/helm-ci.sh @@ -4,7 +4,7 @@ set -eou pipefail diff=$( - ./buildkite/scripts/git/generate-diff.sh + ./buildkite/scripts/generate-diff.sh ) echo "--- Generated change DIFF: ${diff}" From 4608a8ad3050832dbe6ecbfe7d166974191e6f55 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 1 Oct 2024 21:31:10 +0200 Subject: [PATCH 07/47] use toolchain for python check + format & lints --- buildkite/src/Jobs/Lint/Dhall.dhall | 20 +++++++++++-------- .../Jobs/Test/TestnetIntegrationTests.dhall | 11 +++------- .../Test/TestnetIntegrationTestsLong.dhall | 3 +-- 3 files changed, 16 insertions(+), 18 deletions(-) diff --git a/buildkite/src/Jobs/Lint/Dhall.dhall b/buildkite/src/Jobs/Lint/Dhall.dhall index 73dd7e55419..68d5678e684 100644 --- a/buildkite/src/Jobs/Lint/Dhall.dhall +++ b/buildkite/src/Jobs/Lint/Dhall.dhall @@ -12,6 +12,8 @@ let Command = ../../Command/Base.dhall let Docker = ../../Command/Docker/Type.dhall +let RunInToolchain = ../../Command/RunInToolchain.dhall + let Size = ../../Command/Size.dhall in Pipeline.build @@ -56,23 +58,25 @@ in Pipeline.build } , Command.build Command.Config::{ - , commands = [ Cmd.run "cd buildkite && make check_deps" ] + , commands = + RunInToolchain.runInToolchainBullseye + ([] : List Text) + "cd buildkite && make check_deps" , label = "Dhall: deps" , key = "check-dhall-deps" , target = Size.Multi - , docker = Some Docker::{ - , image = (../../Constants/ContainerImages.dhall).toolchainBase - } + , docker = None Docker.Type } , Command.build Command.Config::{ - , commands = [ Cmd.run "cd buildkite && make check_dirty" ] + , commands = + RunInToolchain.runInToolchainBullseye + ([] : List Text) + "cd buildkite && make check_dirty" , label = "Dhall: dirtyWhen" , key = "check-dhall-dirty" , target = Size.Multi - , docker = Some Docker::{ - , image = (../../Constants/ContainerImages.dhall).toolchainBase - } + , docker = None Docker.Type } ] } diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall index cae92fdbfd2..6e0a3796207 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTests.dhall @@ -38,20 +38,15 @@ in Pipeline.build , S.strictlyStart (S.contains "dockerfiles") , S.strictlyStart (S.contains "buildkite/src/Jobs/Test/TestnetIntegrationTest") - , S.strictlyStart - (S.contains "buildkite/src/Command/TestExecutive") + , S.strictlyStart (S.contains "buildkite/src/Command/TestExecutive") , S.strictlyStart (S.contains "automation/terraform/modules/o1-integration") , S.strictlyStart (S.contains "automation/terraform/modules/kubernetes/testnet") , S.strictlyStart - ( S.contains - "buildkite/scripts/run-test-executive-cloud" - ) + (S.contains "buildkite/scripts/run-test-executive-cloud") , S.strictlyStart - ( S.contains - "buildkite/scripts/run-test-executive-local" - ) + (S.contains "buildkite/scripts/run-test-executive-local") ] , path = "Test" , name = "TestnetIntegrationTests" diff --git a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall index 22733be4e00..484e128740d 100644 --- a/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/TestnetIntegrationTestsLong.dhall @@ -38,8 +38,7 @@ in Pipeline.build , S.strictlyStart (S.contains "dockerfiles") , S.strictlyStart (S.contains "buildkite/src/Jobs/Test/TestnetIntegrationTest") - , S.strictlyStart - (S.contains "buildkite/src/Command/TestExecutive") + , S.strictlyStart (S.contains "buildkite/src/Command/TestExecutive") , S.strictlyStart (S.contains "automation/terraform/modules/o1-integration") , S.strictlyStart From bcbb84c5504ff68f16ef4a640af36489000dbe91 Mon Sep 17 00:00:00 2001 From: dkijania Date: Tue, 1 Oct 2024 22:47:17 +0200 Subject: [PATCH 08/47] do not use inner docker when calling deps and dirtyWhen checks --- buildkite/src/Jobs/Lint/Dhall.dhall | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/buildkite/src/Jobs/Lint/Dhall.dhall b/buildkite/src/Jobs/Lint/Dhall.dhall index 68d5678e684..b56a1495e74 100644 --- a/buildkite/src/Jobs/Lint/Dhall.dhall +++ b/buildkite/src/Jobs/Lint/Dhall.dhall @@ -12,8 +12,6 @@ let Command = ../../Command/Base.dhall let Docker = ../../Command/Docker/Type.dhall -let RunInToolchain = ../../Command/RunInToolchain.dhall - let Size = ../../Command/Size.dhall in Pipeline.build @@ -58,10 +56,7 @@ in Pipeline.build } , Command.build Command.Config::{ - , commands = - RunInToolchain.runInToolchainBullseye - ([] : List Text) - "cd buildkite && make check_deps" + , commands = [ Cmd.run "cd buildkite && make check_deps" ] , label = "Dhall: deps" , key = "check-dhall-deps" , target = Size.Multi @@ -69,10 +64,7 @@ in Pipeline.build } , Command.build Command.Config::{ - , commands = - RunInToolchain.runInToolchainBullseye - ([] : List Text) - "cd buildkite && make check_dirty" + , commands = [ Cmd.run "cd buildkite && make check_dirty" ] , label = "Dhall: dirtyWhen" , key = "check-dhall-dirty" , target = Size.Multi From aaf80c63313edacf3e311ec16d4798db96222c71 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 2 Oct 2024 11:32:17 +0200 Subject: [PATCH 09/47] Using dhall-to-yaml in toolchain base while python script in toolchain container --- buildkite/Makefile | 9 +++++-- buildkite/scripts/dhall/checker.py | 21 ++++++--------- .../scripts/dhall/dump_dhall_to_pipelines.sh | 24 +++++++++++++++++ buildkite/src/Jobs/Lint/Dhall.dhall | 26 ++++++++++++++++--- 4 files changed, 61 insertions(+), 19 deletions(-) create mode 100755 buildkite/scripts/dhall/dump_dhall_to_pipelines.sh diff --git a/buildkite/Makefile b/buildkite/Makefile index d163c3bf273..b055603295a 100644 --- a/buildkite/Makefile +++ b/buildkite/Makefile @@ -16,10 +16,15 @@ lint: format: find ./src/ -name "*.dhall" -print0 | xargs -I{} -0 -n1 bash -c 'echo "{}" && dhall --ascii format --inplace {} || exit 255' + check_deps: - python3 scripts/dhall/checker.py --root ./src/Jobs deps + $(eval TMP := $(shell mktemp -d)) + scripts/dhall/dump_dhall_to_pipelines.sh src/Jobs "$(TMP)" + python3 scripts/dhall/checker.py --root "$(TMP)" deps check_dirty: - python3 scripts/dhall/checker.py --root $(PWD)/src/Jobs dirty-when --repo "$(PWD)/../" + $(eval TMP := $(shell mktemp -d)) + scripts/dhall/dump_dhall_to_pipelines.sh src/Jobs "$(TMP)" + python3 scripts/dhall/checker.py --root "$(TMP)" dirty-when --repo "$(PWD)/../" all: check_syntax lint format check_deps check_dirty \ No newline at end of file diff --git a/buildkite/scripts/dhall/checker.py b/buildkite/scripts/dhall/checker.py index 4e7f49f3900..df64bc45092 100755 --- a/buildkite/scripts/dhall/checker.py +++ b/buildkite/scripts/dhall/checker.py @@ -134,25 +134,17 @@ def keys(self): help="root folder for mina repo") subparsers.add_parser('deps') + + run = subparsers.add_parser('print-cmd') run.add_argument("--job", required=True, help="job to run") run.add_argument("--step", required=False, help="job to run") args = parser.parse_args() -tmp = tempfile.mkdtemp() - -print(f"Artifacts are stored in {tmp}") -for file in [y for x in os.walk(args.root) for y in glob(os.path.join(x[0], '*.dhall'))]: - name = Path(file).stem - with open(f"{tmp}/{name}.yml", "w") as outfile: - subprocess.run(["dhall-to-yaml", "--quoted", "--file", - file], stdout=outfile, check=True) - - -pipelinesInfo = [PipelineInfoBuilder(tmp, file).build() - for file in os.listdir(path=tmp)] +pipelinesInfo = [PipelineInfoBuilder(args.root, file).build() + for file in os.listdir(path=args.root)] if args.cmd == "deps": @@ -175,7 +167,8 @@ def keys(self): print( f"\t{CmdColors.FAIL}[FATAL] Unresolved dependency for step '{step.key}' in '{file}' depends on non existing job '{dep}'{CmdColors.ENDC}") exit(1) - + else: + print('Pipelines definitions correct') if args.cmd == "print-cmd": pipeline = next(filter(lambda x: args.job in x.file, pipelinesInfo)) @@ -209,3 +202,5 @@ def get_steps(): print( f"\t{CmdColors.FAIL}[FATAL] Unresolved dirtyWhen path in '{file}' ('{str(dirty)}'){CmdColors.ENDC}") exit(1) + else: + print('Pipelines definitions correct') diff --git a/buildkite/scripts/dhall/dump_dhall_to_pipelines.sh b/buildkite/scripts/dhall/dump_dhall_to_pipelines.sh new file mode 100755 index 00000000000..84193329b76 --- /dev/null +++ b/buildkite/scripts/dhall/dump_dhall_to_pipelines.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +ROOT=$1 +OUTPUT=$2 + +mkdir -p "$OUTPUT" + +shopt -s globstar nullglob + +echo "Dumping pipelines from '$ROOT' to '$OUTPUT'" + +COUNTER=0 + +for file in "$ROOT"/**/*.dhall +do + filename=$(basename "$file") + filename="${filename%.*}" + + dhall-to-yaml --quoted --file "$file" > "$OUTPUT"/"$filename".yml + + COUNTER=$((COUNTER+1)) +done + +echo "Done. $COUNTER jobs exported" diff --git a/buildkite/src/Jobs/Lint/Dhall.dhall b/buildkite/src/Jobs/Lint/Dhall.dhall index b56a1495e74..c26aa6c8698 100644 --- a/buildkite/src/Jobs/Lint/Dhall.dhall +++ b/buildkite/src/Jobs/Lint/Dhall.dhall @@ -14,6 +14,8 @@ let Docker = ../../Command/Docker/Type.dhall let Size = ../../Command/Size.dhall +let RunInToolchain = ../../Command/RunInToolchain.dhall + in Pipeline.build Pipeline.Config::{ , spec = JobSpec::{ @@ -56,19 +58,35 @@ in Pipeline.build } , Command.build Command.Config::{ - , commands = [ Cmd.run "cd buildkite && make check_deps" ] + , commands = + [ Cmd.run + "buildkite/scripts/dhall/dump_dhall_to_pipelines.sh buildkite/src/Jobs _pipelines" + ] + # RunInToolchain.runInToolchainBullseye + ([] : List Text) + "python3 ./buildkite/scripts/dhall/checker.py --root _pipelines deps" , label = "Dhall: deps" , key = "check-dhall-deps" , target = Size.Multi - , docker = None Docker.Type + , docker = Some Docker::{ + , image = (../../Constants/ContainerImages.dhall).toolchainBase + } } , Command.build Command.Config::{ - , commands = [ Cmd.run "cd buildkite && make check_dirty" ] + , commands = + [ Cmd.run + "buildkite/scripts/dhall/dump_dhall_to_pipelines.sh buildkite/src/Jobs _pipelines" + ] + # RunInToolchain.runInToolchainBullseye + ([] : List Text) + "python3 scripts/dhall/checker.py --root _pipelines dirty-when --repo ." , label = "Dhall: dirtyWhen" , key = "check-dhall-dirty" , target = Size.Multi - , docker = None Docker.Type + , docker = Some Docker::{ + , image = (../../Constants/ContainerImages.dhall).toolchainBase + } } ] } From ec996e22a0d695c624452d2ac56f570378806f11 Mon Sep 17 00:00:00 2001 From: dkijania Date: Wed, 2 Oct 2024 16:19:06 +0200 Subject: [PATCH 10/47] run dump pipelines in docker --- buildkite/src/Jobs/Lint/Dhall.dhall | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/buildkite/src/Jobs/Lint/Dhall.dhall b/buildkite/src/Jobs/Lint/Dhall.dhall index c26aa6c8698..999c45712f8 100644 --- a/buildkite/src/Jobs/Lint/Dhall.dhall +++ b/buildkite/src/Jobs/Lint/Dhall.dhall @@ -16,6 +16,13 @@ let Size = ../../Command/Size.dhall let RunInToolchain = ../../Command/RunInToolchain.dhall +let dump_pipelines_cmd = + Cmd.runInDocker + Cmd.Docker::{ + , image = (../../Constants/ContainerImages.dhall).toolchainBase + } + "buildkite/scripts/dhall/dump_dhall_to_pipelines.sh buildkite/src/Jobs _pipelines" + in Pipeline.build Pipeline.Config::{ , spec = JobSpec::{ @@ -59,34 +66,26 @@ in Pipeline.build , Command.build Command.Config::{ , commands = - [ Cmd.run - "buildkite/scripts/dhall/dump_dhall_to_pipelines.sh buildkite/src/Jobs _pipelines" - ] + [ dump_pipelines_cmd ] # RunInToolchain.runInToolchainBullseye ([] : List Text) "python3 ./buildkite/scripts/dhall/checker.py --root _pipelines deps" , label = "Dhall: deps" , key = "check-dhall-deps" , target = Size.Multi - , docker = Some Docker::{ - , image = (../../Constants/ContainerImages.dhall).toolchainBase - } + , docker = None Docker.Type } , Command.build Command.Config::{ , commands = - [ Cmd.run - "buildkite/scripts/dhall/dump_dhall_to_pipelines.sh buildkite/src/Jobs _pipelines" - ] + [ dump_pipelines_cmd ] # RunInToolchain.runInToolchainBullseye ([] : List Text) "python3 scripts/dhall/checker.py --root _pipelines dirty-when --repo ." , label = "Dhall: dirtyWhen" , key = "check-dhall-dirty" , target = Size.Multi - , docker = Some Docker::{ - , image = (../../Constants/ContainerImages.dhall).toolchainBase - } + , docker = None Docker.Type } ] } From 16157b4e66bd09c838b67283dc7c7715846171f5 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 3 Oct 2024 09:29:49 +0200 Subject: [PATCH 11/47] fix path to checker.py for dirtyWhen --- buildkite/src/Jobs/Lint/Dhall.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Jobs/Lint/Dhall.dhall b/buildkite/src/Jobs/Lint/Dhall.dhall index 999c45712f8..c813211fb28 100644 --- a/buildkite/src/Jobs/Lint/Dhall.dhall +++ b/buildkite/src/Jobs/Lint/Dhall.dhall @@ -81,7 +81,7 @@ in Pipeline.build [ dump_pipelines_cmd ] # RunInToolchain.runInToolchainBullseye ([] : List Text) - "python3 scripts/dhall/checker.py --root _pipelines dirty-when --repo ." + "python3 ./buildkite/scripts/dhall/checker.py --root _pipelines dirty-when --repo ." , label = "Dhall: dirtyWhen" , key = "check-dhall-dirty" , target = Size.Multi From 4bc2436e9eee50fab39dad8143f13565a88f4869 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 3 Oct 2024 10:38:38 +0200 Subject: [PATCH 12/47] fix RosettaIntegration tests dependencies from daemon to rosetta docker --- buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall | 2 +- buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall index 9c8e1b199de..19dae98ec15 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTests.dhall @@ -64,7 +64,7 @@ in Pipeline.build Dockers.Type.Bullseye (Some Network.Type.Berkeley) Profiles.Type.Standard - Artifacts.Type.Daemon + Artifacts.Type.Rosetta } ] } diff --git a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall index 53a94bc327d..4f0430533d6 100644 --- a/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall +++ b/buildkite/src/Jobs/Test/RosettaIntegrationTestsLong.dhall @@ -56,7 +56,7 @@ in Pipeline.build Dockers.Type.Bullseye (Some Network.Type.Berkeley) Profiles.Type.Standard - Artifacts.Type.Daemon + Artifacts.Type.Rosetta } ] } From d8bbd270b10743a63baaaa4cb99242d63d54f465 Mon Sep 17 00:00:00 2001 From: dkijania Date: Thu, 3 Oct 2024 20:31:25 +0200 Subject: [PATCH 13/47] remove additional empty line --- buildkite/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/buildkite/Makefile b/buildkite/Makefile index b055603295a..5464c602687 100644 --- a/buildkite/Makefile +++ b/buildkite/Makefile @@ -16,7 +16,6 @@ lint: format: find ./src/ -name "*.dhall" -print0 | xargs -I{} -0 -n1 bash -c 'echo "{}" && dhall --ascii format --inplace {} || exit 255' - check_deps: $(eval TMP := $(shell mktemp -d)) scripts/dhall/dump_dhall_to_pipelines.sh src/Jobs "$(TMP)" From 20a08ebbc352c1f13b2dc096d763871e90edca48 Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Tue, 15 Oct 2024 00:48:46 +0100 Subject: [PATCH 14/47] Use the appropriate logger for snark pool batcher --- src/lib/network_pool/batcher.ml | 5 ++--- src/lib/network_pool/batcher.mli | 2 +- src/lib/network_pool/snark_pool.ml | 2 +- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/lib/network_pool/batcher.ml b/src/lib/network_pool/batcher.ml index c2a9a147165..16759fd478e 100644 --- a/src/lib/network_pool/batcher.ml +++ b/src/lib/network_pool/batcher.ml @@ -431,8 +431,7 @@ module Snark_pool = struct let open Deferred.Or_error.Let_syntax in match%map verify t p with Ok () -> true | Error _ -> false - let create verifier : t = - let logger = Logger.create () in + let create ~logger verifier : t = create (* TODO: Make this a proper config detail once we have data on what a good default would be. @@ -539,7 +538,7 @@ module Snark_pool = struct Envelope.Incoming.gen data_gen let run_test proof_lists = - let batcher = create verifier in + let batcher = create ~logger verifier in Deferred.List.iter proof_lists ~f:(fun (invalid_proofs, proof_list) -> let%map r = verify' batcher proof_list in let (`Invalid ps) = Or_error.ok_exn r in diff --git a/src/lib/network_pool/batcher.mli b/src/lib/network_pool/batcher.mli index e08d8ed30d0..268b5b6bee9 100644 --- a/src/lib/network_pool/batcher.mli +++ b/src/lib/network_pool/batcher.mli @@ -9,7 +9,7 @@ module Snark_pool : sig type t [@@deriving sexp] - val create : Verifier.t -> t + val create : logger:Logger.t -> Verifier.t -> t val verify : t -> proof_envelope -> bool Deferred.Or_error.t end diff --git a/src/lib/network_pool/snark_pool.ml b/src/lib/network_pool/snark_pool.ml index 6ede322b927..8343d8ead0c 100644 --- a/src/lib/network_pool/snark_pool.ml +++ b/src/lib/network_pool/snark_pool.ml @@ -256,7 +256,7 @@ struct } ; frontier = (fun () -> Broadcast_pipe.Reader.peek frontier_broadcast_pipe) - ; batcher = Batcher.Snark_pool.create config.verifier + ; batcher = Batcher.Snark_pool.create ~logger config.verifier ; logger ; config ; account_creation_fee = From 87f72147a72f0c98df761006c8f5604f90231f5d Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Tue, 15 Oct 2024 00:53:10 +0100 Subject: [PATCH 15/47] Show log output for batcher tests --- src/lib/network_pool/batcher.ml | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/lib/network_pool/batcher.ml b/src/lib/network_pool/batcher.ml index 16759fd478e..010a36f8eb0 100644 --- a/src/lib/network_pool/batcher.ml +++ b/src/lib/network_pool/batcher.ml @@ -2,9 +2,6 @@ open Core_kernel open Async_kernel open Network_peer -(* Only show stdout for failed inline tests. *) -open Inline_test_quiet_logs - module Id = Unique_id.Int () type ('init, 'result) elt = From 13b5c833f2d3edd5aa0fbd635859879eed028212 Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Tue, 15 Oct 2024 01:45:41 +0100 Subject: [PATCH 16/47] Don't randomly print out SCHNORR BACKTRACE every 1000 hash verifications --- src/lib/network_pool/batcher.ml | 9 ++++----- src/lib/network_pool/batcher.mli | 4 ++-- src/lib/network_pool/transaction_pool.ml | 4 ++-- src/lib/signature_lib/schnorr.ml | 3 --- 4 files changed, 8 insertions(+), 12 deletions(-) diff --git a/src/lib/network_pool/batcher.ml b/src/lib/network_pool/batcher.ml index 010a36f8eb0..9f42bbc4dfd 100644 --- a/src/lib/network_pool/batcher.ml +++ b/src/lib/network_pool/batcher.ml @@ -42,7 +42,7 @@ type ('init, 'partially_validated, 'result) t = } [@@deriving sexp] -let create ?(how_to_add = `Enqueue_back) ?logger ?compare_init +let create ?(how_to_add = `Enqueue_back) ~logger ?compare_init ?(weight = fun _ -> 1) ?max_weight_per_call verifier = { state = Waiting ; queue = Q.create () @@ -51,7 +51,7 @@ let create ?(how_to_add = `Enqueue_back) ?logger ?compare_init ; verifier ; weight ; max_weight_per_call - ; logger = Option.value logger ~default:(Logger.create ()) + ; logger } let call_verifier t (ps : 'proof list) = t.verifier ps @@ -75,7 +75,7 @@ let rec determine_outcome : (* First separate out all the known results. That information will definitely be included in the outcome. *) - let logger = Logger.create () in + let logger = v.logger in let potentially_invalid = List.filter_map (List.zip_exn ps res) ~f:(fun (elt, r) -> match r with @@ -300,8 +300,7 @@ module Transaction_pool = struct (Array.to_list (Array.map a ~f:(function `Valid c -> Some c | _ -> None)) ) - let create verifier : t = - let logger = Logger.create () in + let create ~logger verifier : t = create ~compare_init:compare_envelope ~logger (fun (ds : input list) -> O1trace.thread "dispatching_transaction_pool_batcher_verification" (fun () -> diff --git a/src/lib/network_pool/batcher.mli b/src/lib/network_pool/batcher.mli index 268b5b6bee9..c46bcb0b2e3 100644 --- a/src/lib/network_pool/batcher.mli +++ b/src/lib/network_pool/batcher.mli @@ -18,7 +18,7 @@ type ('initial, 'partially_validated, 'result) t val create : ?how_to_add:[ `Insert | `Enqueue_back ] - -> ?logger:Logger.t + -> logger:Logger.t -> ?compare_init:('init -> 'init -> int) -> ?weight:('init -> int) -> ?max_weight_per_call:int @@ -42,7 +42,7 @@ module Transaction_pool : sig type t [@@deriving sexp] - val create : Verifier.t -> t + val create : logger:Logger.t -> Verifier.t -> t val verify : t diff --git a/src/lib/network_pool/transaction_pool.ml b/src/lib/network_pool/transaction_pool.ml index e99224ecdb8..9cf697e2c06 100644 --- a/src/lib/network_pool/transaction_pool.ml +++ b/src/lib/network_pool/transaction_pool.ml @@ -831,7 +831,7 @@ struct ; remaining_in_batch = max_per_15_seconds ; config ; logger - ; batcher = Batcher.create config.verifier + ; batcher = Batcher.create ~logger config.verifier ; best_tip_diff_relay = None ; best_tip_ledger = None ; verification_key_table = Vk_refcount_table.create () @@ -1661,7 +1661,7 @@ let%test_module _ = let minimum_fee = Currency.Fee.to_nanomina_int genesis_constants.minimum_user_command_fee - let logger = Logger.create () + let logger = Logger.null () let time_controller = Block_time.Controller.basic ~logger diff --git a/src/lib/signature_lib/schnorr.ml b/src/lib/signature_lib/schnorr.ml index f31d64ee927..a1ef7c23443 100644 --- a/src/lib/signature_lib/schnorr.ml +++ b/src/lib/signature_lib/schnorr.ml @@ -240,9 +240,6 @@ module Make let verify ?signature_kind ((r, s) : Signature.t) (pk : Public_key.t) (m : Message.t) = - if Random.int 1000 = 0 then ( - print_endline "SCHNORR BACKTRACE:" ; - Printexc.print_backtrace stdout ) ; let hash = Message.hash ?signature_kind in let e = hash ~public_key:pk ~r m in let r_pt = Curve.(scale one s + negate (scale pk e)) in From e3da5c15f0f2c74bb6326577dabe8eb43a91bf5e Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Tue, 15 Oct 2024 01:49:18 +0100 Subject: [PATCH 17/47] Turn off debug printing --- src/lib/network_pool/transaction_pool.ml | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/lib/network_pool/transaction_pool.ml b/src/lib/network_pool/transaction_pool.ml index 9cf697e2c06..3a9dd48bcad 100644 --- a/src/lib/network_pool/transaction_pool.ml +++ b/src/lib/network_pool/transaction_pool.ml @@ -2203,19 +2203,22 @@ let%test_module _ = let tm1 = Time.now () in [%log' info test.txn_pool.logger] "Time for add_commands: %0.04f sec" (Time.diff tm1 tm0 |> Time.Span.to_sec) ; + let debug = false in ( match result with | Ok (`Accept, _, rejects) -> - List.iter rejects ~f:(fun (cmd, err) -> - Core.Printf.printf - !"command was rejected because %s: %{Yojson.Safe}\n%!" - (Diff_versioned.Diff_error.to_string_name err) - (User_command.to_yojson cmd) ) + if debug then + List.iter rejects ~f:(fun (cmd, err) -> + Core.Printf.printf + !"command was rejected because %s: %{Yojson.Safe}\n%!" + (Diff_versioned.Diff_error.to_string_name err) + (User_command.to_yojson cmd) ) | Ok (`Reject, _, _) -> failwith "diff was rejected during application" | Error (`Other err) -> - Core.Printf.printf - !"failed to apply diff to pool: %s\n%!" - (Error.to_string_hum err) ) ; + if debug then + Core.Printf.printf + !"failed to apply diff to pool: %s\n%!" + (Error.to_string_hum err) ) ; result let add_commands' ?local test cs = From 92329e9834bf4ba0b3a0be372b9ae4483cca8233 Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Tue, 15 Oct 2024 01:55:10 +0100 Subject: [PATCH 18/47] Stop using Inline_test_quiet_logs for Network_pool tests --- src/lib/network_pool/snark_pool.ml | 3 --- src/lib/network_pool/test.ml | 3 --- src/lib/network_pool/transaction_pool.ml | 2 -- 3 files changed, 8 deletions(-) diff --git a/src/lib/network_pool/snark_pool.ml b/src/lib/network_pool/snark_pool.ml index 8343d8ead0c..5b152112a75 100644 --- a/src/lib/network_pool/snark_pool.ml +++ b/src/lib/network_pool/snark_pool.ml @@ -559,9 +559,6 @@ module Diff_versioned = struct [@@deriving compare, sexp, to_yojson, hash] end -(* Only show stdout for failed inline tests. *) -open Inline_test_quiet_logs - let%test_module "random set test" = ( module struct open Mina_base diff --git a/src/lib/network_pool/test.ml b/src/lib/network_pool/test.ml index b0ddac4e0a4..5181b7aabc8 100644 --- a/src/lib/network_pool/test.ml +++ b/src/lib/network_pool/test.ml @@ -3,9 +3,6 @@ open Core_kernel open Pipe_lib open Network_peer -(* Only show stdout for failed inline tests. *) -open Inline_test_quiet_logs - let%test_module "network pool test" = ( module struct let trust_system = Mocks.trust_system diff --git a/src/lib/network_pool/transaction_pool.ml b/src/lib/network_pool/transaction_pool.ml index 3a9dd48bcad..fe4e4bf2b8a 100644 --- a/src/lib/network_pool/transaction_pool.ml +++ b/src/lib/network_pool/transaction_pool.ml @@ -3,8 +3,6 @@ transactions (user commands) and providing them to the block producer code. *) -(* Only show stdout for failed inline tests.*) -open Inline_test_quiet_logs open Core open Async open Mina_base From 63725a4909a8053371f95961f7b55c61cb13d763 Mon Sep 17 00:00:00 2001 From: Gregor Date: Fri, 18 Oct 2024 21:30:50 +0200 Subject: [PATCH 19/47] fix maybe_lagrange_commitment --- src/lib/crypto/kimchi_bindings/wasm/src/srs.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lib/crypto/kimchi_bindings/wasm/src/srs.rs b/src/lib/crypto/kimchi_bindings/wasm/src/srs.rs index 1d436ab1f91..2b33c56d59f 100644 --- a/src/lib/crypto/kimchi_bindings/wasm/src/srs.rs +++ b/src/lib/crypto/kimchi_bindings/wasm/src/srs.rs @@ -253,6 +253,9 @@ pub mod fp { domain_size: i32, i: i32, ) -> Option { + if !(srs.0.lagrange_bases.contains_key(&(domain_size as usize))) { + return None; + } let basis = srs.get_lagrange_basis_from_domain_size(domain_size as usize); Some(basis[i as usize].clone().into()) } @@ -330,7 +333,10 @@ pub mod fq { domain_size: i32, i: i32, ) -> Option { - let basis = srs.0.get_lagrange_basis_from_domain_size(domain_size as usize); + if !(srs.0.lagrange_bases.contains_key(&(domain_size as usize))) { + return None; + } + let basis = srs.get_lagrange_basis_from_domain_size(domain_size as usize); Some(basis[i as usize].clone().into()) } From 93849b10d072588ff69f35c027e0eddc01107726 Mon Sep 17 00:00:00 2001 From: georgeee Date: Sun, 20 Oct 2024 14:47:08 +0000 Subject: [PATCH 20/47] Remove some unused code in snark-related modules Removes some code which was unused: * Transaction_state.verify * Blockchain_state.verify * Snarked_ledger_state.Statement.With_sok.{to_input,Checked} Code was written but never used. More importantly, implementation of `Snarked_ledger_state.Statement.With_sok.to_input` (and transitively `Transaction_state.verify`) didn't agree with `verify` function exposed by `Proof` module (which used `to_field_elements` derived from `typ` instead of `to_input`). --- .../blockchain_snark_state.ml | 20 +++----- .../blockchain_snark_state.mli | 5 -- src/lib/mina_state/snarked_ledger_state.ml | 50 ------------------- .../mina_state/snarked_ledger_state_intf.ml | 17 +------ .../transaction_snark/transaction_snark.ml | 25 ++-------- .../transaction_snark_intf.ml | 7 +-- 6 files changed, 12 insertions(+), 112 deletions(-) diff --git a/src/lib/blockchain_snark/blockchain_snark_state.ml b/src/lib/blockchain_snark/blockchain_snark_state.ml index 7ea9acaf2ff..6f1c1fa1f52 100644 --- a/src/lib/blockchain_snark/blockchain_snark_state.ml +++ b/src/lib/blockchain_snark/blockchain_snark_state.ml @@ -381,18 +381,12 @@ let%snarkydef_ step ~(logger : Logger.t) ; proof_must_verify = txn_snark_must_verify } ) -module Statement = struct - type t = Protocol_state.Value.t - - let to_field_elements (t : t) : Tick.Field.t array = - [| (Protocol_state.hashes t).state_hash |] -end - -module Statement_var = struct - type t = Protocol_state.Value.t Data_as_hash.t -end - -type tag = (Statement_var.t, Statement.t, Nat.N2.n, Nat.N1.n) Pickles.Tag.t +type tag = + ( Protocol_state.Value.t Data_as_hash.t + , Protocol_state.Value.t + , Nat.N2.n + , Nat.N1.n ) + Pickles.Tag.t let typ = Data_as_hash.typ ~hash:(fun t -> (Protocol_state.hashes t).state_hash) @@ -449,8 +443,6 @@ module type S = sig val constraint_system_digests : (string * Md5_lib.t) list Lazy.t end -let verify ts ~key = Pickles.verify (module Nat.N2) (module Statement) key ts - let constraint_system_digests ~proof_level ~constraint_constants () = let digest = Tick.R1CS_constraint_system.digest in [ ( "blockchain-step" diff --git a/src/lib/blockchain_snark/blockchain_snark_state.mli b/src/lib/blockchain_snark/blockchain_snark_state.mli index 7a8f3908b7d..c43e046ada2 100644 --- a/src/lib/blockchain_snark/blockchain_snark_state.mli +++ b/src/lib/blockchain_snark/blockchain_snark_state.mli @@ -20,11 +20,6 @@ type tag = , Nat.N1.n ) Pickles.Tag.t -val verify : - (Protocol_state.Value.t * Proof.t) list - -> key:Pickles.Verification_key.t - -> unit Or_error.t Async.Deferred.t - val check : Witness.t -> ?handler: diff --git a/src/lib/mina_state/snarked_ledger_state.ml b/src/lib/mina_state/snarked_ledger_state.ml index 55d1d460645..ac564e3fedf 100644 --- a/src/lib/mina_state/snarked_ledger_state.ml +++ b/src/lib/mina_state/snarked_ledger_state.ml @@ -383,56 +383,6 @@ module Make_str (A : Wire_types.Concrete) = struct Poly.typ Frozen_ledger_hash.typ Currency.Amount.Signed.typ Pending_coinbase.Stack.typ Fee_excess.typ Sok_message.Digest.typ Local_state.typ - - let to_input ({ sok_digest; _ } as t : t) = - let input = - let input_without_sok = to_input { t with sok_digest = () } in - Array.reduce_exn ~f:Random_oracle.Input.Chunked.append - [| Sok_message.Digest.to_input sok_digest; input_without_sok |] - in - if !top_hash_logging_enabled then - Format.eprintf - !"Generating unchecked top hash from:@.%{sexp: Tick.Field.t \ - Random_oracle.Input.Chunked.t}@." - input ; - input - - let to_field_elements t = Random_oracle.pack_input (to_input t) - - module Checked = struct - type t = var - - module Checked_without_sok = Checked - - let to_input ({ sok_digest; _ } as t : t) = - let open Tick in - let open Checked.Let_syntax in - let%bind input_without_sok = - Checked_without_sok.to_input { t with sok_digest = () } - in - let input = - Array.reduce_exn ~f:Random_oracle.Input.Chunked.append - [| Sok_message.Digest.Checked.to_input sok_digest - ; input_without_sok - |] - in - let%map () = - as_prover - As_prover.( - if !top_hash_logging_enabled then - let%map input = Random_oracle.read_typ' input in - Format.eprintf - !"Generating checked top hash from:@.%{sexp: Field.t \ - Random_oracle.Input.Chunked.t}@." - input - else return ()) - in - input - - let to_field_elements t = - let open Tick.Checked.Let_syntax in - Tick.Run.run_checked (to_input t >>| Random_oracle.Checked.pack_input) - end end let option lab = diff --git a/src/lib/mina_state/snarked_ledger_state_intf.ml b/src/lib/mina_state/snarked_ledger_state_intf.ml index 68919366c78..1344065f372 100644 --- a/src/lib/mina_state/snarked_ledger_state_intf.ml +++ b/src/lib/mina_state/snarked_ledger_state_intf.ml @@ -223,22 +223,7 @@ module type Full = sig , Local_state.Checked.t ) Poly.t - open Tick - - val typ : (var, t) Typ.t - - val to_input : t -> Field.t Random_oracle.Input.Chunked.t - - val to_field_elements : t -> Field.t array - - module Checked : sig - type t = var - - val to_input : var -> Field.Var.t Random_oracle.Input.Chunked.t Checked.t - - (* This is actually a checked function. *) - val to_field_elements : var -> Field.Var.t array - end + val typ : (var, t) Tick.Typ.t end val gen : t Quickcheck.Generator.t diff --git a/src/lib/transaction_snark/transaction_snark.ml b/src/lib/transaction_snark/transaction_snark.ml index b8966b98d4d..1e1abf87cd9 100644 --- a/src/lib/transaction_snark/transaction_snark.ml +++ b/src/lib/transaction_snark/transaction_snark.ml @@ -1854,7 +1854,7 @@ module Make_str (A : Wire_types.Concrete) = struct [ correct_coinbase_target_stack; valid_init_state ] ) ) let main ?(witness : Witness.t option) (spec : Spec.t) - ~constraint_constants (statement : Statement.With_sok.Checked.t) = + ~constraint_constants (statement : Statement.With_sok.var) = let open Impl in run_checked (dummy_constraints ()) ; let ( ! ) x = Option.value_exn x in @@ -3053,7 +3053,7 @@ module Make_str (A : Wire_types.Concrete) = struct pc: Pending_coinbase_stack_state.t *) let%snarkydef_ main ~constraint_constants - (statement : Statement.With_sok.Checked.t) = + (statement : Statement.With_sok.var) = let%bind () = dummy_constraints () in let%bind (module Shifted) = Tick.Inner_curve.Checked.Shifted.create () in let%bind t = @@ -3199,7 +3199,7 @@ module Make_str (A : Wire_types.Concrete) = struct verify_transition tock_vk _ s1 s2 pending_coinbase_stack12.source, pending_coinbase_stack12.target is true verify_transition tock_vk _ s2 s3 pending_coinbase_stack23.source, pending_coinbase_stack23.target is true *) - let%snarkydef_ main (s : Statement.With_sok.Checked.t) = + let%snarkydef_ main (s : Statement.With_sok.var) = let%bind s1, s2 = exists Typ.(Statement.With_sok.typ * Statement.With_sok.typ) @@ -3302,7 +3302,7 @@ module Make_str (A : Wire_types.Concrete) = struct open Pickles_types type tag = - ( Statement.With_sok.Checked.t + ( Statement.With_sok.var , Statement.With_sok.t , Nat.N2.n , Nat.N5.n ) @@ -3509,23 +3509,6 @@ module Make_str (A : Wire_types.Concrete) = struct } init_stack pending_coinbase_stack_state handler - let verify (ts : (t * _) list) ~key = - if - List.for_all ts ~f:(fun ({ statement; _ }, message) -> - Sok_message.Digest.equal - (Sok_message.digest message) - statement.sok_digest ) - then - Pickles.verify - (module Nat.N2) - (module Statement.With_sok) - key - (List.map ts ~f:(fun ({ statement; proof }, _) -> (statement, proof))) - else - Async.return - (Or_error.error_string - "Transaction_snark.verify: Mismatched sok_message" ) - let constraint_system_digests ~constraint_constants () = let digest = Tick.R1CS_constraint_system.digest in [ ( "transaction-merge" diff --git a/src/lib/transaction_snark/transaction_snark_intf.ml b/src/lib/transaction_snark/transaction_snark_intf.ml index 68e7637176e..7a426286925 100644 --- a/src/lib/transaction_snark/transaction_snark_intf.ml +++ b/src/lib/transaction_snark/transaction_snark_intf.ml @@ -34,17 +34,12 @@ module type Full = sig open Pickles_types type tag = - ( Statement.With_sok.Checked.t + ( Statement.With_sok.var , Statement.With_sok.t , Nat.N2.n , Nat.N5.n ) Pickles.Tag.t - val verify : - (t * Sok_message.t) list - -> key:Pickles.Verification_key.t - -> unit Or_error.t Async.Deferred.t - module Verification : sig module type S = sig val tag : tag From 25c40e3196b127a562a2f9da170569a94c7ad1d2 Mon Sep 17 00:00:00 2001 From: George Agapov Date: Sun, 20 Oct 2024 17:39:02 +0200 Subject: [PATCH 21/47] Reduce repetitions in TestBitswapSmall Existing test is really long to execute. On some machines it exceeds the limit of 60 minutes. With the reduced number of repetitions, testing is still performed, but it's ensured that libp2p unit tests take less than 30m even on slower machines. In future we may look deeper in the test and instead of making lesser amount of repetitions either optimize the software or restructure the test. Tests concerns only bitswap, i.e. functionality that is not activated in mainnet's Mina build. --- src/app/libp2p_helper/src/libp2p_helper/bitswap_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/app/libp2p_helper/src/libp2p_helper/bitswap_test.go b/src/app/libp2p_helper/src/libp2p_helper/bitswap_test.go index 93b25d85cbc..bc2ce793dd7 100644 --- a/src/app/libp2p_helper/src/libp2p_helper/bitswap_test.go +++ b/src/app/libp2p_helper/src/libp2p_helper/bitswap_test.go @@ -549,7 +549,7 @@ func TestBitswapSmoke(t *testing.T) { } func TestBitswapSmall(t *testing.T) { - testBitswap(t, 20, 100, 5, 1<<16, false) + testBitswap(t, 20, 10, 5, 1<<16, false) } func TestBitswapQC(t *testing.T) { From cf2416201d28619d7332b01f92509db9b93fc88f Mon Sep 17 00:00:00 2001 From: georgeee Date: Sun, 20 Oct 2024 18:34:08 +0000 Subject: [PATCH 22/47] Re-introduce {Transaction,Blockchain}_snark.verify Re-implement the verify functions for both `Transaction_snark` and `Blockhain_snark` modules in the way that makes sure same code is used between new `verify` functions and `verify` provided by respective `Proof` modules. --- .../blockchain_snark_state.ml | 24 ++++++++++++----- .../blockchain_snark_state.mli | 5 ++++ src/lib/mina_state/snarked_ledger_state.ml | 4 +++ .../mina_state/snarked_ledger_state_intf.ml | 2 ++ .../transaction_snark/transaction_snark.ml | 27 ++++++++++--------- .../transaction_snark_intf.ml | 5 ++++ 6 files changed, 49 insertions(+), 18 deletions(-) diff --git a/src/lib/blockchain_snark/blockchain_snark_state.ml b/src/lib/blockchain_snark/blockchain_snark_state.ml index 6f1c1fa1f52..51cdc3e9063 100644 --- a/src/lib/blockchain_snark/blockchain_snark_state.ml +++ b/src/lib/blockchain_snark/blockchain_snark_state.ml @@ -381,22 +381,31 @@ let%snarkydef_ step ~(logger : Logger.t) ; proof_must_verify = txn_snark_must_verify } ) +module Statement = struct + type t = Protocol_state.Value.t + + let typ = + Data_as_hash.typ ~hash:(fun t -> (Protocol_state.hashes t).state_hash) + + let to_field_elements = + let (Typ { value_to_fields; _ }) = typ in + Fn.compose fst value_to_fields +end + type tag = ( Protocol_state.Value.t Data_as_hash.t - , Protocol_state.Value.t + , Statement.t , Nat.N2.n , Nat.N1.n ) Pickles.Tag.t -let typ = Data_as_hash.typ ~hash:(fun t -> (Protocol_state.hashes t).state_hash) - let check w ?handler ~proof_level ~constraint_constants new_state_hash : unit Or_error.t = let open Tick in check (Fn.flip handle (wrap_handler handler w) (fun () -> let%bind curr = - exists typ ~compute:(As_prover.return new_state_hash) + exists Statement.typ ~compute:(As_prover.return new_state_hash) in step ~proof_level ~constraint_constants ~logger:(Logger.create ()) curr ) ) @@ -443,6 +452,8 @@ module type S = sig val constraint_system_digests : (string * Md5_lib.t) list Lazy.t end +let verify ts ~key = Pickles.verify (module Nat.N2) (module Statement) key ts + let constraint_system_digests ~proof_level ~constraint_constants () = let digest = Tick.R1CS_constraint_system.digest in [ ( "blockchain-step" @@ -455,7 +466,7 @@ let constraint_system_digests ~proof_level ~constraint_constants () = in () in - Tick.constraint_system ~input_typ:typ + Tick.constraint_system ~input_typ:Statement.typ ~return_typ:(Snarky_backendless.Typ.unit ()) main ) ) ] @@ -470,7 +481,8 @@ end) : S = struct open T let tag, cache_handle, p, Pickles.Provers.[ step ] = - Pickles.compile () ~cache:Cache_dir.cache ~public_input:(Input typ) + Pickles.compile () ~cache:Cache_dir.cache + ~public_input:(Input Statement.typ) ~override_wrap_domain:Pickles_base.Proofs_verified.N1 ~auxiliary_typ:Typ.unit ~branches:(module Nat.N1) diff --git a/src/lib/blockchain_snark/blockchain_snark_state.mli b/src/lib/blockchain_snark/blockchain_snark_state.mli index c43e046ada2..7a8f3908b7d 100644 --- a/src/lib/blockchain_snark/blockchain_snark_state.mli +++ b/src/lib/blockchain_snark/blockchain_snark_state.mli @@ -20,6 +20,11 @@ type tag = , Nat.N1.n ) Pickles.Tag.t +val verify : + (Protocol_state.Value.t * Proof.t) list + -> key:Pickles.Verification_key.t + -> unit Or_error.t Async.Deferred.t + val check : Witness.t -> ?handler: diff --git a/src/lib/mina_state/snarked_ledger_state.ml b/src/lib/mina_state/snarked_ledger_state.ml index ac564e3fedf..5ae5e5acd07 100644 --- a/src/lib/mina_state/snarked_ledger_state.ml +++ b/src/lib/mina_state/snarked_ledger_state.ml @@ -383,6 +383,10 @@ module Make_str (A : Wire_types.Concrete) = struct Poly.typ Frozen_ledger_hash.typ Currency.Amount.Signed.typ Pending_coinbase.Stack.typ Fee_excess.typ Sok_message.Digest.typ Local_state.typ + + let to_field_elements = + let (Typ { value_to_fields; _ }) = typ in + Fn.compose fst value_to_fields end let option lab = diff --git a/src/lib/mina_state/snarked_ledger_state_intf.ml b/src/lib/mina_state/snarked_ledger_state_intf.ml index 1344065f372..39b6150823f 100644 --- a/src/lib/mina_state/snarked_ledger_state_intf.ml +++ b/src/lib/mina_state/snarked_ledger_state_intf.ml @@ -224,6 +224,8 @@ module type Full = sig Poly.t val typ : (var, t) Tick.Typ.t + + val to_field_elements : t -> Tick.Field.t array end val gen : t Quickcheck.Generator.t diff --git a/src/lib/transaction_snark/transaction_snark.ml b/src/lib/transaction_snark/transaction_snark.ml index 1e1abf87cd9..a7822ea0cf7 100644 --- a/src/lib/transaction_snark/transaction_snark.ml +++ b/src/lib/transaction_snark/transaction_snark.ml @@ -3509,6 +3509,20 @@ module Make_str (A : Wire_types.Concrete) = struct } init_stack pending_coinbase_stack_state handler + let verify_impl ~f ts = + if + List.for_all ts ~f:(fun (p, m) -> + Sok_message.Digest.equal (Sok_message.digest m) p.statement.sok_digest ) + then f (List.map ts ~f:(fun ({ statement; proof }, _) -> (statement, proof))) + else + Async.return + (Or_error.error_string + "Transaction_snark.verify: Mismatched sok_message" ) + + let verify ~key = + verify_impl + ~f:(Pickles.verify (module Nat.N2) (module Statement.With_sok) key) + let constraint_system_digests ~constraint_constants () = let digest = Tick.R1CS_constraint_system.digest in [ ( "transaction-merge" @@ -3947,18 +3961,7 @@ module Make_str (A : Wire_types.Concrete) = struct let verify_against_digest { statement; proof } = Proof.verify [ (statement, proof) ] - let verify ts = - if - List.for_all ts ~f:(fun (p, m) -> - Sok_message.Digest.equal (Sok_message.digest m) - p.statement.sok_digest ) - then - Proof.verify - (List.map ts ~f:(fun ({ statement; proof }, _) -> (statement, proof))) - else - Async.return - (Or_error.error_string - "Transaction_snark.verify: Mismatched sok_message" ) + let verify = verify_impl ~f:Proof.verify let first_account_update (witness : Transaction_witness.Zkapp_command_segment_witness.t) = diff --git a/src/lib/transaction_snark/transaction_snark_intf.ml b/src/lib/transaction_snark/transaction_snark_intf.ml index 7a426286925..7625c08890c 100644 --- a/src/lib/transaction_snark/transaction_snark_intf.ml +++ b/src/lib/transaction_snark/transaction_snark_intf.ml @@ -40,6 +40,11 @@ module type Full = sig , Nat.N5.n ) Pickles.Tag.t + val verify : + key:Pickles.Verification_key.t + -> (t * Sok_message.t) list + -> unit Or_error.t Async.Deferred.t + module Verification : sig module type S = sig val tag : tag From b8e0ada675d3c7ef0820dd7bd6d3025b981dbe64 Mon Sep 17 00:00:00 2001 From: dkijania Date: Mon, 21 Oct 2024 23:31:30 +0200 Subject: [PATCH 23/47] fix verify promoted docker check --- buildkite/scripts/run_verify_promoted_build_job.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/scripts/run_verify_promoted_build_job.sh b/buildkite/scripts/run_verify_promoted_build_job.sh index da2a3789d3f..713749f642f 100755 --- a/buildkite/scripts/run_verify_promoted_build_job.sh +++ b/buildkite/scripts/run_verify_promoted_build_job.sh @@ -107,4 +107,4 @@ if [[ "${REMOVE_PROFILE_FROM_NAME}" -eq 0 ]]; then else REMOVE_PROFILE_FROM_NAME="True" fi -echo $PROMOTE_PACKAGE_DHALL_DEF'.verify_artifacts '"$DHALL_DEBIANS"' '"$DHALL_DOCKERS"' "'"${NEW_VERSION}"'" '$PROFILES_DHALL_DEF'.Type.'"${PROFILE}"' '$NETWORK_DHALL_DEF'.Type.'"${NETWORK}"' '"${DHALL_CODENAMES}"' '$DEBIAN_CHANNEL_DHALL_DEF'.Type.'"${TO_CHANNEL}"' "'"${TAG}"'" '${REMOVE_PROFILE_FROM_NAME}' '${DHALL_PUBLISH}' ' | dhall-to-yaml --quoted +echo $PROMOTE_PACKAGE_DHALL_DEF'.verify_artifacts '"$DHALL_DEBIANS"' '"$DHALL_DOCKERS"' "'"${NEW_VERSION}"'" '$PROFILES_DHALL_DEF'.Type.'"${PROFILE}"' '$NETWORK_DHALL_DEF'.Type.'"${NETWORK}"' '"${DHALL_CODENAMES}"' '$DEBIAN_CHANNEL_DHALL_DEF'.Type.'"${TO_CHANNEL}"' "'"${NEW_VERSION}"'" '${REMOVE_PROFILE_FROM_NAME}' '${DHALL_PUBLISH}' ' | dhall-to-yaml --quoted From d556cd9ffb5fff7faec41ad267cf17276c2e6ba6 Mon Sep 17 00:00:00 2001 From: Mikhail Volkhov Date: Tue, 22 Oct 2024 14:06:07 +0100 Subject: [PATCH 24/47] Remove Mutex lagrange cache SRS related unsafes from bindings --- .../stubs/src/lagrange_basis.rs | 10 ++-- .../stubs/src/pasta_fp_plonk_index.rs | 7 +-- .../stubs/src/pasta_fp_plonk_proof.rs | 52 +++++++++---------- .../src/pasta_fp_plonk_verifier_index.rs | 10 ++-- .../stubs/src/pasta_fq_plonk_index.rs | 7 +-- .../stubs/src/pasta_fq_plonk_proof.rs | 8 +-- .../src/pasta_fq_plonk_verifier_index.rs | 10 ++-- .../crypto/kimchi_bindings/stubs/src/srs.rs | 18 ++----- 8 files changed, 52 insertions(+), 70 deletions(-) diff --git a/src/lib/crypto/kimchi_bindings/stubs/src/lagrange_basis.rs b/src/lib/crypto/kimchi_bindings/stubs/src/lagrange_basis.rs index 818edc09776..a132e142621 100644 --- a/src/lib/crypto/kimchi_bindings/stubs/src/lagrange_basis.rs +++ b/src/lib/crypto/kimchi_bindings/stubs/src/lagrange_basis.rs @@ -6,11 +6,11 @@ use poly_commitment::{commitment::CommitmentCurve, srs::SRS}; use std::env; pub trait WithLagrangeBasis { - fn with_lagrange_basis(&mut self, domain: D); + fn with_lagrange_basis(&self, domain: D); } impl WithLagrangeBasis for SRS { - fn with_lagrange_basis(&mut self, domain: D<::ScalarField>) { + fn with_lagrange_basis(&self, domain: D<::ScalarField>) { match env::var("LAGRANGE_CACHE_DIR") { Ok(_) => add_lagrange_basis_with_cache(self, domain, cache::get_vesta_file_cache()), Err(_) => { @@ -21,7 +21,7 @@ impl WithLagrangeBasis for SRS { } impl WithLagrangeBasis for SRS { - fn with_lagrange_basis(&mut self, domain: D<::ScalarField>) { + fn with_lagrange_basis(&self, domain: D<::ScalarField>) { match env::var("LAGRANGE_CACHE_DIR") { Ok(_) => add_lagrange_basis_with_cache(self, domain, cache::get_pallas_file_cache()), Err(_) => { @@ -32,7 +32,7 @@ impl WithLagrangeBasis for SRS { } fn add_lagrange_basis_with_cache>( - srs: &mut SRS, + srs: &SRS, domain: D, cache: &C, ) { @@ -41,7 +41,7 @@ fn add_lagrange_basis_with_cache>( return; } if let Some(basis) = cache.load_lagrange_basis_from_cache(srs.g.len(), &domain) { - srs.lagrange_bases.get_or_generate(n, || { basis }); + srs.lagrange_bases.get_or_generate(n, || basis); return; } else { let basis = srs.get_lagrange_basis(domain); diff --git a/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fp_plonk_index.rs b/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fp_plonk_index.rs index f750f8769c0..acd6f90a390 100644 --- a/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fp_plonk_index.rs +++ b/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fp_plonk_index.rs @@ -87,12 +87,7 @@ pub fn caml_pasta_fp_plonk_index_create( // endo let (endo_q, _endo_r) = poly_commitment::srs::endos::(); - // Unsafe if we are in a multi-core ocaml - { - let ptr: &mut poly_commitment::srs::SRS = - unsafe { &mut *(std::sync::Arc::as_ptr(&srs.0) as *mut _) }; - ptr.with_lagrange_basis(cs.domain.d1); - } + srs.0.with_lagrange_basis(cs.domain.d1); // create index let mut index = ProverIndex::>::create(cs, endo_q, srs.clone()); diff --git a/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fp_plonk_proof.rs b/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fp_plonk_proof.rs index dd15d0e1303..23816ccf39c 100644 --- a/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fp_plonk_proof.rs +++ b/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fp_plonk_proof.rs @@ -46,10 +46,13 @@ pub fn caml_pasta_fp_plonk_proof_create( prev_sgs: Vec, ) -> Result, ocaml::Error> { { - let ptr: &mut poly_commitment::srs::SRS = - unsafe { &mut *(std::sync::Arc::as_ptr(&index.as_ref().0.srs) as *mut _) }; - ptr.with_lagrange_basis(index.as_ref().0.cs.domain.d1); + index + .as_ref() + .0 + .srs + .with_lagrange_basis(index.as_ref().0.cs.domain.d1); } + let prev = if prev_challenges.is_empty() { Vec::new() } else { @@ -112,9 +115,11 @@ pub fn caml_pasta_fp_plonk_proof_create_and_verify( prev_sgs: Vec, ) -> Result, ocaml::Error> { { - let ptr: &mut poly_commitment::srs::SRS = - unsafe { &mut *(std::sync::Arc::as_ptr(&index.as_ref().0.srs) as *mut _) }; - ptr.with_lagrange_basis(index.as_ref().0.cs.domain.d1); + index + .as_ref() + .0 + .srs + .with_lagrange_basis(index.as_ref().0.cs.domain.d1); } let prev = if prev_challenges.is_empty() { Vec::new() @@ -199,7 +204,7 @@ pub fn caml_pasta_fp_plonk_proof_example_with_lookup( polynomial::COLUMNS, wires::Wire, }; - use poly_commitment::srs::{endos, SRS}; + use poly_commitment::srs::endos; let num_gates = 1000; let num_tables: usize = 5; @@ -276,8 +281,7 @@ pub fn caml_pasta_fp_plonk_proof_example_with_lookup( .build() .unwrap(); - let ptr: &mut SRS = unsafe { &mut *(std::sync::Arc::as_ptr(&srs.0) as *mut _) }; - ptr.with_lagrange_basis(cs.domain.d1); + srs.0.with_lagrange_basis(cs.domain.d1); let (endo_q, _endo_r) = endos::(); let index = ProverIndex::>::create(cs, endo_q, srs.0); @@ -321,7 +325,7 @@ pub fn caml_pasta_fp_plonk_proof_example_with_foreign_field_mul( use num_bigint::BigUint; use num_bigint::RandBigInt; use o1_utils::{foreign_field::BigUintForeignFieldHelpers, FieldHelpers}; - use poly_commitment::srs::{endos, SRS}; + use poly_commitment::srs::endos; use rand::{rngs::StdRng, SeedableRng}; let foreign_field_modulus = Fq::modulus_biguint(); @@ -441,8 +445,7 @@ pub fn caml_pasta_fp_plonk_proof_example_with_foreign_field_mul( // Create constraint system let cs = ConstraintSystem::::create(gates).build().unwrap(); - let ptr: &mut SRS = unsafe { &mut *(std::sync::Arc::as_ptr(&srs.0) as *mut _) }; - ptr.with_lagrange_basis(cs.domain.d1); + srs.0.with_lagrange_basis(cs.domain.d1); let (endo_q, _endo_r) = endos::(); let index = ProverIndex::>::create(cs, endo_q, srs.0); @@ -478,7 +481,7 @@ pub fn caml_pasta_fp_plonk_proof_example_with_range_check( use num_bigint::BigUint; use num_bigint::RandBigInt; use o1_utils::{foreign_field::BigUintForeignFieldHelpers, BigUintFieldHelpers}; - use poly_commitment::srs::{endos, SRS}; + use poly_commitment::srs::endos; use rand::{rngs::StdRng, SeedableRng}; let rng = &mut StdRng::from_seed([255u8; 32]); @@ -508,8 +511,7 @@ pub fn caml_pasta_fp_plonk_proof_example_with_range_check( // Create constraint system let cs = ConstraintSystem::::create(gates).build().unwrap(); - let ptr: &mut SRS = unsafe { &mut *(std::sync::Arc::as_ptr(&srs.0) as *mut _) }; - ptr.with_lagrange_basis(cs.domain.d1); + srs.0.with_lagrange_basis(cs.domain.d1); let (endo_q, _endo_r) = endos::(); let index = ProverIndex::>::create(cs, endo_q, srs.0); @@ -546,7 +548,7 @@ pub fn caml_pasta_fp_plonk_proof_example_with_range_check0( polynomials::{generic::GenericGateSpec, range_check}, wires::Wire, }; - use poly_commitment::srs::{endos, SRS}; + use poly_commitment::srs::endos; let gates = { // Public input row with value 0 @@ -581,8 +583,7 @@ pub fn caml_pasta_fp_plonk_proof_example_with_range_check0( // not sure if theres a smarter way instead of the double unwrap, but should be fine in the test let cs = ConstraintSystem::::create(gates).build().unwrap(); - let ptr: &mut SRS = unsafe { &mut *(std::sync::Arc::as_ptr(&srs.0) as *mut _) }; - ptr.with_lagrange_basis(cs.domain.d1); + srs.0.with_lagrange_basis(cs.domain.d1); let (endo_q, _endo_r) = endos::(); let index = ProverIndex::>::create(cs, endo_q, srs.0); @@ -625,7 +626,7 @@ pub fn caml_pasta_fp_plonk_proof_example_with_ffadd( wires::Wire, }; use num_bigint::BigUint; - use poly_commitment::srs::{endos, SRS}; + use poly_commitment::srs::endos; // Includes a row to store value 1 let num_public_inputs = 1; @@ -706,8 +707,7 @@ pub fn caml_pasta_fp_plonk_proof_example_with_ffadd( .build() .unwrap(); - let ptr: &mut SRS = unsafe { &mut *(std::sync::Arc::as_ptr(&srs.0) as *mut _) }; - ptr.with_lagrange_basis(cs.domain.d1); + srs.0.with_lagrange_basis(cs.domain.d1); let (endo_q, _endo_r) = endos::(); let index = ProverIndex::>::create(cs, endo_q, srs.0); @@ -747,7 +747,7 @@ pub fn caml_pasta_fp_plonk_proof_example_with_xor( polynomials::{generic::GenericGateSpec, xor}, wires::Wire, }; - use poly_commitment::srs::{endos, SRS}; + use poly_commitment::srs::endos; let num_public_inputs = 2; @@ -795,8 +795,7 @@ pub fn caml_pasta_fp_plonk_proof_example_with_xor( .build() .unwrap(); - let ptr: &mut SRS = unsafe { &mut *(std::sync::Arc::as_ptr(&srs.0) as *mut _) }; - ptr.with_lagrange_basis(cs.domain.d1); + srs.0.with_lagrange_basis(cs.domain.d1); let (endo_q, _endo_r) = endos::(); let index = ProverIndex::>::create(cs, endo_q, srs.0); @@ -839,7 +838,7 @@ pub fn caml_pasta_fp_plonk_proof_example_with_rot( }, wires::Wire, }; - use poly_commitment::srs::{endos, SRS}; + use poly_commitment::srs::endos; // Includes the actual input of the rotation and a row with the zero value let num_public_inputs = 2; @@ -889,8 +888,7 @@ pub fn caml_pasta_fp_plonk_proof_example_with_rot( .build() .unwrap(); - let ptr: &mut SRS = unsafe { &mut *(std::sync::Arc::as_ptr(&srs.0) as *mut _) }; - ptr.with_lagrange_basis(cs.domain.d1); + srs.0.with_lagrange_basis(cs.domain.d1); let (endo_q, _endo_r) = endos::(); let index = ProverIndex::>::create(cs, endo_q, srs.0); diff --git a/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fp_plonk_verifier_index.rs b/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fp_plonk_verifier_index.rs index 6f05f08773b..ed950841f3f 100644 --- a/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fp_plonk_verifier_index.rs +++ b/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fp_plonk_verifier_index.rs @@ -221,11 +221,11 @@ pub fn caml_pasta_fp_plonk_verifier_index_write( pub fn caml_pasta_fp_plonk_verifier_index_create( index: CamlPastaFpPlonkIndexPtr, ) -> CamlPastaFpPlonkVerifierIndex { - { - let ptr: &mut poly_commitment::srs::SRS = - unsafe { &mut *(std::sync::Arc::as_ptr(&index.as_ref().0.srs) as *mut _) }; - ptr.with_lagrange_basis(index.as_ref().0.cs.domain.d1); - } + index + .as_ref() + .0 + .srs + .with_lagrange_basis(index.as_ref().0.cs.domain.d1); let verifier_index = index.as_ref().0.verifier_index(); verifier_index.into() } diff --git a/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fq_plonk_index.rs b/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fq_plonk_index.rs index c1f6f50f9e9..fc360eb168b 100644 --- a/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fq_plonk_index.rs +++ b/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fq_plonk_index.rs @@ -86,12 +86,7 @@ pub fn caml_pasta_fq_plonk_index_create( // endo let (endo_q, _endo_r) = poly_commitment::srs::endos::(); - // Unsafe if we are in a multi-core ocaml - { - let ptr: &mut poly_commitment::srs::SRS = - unsafe { &mut *(std::sync::Arc::as_ptr(&srs.0) as *mut _) }; - ptr.with_lagrange_basis(cs.domain.d1); - } + srs.0.with_lagrange_basis(cs.domain.d1); // create index let mut index = ProverIndex::>::create(cs, endo_q, srs.clone()); diff --git a/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fq_plonk_proof.rs b/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fq_plonk_proof.rs index 607d28691ae..14be39c2269 100644 --- a/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fq_plonk_proof.rs +++ b/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fq_plonk_proof.rs @@ -41,9 +41,11 @@ pub fn caml_pasta_fq_plonk_proof_create( prev_sgs: Vec, ) -> Result, ocaml::Error> { { - let ptr: &mut poly_commitment::srs::SRS = - unsafe { &mut *(std::sync::Arc::as_ptr(&index.as_ref().0.srs) as *mut _) }; - ptr.with_lagrange_basis(index.as_ref().0.cs.domain.d1); + index + .as_ref() + .0 + .srs + .with_lagrange_basis(index.as_ref().0.cs.domain.d1); } let prev = if prev_challenges.is_empty() { Vec::new() diff --git a/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fq_plonk_verifier_index.rs b/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fq_plonk_verifier_index.rs index 7b81e74a7a8..5251923a42d 100644 --- a/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fq_plonk_verifier_index.rs +++ b/src/lib/crypto/kimchi_bindings/stubs/src/pasta_fq_plonk_verifier_index.rs @@ -220,11 +220,11 @@ pub fn caml_pasta_fq_plonk_verifier_index_write( pub fn caml_pasta_fq_plonk_verifier_index_create( index: CamlPastaFqPlonkIndexPtr, ) -> CamlPastaFqPlonkVerifierIndex { - { - let ptr: &mut poly_commitment::srs::SRS = - unsafe { &mut *(std::sync::Arc::as_ptr(&index.as_ref().0.srs) as *mut _) }; - ptr.with_lagrange_basis(index.as_ref().0.cs.domain.d1); - } + index + .as_ref() + .0 + .srs + .with_lagrange_basis(index.as_ref().0.cs.domain.d1); let verifier_index = index.as_ref().0.verifier_index(); verifier_index.into() } diff --git a/src/lib/crypto/kimchi_bindings/stubs/src/srs.rs b/src/lib/crypto/kimchi_bindings/stubs/src/srs.rs index e32617aeb6e..2a96df53ba9 100644 --- a/src/lib/crypto/kimchi_bindings/stubs/src/srs.rs +++ b/src/lib/crypto/kimchi_bindings/stubs/src/srs.rs @@ -1,4 +1,4 @@ -use crate::WithLagrangeBasis; +use crate::lagrange_basis::WithLagrangeBasis; use ark_poly::DenseUVPolynomial; use ark_poly::{univariate::DensePolynomial, EvaluationDomain, Evaluations}; use paste::paste; @@ -84,15 +84,9 @@ macro_rules! impl_srs { .err() .unwrap() })?; - - { - // We're single-threaded, so it's safe to grab this pointer as mutable. - // Do not try this at home. - let srs = unsafe { &mut *((&**srs as *const SRS<$G>) as *mut SRS<$G>) as &mut SRS<$G> }; - srs.with_lagrange_basis(x_domain); - } - - Ok(srs.get_lagrange_basis(x_domain)[i as usize].clone().into()) + srs.with_lagrange_basis(x_domain); + let vec_polycomm = srs.get_lagrange_basis_from_domain_size(domain_size as usize); + Ok(vec_polycomm[i as usize].clone().into()) } #[ocaml_gen::func] @@ -101,10 +95,8 @@ macro_rules! impl_srs { srs: $name, log2_size: ocaml::Int, ) { - let ptr: &mut poly_commitment::srs::SRS<$G> = - unsafe { &mut *(std::sync::Arc::as_ptr(&srs) as *mut _) }; let domain = EvaluationDomain::<$F>::new(1 << (log2_size as usize)).expect("invalid domain size"); - ptr.with_lagrange_basis(domain); + srs.with_lagrange_basis(domain); } #[ocaml_gen::func] From f5c89f321051b5dec874d6f0bc41bc8a146063d7 Mon Sep 17 00:00:00 2001 From: Mikhail Volkhov Date: Mon, 23 Sep 2024 15:46:16 +0100 Subject: [PATCH 25/47] [#16112] Improve lagrange_commitments_whole_domain --- .../kimchi_backend/pasta/pallas_based_plonk.ml | 7 ++++--- .../crypto/kimchi_backend/pasta/vesta_based_plonk.ml | 4 ++-- .../crypto/kimchi_bindings/stubs/kimchi_bindings.ml | 12 ++++++++++++ src/lib/crypto/kimchi_bindings/stubs/src/main.rs | 2 ++ src/lib/crypto/kimchi_bindings/stubs/src/srs.rs | 12 ++++++++++++ 5 files changed, 32 insertions(+), 5 deletions(-) diff --git a/src/lib/crypto/kimchi_backend/pasta/pallas_based_plonk.ml b/src/lib/crypto/kimchi_backend/pasta/pallas_based_plonk.ml index cf747c75755..3c0fc37c3fb 100644 --- a/src/lib/crypto/kimchi_backend/pasta/pallas_based_plonk.ml +++ b/src/lib/crypto/kimchi_backend/pasta/pallas_based_plonk.ml @@ -34,10 +34,11 @@ end module R1CS_constraint_system = Kimchi_pasta_constraint_system.Pallas_constraint_system -let lagrange srs domain_log2 : _ Kimchi_types.poly_comm array = +let lagrange (srs : Kimchi_bindings.Protocol.SRS.Fq.t) domain_log2 : + _ Kimchi_types.poly_comm array = let domain_size = Int.pow 2 domain_log2 in - Array.init domain_size ~f:(fun i -> - Kimchi_bindings.Protocol.SRS.Fq.lagrange_commitment srs domain_size i ) + Kimchi_bindings.Protocol.SRS.Fq.lagrange_commitments_whole_domain srs + domain_size let with_lagrange f (vk : Verification_key.t) = f (lagrange vk.srs vk.domain.log_size_of_group) vk diff --git a/src/lib/crypto/kimchi_backend/pasta/vesta_based_plonk.ml b/src/lib/crypto/kimchi_backend/pasta/vesta_based_plonk.ml index 4321b8963eb..1cc6717a270 100644 --- a/src/lib/crypto/kimchi_backend/pasta/vesta_based_plonk.ml +++ b/src/lib/crypto/kimchi_backend/pasta/vesta_based_plonk.ml @@ -35,8 +35,8 @@ module R1CS_constraint_system = let lagrange srs domain_log2 : _ Kimchi_types.poly_comm array = let domain_size = Int.pow 2 domain_log2 in - Array.init domain_size ~f:(fun i -> - Kimchi_bindings.Protocol.SRS.Fp.lagrange_commitment srs domain_size i ) + Kimchi_bindings.Protocol.SRS.Fp.lagrange_commitments_whole_domain srs + domain_size let with_lagrange f (vk : Verification_key.t) = f (lagrange vk.srs vk.domain.log_size_of_group) vk diff --git a/src/lib/crypto/kimchi_bindings/stubs/kimchi_bindings.ml b/src/lib/crypto/kimchi_bindings/stubs/kimchi_bindings.ml index 07a4d89b681..7d2ebb242f8 100644 --- a/src/lib/crypto/kimchi_bindings/stubs/kimchi_bindings.ml +++ b/src/lib/crypto/kimchi_bindings/stubs/kimchi_bindings.ml @@ -107,6 +107,12 @@ module Protocol = struct -> Pasta_bindings.Fq.t Kimchi_types.or_infinity Kimchi_types.poly_comm = "caml_fp_srs_lagrange_commitment" + external lagrange_commitments_whole_domain : + t + -> int + -> Pasta_bindings.Fq.t Kimchi_types.or_infinity Kimchi_types.poly_comm + array = "caml_fp_srs_lagrange_commitments_whole_domain" + external add_lagrange_basis : t -> int -> unit = "caml_fp_srs_add_lagrange_basis" @@ -156,6 +162,12 @@ module Protocol = struct -> Pasta_bindings.Fp.t Kimchi_types.or_infinity Kimchi_types.poly_comm = "caml_fq_srs_lagrange_commitment" + external lagrange_commitments_whole_domain : + t + -> int + -> Pasta_bindings.Fp.t Kimchi_types.or_infinity Kimchi_types.poly_comm + array = "caml_fq_srs_lagrange_commitments_whole_domain" + external add_lagrange_basis : t -> int -> unit = "caml_fq_srs_add_lagrange_basis" diff --git a/src/lib/crypto/kimchi_bindings/stubs/src/main.rs b/src/lib/crypto/kimchi_bindings/stubs/src/main.rs index 5b74b542a3f..a3421e345b6 100644 --- a/src/lib/crypto/kimchi_bindings/stubs/src/main.rs +++ b/src/lib/crypto/kimchi_bindings/stubs/src/main.rs @@ -363,6 +363,7 @@ fn generate_kimchi_bindings(mut w: impl std::io::Write, env: &mut Env) { decl_func!(w, env, caml_fp_srs_write => "write"); decl_func!(w, env, caml_fp_srs_read => "read"); decl_func!(w, env, caml_fp_srs_lagrange_commitment => "lagrange_commitment"); + decl_func!(w, env, caml_fp_srs_lagrange_commitments_whole_domain => "lagrange_commitments_whole_domain"); decl_func!(w, env, caml_fp_srs_add_lagrange_basis=> "add_lagrange_basis"); decl_func!(w, env, caml_fp_srs_commit_evaluations => "commit_evaluations"); decl_func!(w, env, caml_fp_srs_b_poly_commitment => "b_poly_commitment"); @@ -378,6 +379,7 @@ fn generate_kimchi_bindings(mut w: impl std::io::Write, env: &mut Env) { decl_func!(w, env, caml_fq_srs_write => "write"); decl_func!(w, env, caml_fq_srs_read => "read"); decl_func!(w, env, caml_fq_srs_lagrange_commitment => "lagrange_commitment"); + decl_func!(w, env, caml_fq_srs_lagrange_commitments_whole_domain => "lagrange_commitments_whole_domain"); decl_func!(w, env, caml_fq_srs_add_lagrange_basis=> "add_lagrange_basis"); decl_func!(w, env, caml_fq_srs_commit_evaluations => "commit_evaluations"); decl_func!(w, env, caml_fq_srs_b_poly_commitment => "b_poly_commitment"); diff --git a/src/lib/crypto/kimchi_bindings/stubs/src/srs.rs b/src/lib/crypto/kimchi_bindings/stubs/src/srs.rs index 2a96df53ba9..6269de0aa9d 100644 --- a/src/lib/crypto/kimchi_bindings/stubs/src/srs.rs +++ b/src/lib/crypto/kimchi_bindings/stubs/src/srs.rs @@ -72,6 +72,18 @@ macro_rules! impl_srs { Ok(Some($name::new(srs))) } + #[ocaml_gen::func] + #[ocaml::func] + /// This is same as _lagrange_commitments, but returns the result for every + /// i <= domain_size. + pub fn [<$name:snake _lagrange_commitments_whole_domain>]( + srs: $name, + domain_size: ocaml::Int, + ) -> Result>, ocaml::Error> { + Ok(srs.get_lagrange_basis_from_domain_size(domain_size as usize).clone().into_iter().map(|x| x.into()).collect()) + } + + #[ocaml_gen::func] #[ocaml::func] pub fn [<$name:snake _lagrange_commitment>]( From 64dc9a37a5aafe37b209db1decaa6e36d1ab807d Mon Sep 17 00:00:00 2001 From: georgeee Date: Mon, 21 Oct 2024 12:35:48 +0000 Subject: [PATCH 26/47] Extract Call_forest from Zkapp_command This module is 600+ lines long and is easily extracted out of much larger Zkapp_command. --- src/lib/mina_base/zkapp_call_forest_base.ml | 606 +++++++++++++++++++ src/lib/mina_base/zkapp_command.ml | 625 +------------------- 2 files changed, 607 insertions(+), 624 deletions(-) create mode 100644 src/lib/mina_base/zkapp_call_forest_base.ml diff --git a/src/lib/mina_base/zkapp_call_forest_base.ml b/src/lib/mina_base/zkapp_call_forest_base.ml new file mode 100644 index 00000000000..7bf01f807bc --- /dev/null +++ b/src/lib/mina_base/zkapp_call_forest_base.ml @@ -0,0 +1,606 @@ +open Core_kernel + +let empty = Outside_hash_image.t + +module Tree = struct + [%%versioned + module Stable = struct + module V1 = struct + type ('account_update, 'account_update_digest, 'digest) t = + ( 'account_update + , 'account_update_digest + , 'digest ) + Mina_wire_types.Mina_base.Zkapp_command.Call_forest.Tree.V1.t = + { account_update : 'account_update + ; account_update_digest : 'account_update_digest + ; calls : + ( ('account_update, 'account_update_digest, 'digest) t + , 'digest ) + With_stack_hash.Stable.V1.t + list + } + [@@deriving sexp, compare, equal, hash, yojson] + + let to_latest = Fn.id + end + end] + + let rec fold_forest (ts : (_ t, _) With_stack_hash.t list) ~f ~init = + List.fold ts ~init ~f:(fun acc { elt; stack_hash = _ } -> + fold elt ~init:acc ~f ) + + and fold { account_update; calls; account_update_digest = _ } ~f ~init = + fold_forest calls ~f ~init:(f init account_update) + + let rec fold_forest2_exn (ts1 : (_ t, _) With_stack_hash.t list) + (ts2 : (_ t, _) With_stack_hash.t list) ~f ~init = + List.fold2_exn ts1 ts2 ~init + ~f:(fun acc { elt = elt1; stack_hash = _ } { elt = elt2; stack_hash = _ } + -> fold2_exn elt1 elt2 ~init:acc ~f ) + + and fold2_exn + { account_update = account_update1 + ; calls = calls1 + ; account_update_digest = _ + } + { account_update = account_update2 + ; calls = calls2 + ; account_update_digest = _ + } ~f ~init = + fold_forest2_exn calls1 calls2 ~f + ~init:(f init account_update1 account_update2) + + let iter_forest2_exn ts1 ts2 ~f = + fold_forest2_exn ts1 ts2 ~init:() ~f:(fun () p1 p2 -> f p1 p2) + + let iter2_exn ts1 ts2 ~f = + fold2_exn ts1 ts2 ~init:() ~f:(fun () p1 p2 -> f p1 p2) + + let rec mapi_with_trees' ~i (t : _ t) ~f = + let account_update = f i t.account_update t in + let l, calls = mapi_forest_with_trees' ~i:(i + 1) t.calls ~f in + ( l + , { calls; account_update; account_update_digest = t.account_update_digest } + ) + + and mapi_forest_with_trees' ~i x ~f = + let rec go i acc = function + | [] -> + (i, List.rev acc) + | t :: ts -> + let l, elt' = mapi_with_trees' ~i ~f (With_stack_hash.elt t) in + go l (With_stack_hash.map t ~f:(fun _ -> elt') :: acc) ts + in + go i [] x + + let mapi_with_trees t ~f = mapi_with_trees' ~i:0 t ~f |> snd + + let mapi_forest_with_trees t ~f = mapi_forest_with_trees' ~i:0 t ~f |> snd + + let mapi' ~i t ~f = + mapi_with_trees' ~i t ~f:(fun i account_update _ -> f i account_update) + + let mapi_forest' ~i t ~f = + mapi_forest_with_trees' ~i t ~f:(fun i account_update _ -> + f i account_update ) + + let rec deferred_mapi_with_trees' ~i (t : _ t) ~f = + let open Async_kernel.Deferred.Let_syntax in + let%bind l, calls = + deferred_mapi_forest_with_trees' ~i:(i + 1) t.calls ~f + in + let%map account_update = f i t.account_update t in + ( l + , { calls; account_update; account_update_digest = t.account_update_digest } + ) + + and deferred_mapi_forest_with_trees' ~i x ~f = + let open Async_kernel.Deferred.Let_syntax in + let rec go i acc = function + | [] -> + return (i, List.rev acc) + | t :: ts -> + let%bind l, elt' = + deferred_mapi_with_trees' ~i ~f (With_stack_hash.elt t) + in + go l (With_stack_hash.map t ~f:(fun _ -> elt') :: acc) ts + in + go i [] x + + let map_forest ~f t = mapi_forest' ~i:0 ~f:(fun _ x -> f x) t |> snd + + let mapi_forest ~f t = mapi_forest' ~i:0 ~f t |> snd + + let deferred_map_forest ~f t = + let open Async_kernel.Deferred in + deferred_mapi_forest_with_trees' ~i:0 ~f:(fun _ x -> f x) t >>| snd + + let deferred_mapi_forest ~f t = + let open Async_kernel.Deferred in + deferred_mapi_forest_with_trees' ~i:0 ~f t >>| snd + + let hash { account_update = _; calls; account_update_digest } = + let stack_hash = match calls with [] -> empty | e :: _ -> e.stack_hash in + Random_oracle.hash ~init:Hash_prefix_states.account_update_node + [| account_update_digest; stack_hash |] +end + +type ('a, 'b, 'c) tree = ('a, 'b, 'c) Tree.t + +module type Digest_intf = sig + module Account_update : sig + include Digest_intf.S + + module Checked : sig + include Digest_intf.S_checked + + val create : ?chain:Mina_signature_kind.t -> Account_update.Checked.t -> t + + val create_body : + ?chain:Mina_signature_kind.t -> Account_update.Body.Checked.t -> t + end + + include Digest_intf.S_aux with type t := t and type checked := Checked.t + + val create : ?chain:Mina_signature_kind.t -> Account_update.t -> t + + val create_body : ?chain:Mina_signature_kind.t -> Account_update.Body.t -> t + end + + module rec Forest : sig + include Digest_intf.S + + module Checked : sig + include Digest_intf.S_checked + + val empty : t + + val cons : Tree.Checked.t -> t -> t + end + + include Digest_intf.S_aux with type t := t and type checked := Checked.t + + val empty : t + + val cons : Tree.t -> Forest.t -> Forest.t + end + + and Tree : sig + include Digest_intf.S + + module Checked : sig + include Digest_intf.S_checked + + val create : + account_update:Account_update.Checked.t + -> calls:Forest.Checked.t + -> Tree.Checked.t + end + + include Digest_intf.S_aux with type t := t and type checked := Checked.t + + val create : (_, Account_update.t, Forest.t) tree -> Tree.t + end +end + +module Make_digest_sig + (T : Mina_wire_types.Mina_base.Zkapp_command.Digest_types.S) = +struct + module type S = + Digest_intf + with type Account_update.Stable.V1.t = T.Account_update.V1.t + and type Forest.Stable.V1.t = T.Forest.V1.t +end + +module Make_digest_types = struct + module Account_update = struct + [%%versioned + module Stable = struct + module V1 = struct + type t = Kimchi_backend.Pasta.Basic.Fp.Stable.V1.t + [@@deriving sexp, compare, equal, hash, yojson] + + let to_latest = Fn.id + end + end] + end + + module Forest = struct + [%%versioned + module Stable = struct + module V1 = struct + type t = Kimchi_backend.Pasta.Basic.Fp.Stable.V1.t + [@@deriving sexp, compare, equal, hash, yojson] + + let to_latest = Fn.id + end + end] + end + + module Tree = struct + [%%versioned + module Stable = struct + module V1 = struct + type t = Kimchi_backend.Pasta.Basic.Fp.Stable.V1.t + [@@deriving sexp, compare, equal, hash, yojson] + + let to_latest = Fn.id + end + end] + end +end + +module Make_digest_str + (T : Mina_wire_types.Mina_base.Zkapp_command.Digest_concrete) : + Make_digest_sig(T).S = struct + module M = struct + open Pickles.Impls.Step.Field + module Checked = Pickles.Impls.Step.Field + + let typ = typ + + let constant = constant + end + + module Account_update = struct + include Make_digest_types.Account_update + include M + + module Checked = struct + include Checked + + let create = Account_update.Checked.digest + + let create_body = Account_update.Body.Checked.digest + end + + let create : ?chain:Mina_signature_kind.t -> Account_update.t -> t = + Account_update.digest + + let create_body : ?chain:Mina_signature_kind.t -> Account_update.Body.t -> t + = + Account_update.Body.digest + end + + module Forest = struct + include Make_digest_types.Forest + include M + + module Checked = struct + include Checked + + let empty = constant empty + + let cons hash h_tl = + Random_oracle.Checked.hash ~init:Hash_prefix_states.account_update_cons + [| hash; h_tl |] + end + + let empty = empty + + let cons hash h_tl = + Random_oracle.hash ~init:Hash_prefix_states.account_update_cons + [| hash; h_tl |] + end + + module Tree = struct + include Make_digest_types.Tree + include M + + module Checked = struct + include Checked + + let create ~(account_update : Account_update.Checked.t) + ~(calls : Forest.Checked.t) = + Random_oracle.Checked.hash ~init:Hash_prefix_states.account_update_node + [| (account_update :> t); (calls :> t) |] + end + + let create ({ account_update = _; calls; account_update_digest } : _ tree) = + let stack_hash = + match calls with [] -> empty | e :: _ -> e.stack_hash + in + Random_oracle.hash ~init:Hash_prefix_states.account_update_node + [| account_update_digest; stack_hash |] + end +end + +module Digest = + Mina_wire_types.Mina_base.Zkapp_command.Digest_make + (Make_digest_sig) + (Make_digest_str) + +let fold = Tree.fold_forest + +let iteri t ~(f : int -> 'a -> unit) : unit = + let (_ : int) = fold t ~init:0 ~f:(fun acc x -> f acc x ; acc + 1) in + () + +[%%versioned +module Stable = struct + module V1 = struct + type ('account_update, 'account_update_digest, 'digest) t = + ( ('account_update, 'account_update_digest, 'digest) Tree.Stable.V1.t + , 'digest ) + With_stack_hash.Stable.V1.t + list + [@@deriving sexp, compare, equal, hash, yojson] + + let to_latest = Fn.id + end +end] + +module Shape = struct + module I = struct + type t = int + + let quickcheck_shrinker = Quickcheck.Shrinker.empty () + + let quickcheck_generator = [%quickcheck.generator: int] + + let quickcheck_observer = [%quickcheck.observer: int] + end + + type t = Node of (I.t * t) list [@@deriving quickcheck] +end + +let rec shape (t : _ t) : Shape.t = + Node (List.mapi t ~f:(fun i { elt; stack_hash = _ } -> (i, shape elt.calls))) + +let match_up (type a b) (xs : a list) (ys : (int * b) list) : (a * b) list = + let rec go i_curr xs ys = + match (xs, ys) with + | [], [] -> + [] + | x :: xs', (i, y) :: ys' -> + if i_curr = i then (x, y) :: go (i_curr + 1) xs' ys' + else if i_curr < i then go (i_curr + 1) xs' ys' + else assert false + | [], _ :: _ -> + assert false + | _ :: _, [] -> + [] + in + go 0 xs ys + +let rec mask (t : ('p, 'h1, unit) t) (Node shape : Shape.t) : ('p, 'h1, unit) t + = + List.map (match_up t shape) + ~f:(fun ({ With_stack_hash.elt = t_sub; stack_hash = () }, shape_sub) -> + { With_stack_hash.elt = { t_sub with calls = mask t_sub.calls shape_sub } + ; stack_hash = () + } ) + +let rec of_account_updates_map ~(f : 'p1 -> 'p2) + ~(account_update_depth : 'p1 -> int) (account_updates : 'p1 list) : + ('p2, unit, unit) t = + match account_updates with + | [] -> + [] + | p :: ps -> + let depth = account_update_depth p in + let children, siblings = + List.split_while ps ~f:(fun p' -> account_update_depth p' > depth) + in + { With_stack_hash.elt = + { Tree.account_update = f p + ; account_update_digest = () + ; calls = of_account_updates_map ~f ~account_update_depth children + } + ; stack_hash = () + } + :: of_account_updates_map ~f ~account_update_depth siblings + +let of_account_updates ~account_update_depth account_updates = + of_account_updates_map ~f:Fn.id ~account_update_depth account_updates + +let to_account_updates_map ~f (xs : _ t) = + let rec collect depth (xs : _ t) acc = + match xs with + | [] -> + acc + | { elt = { account_update; calls; account_update_digest = _ } + ; stack_hash = _ + } + :: xs -> + f ~depth account_update :: acc + |> collect (depth + 1) calls + |> collect depth xs + in + List.rev (collect 0 xs []) + +let to_account_updates xs = + to_account_updates_map ~f:(fun ~depth:_ account_update -> account_update) xs + +let hd_account_update (xs : _ t) = + match xs with + | [] -> + None + | { elt = { account_update; calls = _; account_update_digest = _ } + ; stack_hash = _ + } + :: _ -> + Some account_update + +let map = Tree.map_forest + +let mapi = Tree.mapi_forest + +let mapi_with_trees = Tree.mapi_forest_with_trees + +let deferred_mapi = Tree.deferred_mapi_forest + +let to_zkapp_command_with_hashes_list (xs : _ t) = + let rec collect (xs : _ t) acc = + match xs with + | [] -> + acc + | { elt = { account_update; calls; account_update_digest = _ }; stack_hash } + :: xs -> + (account_update, stack_hash) :: acc |> collect calls |> collect xs + in + List.rev (collect xs []) + +let hash_cons hash h_tl = + Random_oracle.hash ~init:Hash_prefix_states.account_update_cons + [| hash; h_tl |] + +let hash = function + | [] -> + Digest.Forest.empty + | x :: _ -> + With_stack_hash.stack_hash x + +let cons_tree tree (forest : _ t) : _ t = + { elt = tree + ; stack_hash = Digest.Forest.cons (Digest.Tree.create tree) (hash forest) + } + :: forest + +let cons_aux (type p) ~(digest_account_update : p -> _) ?(calls = []) + (account_update : p) (xs : _ t) : _ t = + let account_update_digest = digest_account_update account_update in + let tree : _ Tree.t = { account_update; account_update_digest; calls } in + cons_tree tree xs + +let cons ?calls (account_update : Account_update.t) xs = + cons_aux ~digest_account_update:Digest.Account_update.create ?calls + account_update xs + +let rec accumulate_hashes ~hash_account_update (xs : _ t) = + let go = accumulate_hashes ~hash_account_update in + match xs with + | [] -> + [] + | { elt = { account_update; calls; account_update_digest = _ } + ; stack_hash = _ + } + :: xs -> + let calls = go calls in + let xs = go xs in + let node = + { Tree.account_update + ; calls + ; account_update_digest = hash_account_update account_update + } + in + let node_hash = Digest.Tree.create node in + { elt = node; stack_hash = Digest.Forest.cons node_hash (hash xs) } :: xs + +let accumulate_hashes' (type a b) (xs : (Account_update.t, a, b) t) : + (Account_update.t, Digest.Account_update.t, Digest.Forest.t) t = + let hash_account_update (p : Account_update.t) = + Digest.Account_update.create p + in + accumulate_hashes ~hash_account_update xs + +let accumulate_hashes_predicated xs = + accumulate_hashes ~hash_account_update:Digest.Account_update.create xs + +module With_hashes_and_data = struct + [%%versioned + module Stable = struct + module V1 = struct + type 'data t = + ( Account_update.Stable.V1.t * 'data + , Digest.Account_update.Stable.V1.t + , Digest.Forest.Stable.V1.t ) + Stable.V1.t + [@@deriving sexp, compare, equal, hash, yojson] + + let to_latest = Fn.id + end + end] + + let empty = Digest.Forest.empty + + let hash_account_update ((p : Account_update.t), _) = + Digest.Account_update.create p + + let accumulate_hashes xs : _ t = accumulate_hashes ~hash_account_update xs + + let of_zkapp_command_simple_list (xs : (Account_update.Simple.t * 'a) list) : + _ t = + of_account_updates xs + ~account_update_depth:(fun ((p : Account_update.Simple.t), _) -> + p.body.call_depth ) + |> map ~f:(fun (p, x) -> (Account_update.of_simple p, x)) + |> accumulate_hashes + + let of_account_updates (xs : (Account_update.Graphql_repr.t * 'a) list) : _ t + = + of_account_updates_map + ~account_update_depth:(fun ((p : Account_update.Graphql_repr.t), _) -> + p.body.call_depth ) + ~f:(fun (p, x) -> (Account_update.of_graphql_repr p, x)) + xs + |> accumulate_hashes + + let to_account_updates (x : _ t) = to_account_updates x + + let to_zkapp_command_with_hashes_list (x : _ t) = + to_zkapp_command_with_hashes_list x + + let account_updates_hash' xs = of_account_updates xs |> hash + + let account_updates_hash xs = + List.map ~f:(fun x -> (x, ())) xs |> account_updates_hash' +end + +module With_hashes = struct + [%%versioned + module Stable = struct + module V1 = struct + type t = + ( Account_update.Stable.V1.t + , Digest.Account_update.Stable.V1.t + , Digest.Forest.Stable.V1.t ) + Stable.V1.t + [@@deriving sexp, compare, equal, hash, yojson] + + let to_latest = Fn.id + end + end] + + let empty = Digest.Forest.empty + + let hash_account_update (p : Account_update.t) = + Digest.Account_update.create p + + let accumulate_hashes xs : t = accumulate_hashes ~hash_account_update xs + + let of_zkapp_command_simple_list (xs : Account_update.Simple.t list) : t = + of_account_updates xs + ~account_update_depth:(fun (p : Account_update.Simple.t) -> + p.body.call_depth ) + |> map ~f:Account_update.of_simple + |> accumulate_hashes + + let of_account_updates (xs : Account_update.Graphql_repr.t list) : t = + of_account_updates_map + ~account_update_depth:(fun (p : Account_update.Graphql_repr.t) -> + p.body.call_depth ) + ~f:(fun p -> Account_update.of_graphql_repr p) + xs + |> accumulate_hashes + + let to_account_updates (x : t) = to_account_updates x + + let to_zkapp_command_with_hashes_list (x : t) = + to_zkapp_command_with_hashes_list x + + let account_updates_hash' xs = of_account_updates xs |> hash + + let account_updates_hash xs = + List.map ~f:(fun x -> x) xs |> account_updates_hash' +end + +let is_empty : _ t -> bool = List.is_empty + +let to_list (type p) (t : (p, _, _) t) : p list = + List.rev @@ fold t ~init:[] ~f:(fun acc p -> p :: acc) + +let exists (type p) (t : (p, _, _) t) ~(f : p -> bool) : bool = + with_return (fun { return } -> + fold t ~init:() ~f:(fun () p -> if f p then return true else ()) ; + false ) diff --git a/src/lib/mina_base/zkapp_command.ml b/src/lib/mina_base/zkapp_command.ml index 0583e18f6ec..b2203540f3e 100644 --- a/src/lib/mina_base/zkapp_command.ml +++ b/src/lib/mina_base/zkapp_command.ml @@ -1,630 +1,6 @@ open Core_kernel open Signature_lib -module Call_forest = struct - let empty = Outside_hash_image.t - - module Tree = struct - [%%versioned - module Stable = struct - module V1 = struct - type ('account_update, 'account_update_digest, 'digest) t = - ( 'account_update - , 'account_update_digest - , 'digest ) - Mina_wire_types.Mina_base.Zkapp_command.Call_forest.Tree.V1.t = - { account_update : 'account_update - ; account_update_digest : 'account_update_digest - ; calls : - ( ('account_update, 'account_update_digest, 'digest) t - , 'digest ) - With_stack_hash.Stable.V1.t - list - } - [@@deriving sexp, compare, equal, hash, yojson] - - let to_latest = Fn.id - end - end] - - let rec fold_forest (ts : (_ t, _) With_stack_hash.t list) ~f ~init = - List.fold ts ~init ~f:(fun acc { elt; stack_hash = _ } -> - fold elt ~init:acc ~f ) - - and fold { account_update; calls; account_update_digest = _ } ~f ~init = - fold_forest calls ~f ~init:(f init account_update) - - let rec fold_forest2_exn (ts1 : (_ t, _) With_stack_hash.t list) - (ts2 : (_ t, _) With_stack_hash.t list) ~f ~init = - List.fold2_exn ts1 ts2 ~init - ~f:(fun - acc - { elt = elt1; stack_hash = _ } - { elt = elt2; stack_hash = _ } - -> fold2_exn elt1 elt2 ~init:acc ~f ) - - and fold2_exn - { account_update = account_update1 - ; calls = calls1 - ; account_update_digest = _ - } - { account_update = account_update2 - ; calls = calls2 - ; account_update_digest = _ - } ~f ~init = - fold_forest2_exn calls1 calls2 ~f - ~init:(f init account_update1 account_update2) - - let iter_forest2_exn ts1 ts2 ~f = - fold_forest2_exn ts1 ts2 ~init:() ~f:(fun () p1 p2 -> f p1 p2) - - let iter2_exn ts1 ts2 ~f = - fold2_exn ts1 ts2 ~init:() ~f:(fun () p1 p2 -> f p1 p2) - - let rec mapi_with_trees' ~i (t : _ t) ~f = - let account_update = f i t.account_update t in - let l, calls = mapi_forest_with_trees' ~i:(i + 1) t.calls ~f in - ( l - , { calls - ; account_update - ; account_update_digest = t.account_update_digest - } ) - - and mapi_forest_with_trees' ~i x ~f = - let rec go i acc = function - | [] -> - (i, List.rev acc) - | t :: ts -> - let l, elt' = mapi_with_trees' ~i ~f (With_stack_hash.elt t) in - go l (With_stack_hash.map t ~f:(fun _ -> elt') :: acc) ts - in - go i [] x - - let mapi_with_trees t ~f = mapi_with_trees' ~i:0 t ~f |> snd - - let mapi_forest_with_trees t ~f = mapi_forest_with_trees' ~i:0 t ~f |> snd - - let mapi' ~i t ~f = - mapi_with_trees' ~i t ~f:(fun i account_update _ -> f i account_update) - - let mapi_forest' ~i t ~f = - mapi_forest_with_trees' ~i t ~f:(fun i account_update _ -> - f i account_update ) - - let rec deferred_mapi_with_trees' ~i (t : _ t) ~f = - let open Async_kernel.Deferred.Let_syntax in - let%bind l, calls = - deferred_mapi_forest_with_trees' ~i:(i + 1) t.calls ~f - in - let%map account_update = f i t.account_update t in - ( l - , { calls - ; account_update - ; account_update_digest = t.account_update_digest - } ) - - and deferred_mapi_forest_with_trees' ~i x ~f = - let open Async_kernel.Deferred.Let_syntax in - let rec go i acc = function - | [] -> - return (i, List.rev acc) - | t :: ts -> - let%bind l, elt' = - deferred_mapi_with_trees' ~i ~f (With_stack_hash.elt t) - in - go l (With_stack_hash.map t ~f:(fun _ -> elt') :: acc) ts - in - go i [] x - - let map_forest ~f t = mapi_forest' ~i:0 ~f:(fun _ x -> f x) t |> snd - - let mapi_forest ~f t = mapi_forest' ~i:0 ~f t |> snd - - let deferred_map_forest ~f t = - let open Async_kernel.Deferred in - deferred_mapi_forest_with_trees' ~i:0 ~f:(fun _ x -> f x) t >>| snd - - let deferred_mapi_forest ~f t = - let open Async_kernel.Deferred in - deferred_mapi_forest_with_trees' ~i:0 ~f t >>| snd - - let hash { account_update = _; calls; account_update_digest } = - let stack_hash = - match calls with [] -> empty | e :: _ -> e.stack_hash - in - Random_oracle.hash ~init:Hash_prefix_states.account_update_node - [| account_update_digest; stack_hash |] - end - - type ('a, 'b, 'c) tree = ('a, 'b, 'c) Tree.t - - module type Digest_intf = sig - module Account_update : sig - include Digest_intf.S - - module Checked : sig - include Digest_intf.S_checked - - val create : - ?chain:Mina_signature_kind.t -> Account_update.Checked.t -> t - - val create_body : - ?chain:Mina_signature_kind.t -> Account_update.Body.Checked.t -> t - end - - include Digest_intf.S_aux with type t := t and type checked := Checked.t - - val create : ?chain:Mina_signature_kind.t -> Account_update.t -> t - - val create_body : - ?chain:Mina_signature_kind.t -> Account_update.Body.t -> t - end - - module rec Forest : sig - include Digest_intf.S - - module Checked : sig - include Digest_intf.S_checked - - val empty : t - - val cons : Tree.Checked.t -> t -> t - end - - include Digest_intf.S_aux with type t := t and type checked := Checked.t - - val empty : t - - val cons : Tree.t -> Forest.t -> Forest.t - end - - and Tree : sig - include Digest_intf.S - - module Checked : sig - include Digest_intf.S_checked - - val create : - account_update:Account_update.Checked.t - -> calls:Forest.Checked.t - -> Tree.Checked.t - end - - include Digest_intf.S_aux with type t := t and type checked := Checked.t - - val create : (_, Account_update.t, Forest.t) tree -> Tree.t - end - end - - module Make_digest_sig - (T : Mina_wire_types.Mina_base.Zkapp_command.Digest_types.S) = - struct - module type S = - Digest_intf - with type Account_update.Stable.V1.t = T.Account_update.V1.t - and type Forest.Stable.V1.t = T.Forest.V1.t - end - - module Make_digest_types = struct - module Account_update = struct - [%%versioned - module Stable = struct - module V1 = struct - type t = Kimchi_backend.Pasta.Basic.Fp.Stable.V1.t - [@@deriving sexp, compare, equal, hash, yojson] - - let to_latest = Fn.id - end - end] - end - - module Forest = struct - [%%versioned - module Stable = struct - module V1 = struct - type t = Kimchi_backend.Pasta.Basic.Fp.Stable.V1.t - [@@deriving sexp, compare, equal, hash, yojson] - - let to_latest = Fn.id - end - end] - end - - module Tree = struct - [%%versioned - module Stable = struct - module V1 = struct - type t = Kimchi_backend.Pasta.Basic.Fp.Stable.V1.t - [@@deriving sexp, compare, equal, hash, yojson] - - let to_latest = Fn.id - end - end] - end - end - - module Make_digest_str - (T : Mina_wire_types.Mina_base.Zkapp_command.Digest_concrete) : - Make_digest_sig(T).S = struct - module M = struct - open Pickles.Impls.Step.Field - module Checked = Pickles.Impls.Step.Field - - let typ = typ - - let constant = constant - end - - module Account_update = struct - include Make_digest_types.Account_update - include M - - module Checked = struct - include Checked - - let create = Account_update.Checked.digest - - let create_body = Account_update.Body.Checked.digest - end - - let create : ?chain:Mina_signature_kind.t -> Account_update.t -> t = - Account_update.digest - - let create_body : - ?chain:Mina_signature_kind.t -> Account_update.Body.t -> t = - Account_update.Body.digest - end - - module Forest = struct - include Make_digest_types.Forest - include M - - module Checked = struct - include Checked - - let empty = constant empty - - let cons hash h_tl = - Random_oracle.Checked.hash - ~init:Hash_prefix_states.account_update_cons [| hash; h_tl |] - end - - let empty = empty - - let cons hash h_tl = - Random_oracle.hash ~init:Hash_prefix_states.account_update_cons - [| hash; h_tl |] - end - - module Tree = struct - include Make_digest_types.Tree - include M - - module Checked = struct - include Checked - - let create ~(account_update : Account_update.Checked.t) - ~(calls : Forest.Checked.t) = - Random_oracle.Checked.hash - ~init:Hash_prefix_states.account_update_node - [| (account_update :> t); (calls :> t) |] - end - - let create ({ account_update = _; calls; account_update_digest } : _ tree) - = - let stack_hash = - match calls with [] -> empty | e :: _ -> e.stack_hash - in - Random_oracle.hash ~init:Hash_prefix_states.account_update_node - [| account_update_digest; stack_hash |] - end - end - - module Digest = - Mina_wire_types.Mina_base.Zkapp_command.Digest_make - (Make_digest_sig) - (Make_digest_str) - - let fold = Tree.fold_forest - - let iteri t ~(f : int -> 'a -> unit) : unit = - let (_ : int) = fold t ~init:0 ~f:(fun acc x -> f acc x ; acc + 1) in - () - - [%%versioned - module Stable = struct - module V1 = struct - type ('account_update, 'account_update_digest, 'digest) t = - ( ('account_update, 'account_update_digest, 'digest) Tree.Stable.V1.t - , 'digest ) - With_stack_hash.Stable.V1.t - list - [@@deriving sexp, compare, equal, hash, yojson] - - let to_latest = Fn.id - end - end] - - module Shape = struct - module I = struct - type t = int - - let quickcheck_shrinker = Quickcheck.Shrinker.empty () - - let quickcheck_generator = [%quickcheck.generator: int] - - let quickcheck_observer = [%quickcheck.observer: int] - end - - type t = Node of (I.t * t) list [@@deriving quickcheck] - end - - let rec shape (t : _ t) : Shape.t = - Node (List.mapi t ~f:(fun i { elt; stack_hash = _ } -> (i, shape elt.calls))) - - let match_up (type a b) (xs : a list) (ys : (int * b) list) : (a * b) list = - let rec go i_curr xs ys = - match (xs, ys) with - | [], [] -> - [] - | x :: xs', (i, y) :: ys' -> - if i_curr = i then (x, y) :: go (i_curr + 1) xs' ys' - else if i_curr < i then go (i_curr + 1) xs' ys' - else assert false - | [], _ :: _ -> - assert false - | _ :: _, [] -> - [] - in - go 0 xs ys - - let rec mask (t : ('p, 'h1, unit) t) (Node shape : Shape.t) : - ('p, 'h1, unit) t = - List.map (match_up t shape) - ~f:(fun ({ With_stack_hash.elt = t_sub; stack_hash = () }, shape_sub) -> - { With_stack_hash.elt = - { t_sub with calls = mask t_sub.calls shape_sub } - ; stack_hash = () - } ) - - let rec of_account_updates_map ~(f : 'p1 -> 'p2) - ~(account_update_depth : 'p1 -> int) (account_updates : 'p1 list) : - ('p2, unit, unit) t = - match account_updates with - | [] -> - [] - | p :: ps -> - let depth = account_update_depth p in - let children, siblings = - List.split_while ps ~f:(fun p' -> account_update_depth p' > depth) - in - { With_stack_hash.elt = - { Tree.account_update = f p - ; account_update_digest = () - ; calls = of_account_updates_map ~f ~account_update_depth children - } - ; stack_hash = () - } - :: of_account_updates_map ~f ~account_update_depth siblings - - let of_account_updates ~account_update_depth account_updates = - of_account_updates_map ~f:Fn.id ~account_update_depth account_updates - - let to_account_updates_map ~f (xs : _ t) = - let rec collect depth (xs : _ t) acc = - match xs with - | [] -> - acc - | { elt = { account_update; calls; account_update_digest = _ } - ; stack_hash = _ - } - :: xs -> - f ~depth account_update :: acc - |> collect (depth + 1) calls - |> collect depth xs - in - List.rev (collect 0 xs []) - - let to_account_updates xs = - to_account_updates_map ~f:(fun ~depth:_ account_update -> account_update) xs - - let hd_account_update (xs : _ t) = - match xs with - | [] -> - None - | { elt = { account_update; calls = _; account_update_digest = _ } - ; stack_hash = _ - } - :: _ -> - Some account_update - - let map = Tree.map_forest - - let mapi = Tree.mapi_forest - - let mapi_with_trees = Tree.mapi_forest_with_trees - - let deferred_mapi = Tree.deferred_mapi_forest - - let to_zkapp_command_with_hashes_list (xs : _ t) = - let rec collect (xs : _ t) acc = - match xs with - | [] -> - acc - | { elt = { account_update; calls; account_update_digest = _ } - ; stack_hash - } - :: xs -> - (account_update, stack_hash) :: acc |> collect calls |> collect xs - in - List.rev (collect xs []) - - let hash_cons hash h_tl = - Random_oracle.hash ~init:Hash_prefix_states.account_update_cons - [| hash; h_tl |] - - let hash = function - | [] -> - Digest.Forest.empty - | x :: _ -> - With_stack_hash.stack_hash x - - let cons_tree tree (forest : _ t) : _ t = - { elt = tree - ; stack_hash = Digest.Forest.cons (Digest.Tree.create tree) (hash forest) - } - :: forest - - let cons_aux (type p) ~(digest_account_update : p -> _) ?(calls = []) - (account_update : p) (xs : _ t) : _ t = - let account_update_digest = digest_account_update account_update in - let tree : _ Tree.t = { account_update; account_update_digest; calls } in - cons_tree tree xs - - let cons ?calls (account_update : Account_update.t) xs = - cons_aux ~digest_account_update:Digest.Account_update.create ?calls - account_update xs - - let rec accumulate_hashes ~hash_account_update (xs : _ t) = - let go = accumulate_hashes ~hash_account_update in - match xs with - | [] -> - [] - | { elt = { account_update; calls; account_update_digest = _ } - ; stack_hash = _ - } - :: xs -> - let calls = go calls in - let xs = go xs in - let node = - { Tree.account_update - ; calls - ; account_update_digest = hash_account_update account_update - } - in - let node_hash = Digest.Tree.create node in - { elt = node; stack_hash = Digest.Forest.cons node_hash (hash xs) } - :: xs - - let accumulate_hashes' (type a b) (xs : (Account_update.t, a, b) t) : - (Account_update.t, Digest.Account_update.t, Digest.Forest.t) t = - let hash_account_update (p : Account_update.t) = - Digest.Account_update.create p - in - accumulate_hashes ~hash_account_update xs - - let accumulate_hashes_predicated xs = - accumulate_hashes ~hash_account_update:Digest.Account_update.create xs - - module With_hashes_and_data = struct - [%%versioned - module Stable = struct - module V1 = struct - type 'data t = - ( Account_update.Stable.V1.t * 'data - , Digest.Account_update.Stable.V1.t - , Digest.Forest.Stable.V1.t ) - Stable.V1.t - [@@deriving sexp, compare, equal, hash, yojson] - - let to_latest = Fn.id - end - end] - - let empty = Digest.Forest.empty - - let hash_account_update ((p : Account_update.t), _) = - Digest.Account_update.create p - - let accumulate_hashes xs : _ t = accumulate_hashes ~hash_account_update xs - - let of_zkapp_command_simple_list (xs : (Account_update.Simple.t * 'a) list) - : _ t = - of_account_updates xs - ~account_update_depth:(fun ((p : Account_update.Simple.t), _) -> - p.body.call_depth ) - |> map ~f:(fun (p, x) -> (Account_update.of_simple p, x)) - |> accumulate_hashes - - let of_account_updates (xs : (Account_update.Graphql_repr.t * 'a) list) : - _ t = - of_account_updates_map - ~account_update_depth:(fun ((p : Account_update.Graphql_repr.t), _) -> - p.body.call_depth ) - ~f:(fun (p, x) -> (Account_update.of_graphql_repr p, x)) - xs - |> accumulate_hashes - - let to_account_updates (x : _ t) = to_account_updates x - - let to_zkapp_command_with_hashes_list (x : _ t) = - to_zkapp_command_with_hashes_list x - - let account_updates_hash' xs = of_account_updates xs |> hash - - let account_updates_hash xs = - List.map ~f:(fun x -> (x, ())) xs |> account_updates_hash' - end - - module With_hashes = struct - [%%versioned - module Stable = struct - module V1 = struct - type t = - ( Account_update.Stable.V1.t - , Digest.Account_update.Stable.V1.t - , Digest.Forest.Stable.V1.t ) - Stable.V1.t - [@@deriving sexp, compare, equal, hash, yojson] - - let to_latest = Fn.id - end - end] - - let empty = Digest.Forest.empty - - let hash_account_update (p : Account_update.t) = - Digest.Account_update.create p - - let accumulate_hashes xs : t = accumulate_hashes ~hash_account_update xs - - let of_zkapp_command_simple_list (xs : Account_update.Simple.t list) : t = - of_account_updates xs - ~account_update_depth:(fun (p : Account_update.Simple.t) -> - p.body.call_depth ) - |> map ~f:Account_update.of_simple - |> accumulate_hashes - - let of_account_updates (xs : Account_update.Graphql_repr.t list) : t = - of_account_updates_map - ~account_update_depth:(fun (p : Account_update.Graphql_repr.t) -> - p.body.call_depth ) - ~f:(fun p -> Account_update.of_graphql_repr p) - xs - |> accumulate_hashes - - let to_account_updates (x : t) = to_account_updates x - - let to_zkapp_command_with_hashes_list (x : t) = - to_zkapp_command_with_hashes_list x - - let account_updates_hash' xs = of_account_updates xs |> hash - - let account_updates_hash xs = - List.map ~f:(fun x -> x) xs |> account_updates_hash' - end - - let is_empty : _ t -> bool = List.is_empty - - let to_list (type p) (t : (p, _, _) t) : p list = - List.rev @@ fold t ~init:[] ~f:(fun acc p -> p :: acc) - - let exists (type p) (t : (p, _, _) t) ~(f : p -> bool) : bool = - with_return (fun { return } -> - fold t ~init:() ~f:(fun () p -> if f p then return true else ()) ; - false ) -end - module Graphql_repr = struct [%%versioned module Stable = struct @@ -658,6 +34,7 @@ module Simple = struct end] end +module Call_forest = Zkapp_call_forest_base module Digest = Call_forest.Digest module T = struct From fa9ecafd66fbe6ac5451cf9799bf269986639821 Mon Sep 17 00:00:00 2001 From: georgeee Date: Mon, 21 Oct 2024 13:45:32 +0000 Subject: [PATCH 27/47] Extract Transaction_applied from Transaction_logic Module is standalone and 200+ lines long. --- .../mina_transaction_logic.ml | 247 +----------------- .../transaction_logic/transaction_applied.ml | 241 +++++++++++++++++ 2 files changed, 243 insertions(+), 245 deletions(-) create mode 100644 src/lib/transaction_logic/transaction_applied.ml diff --git a/src/lib/transaction_logic/mina_transaction_logic.ml b/src/lib/transaction_logic/mina_transaction_logic.ml index 381b06b85ea..3e4854a3bbf 100644 --- a/src/lib/transaction_logic/mina_transaction_logic.ml +++ b/src/lib/transaction_logic/mina_transaction_logic.ml @@ -6,251 +6,6 @@ open Mina_transaction module Zkapp_command_logic = Zkapp_command_logic module Global_slot_since_genesis = Mina_numbers.Global_slot_since_genesis -module Transaction_applied = struct - module UC = Signed_command - - module Signed_command_applied = struct - module Common = struct - [%%versioned - module Stable = struct - module V2 = struct - type t = - { user_command : Signed_command.Stable.V2.t With_status.Stable.V2.t - } - [@@deriving sexp, to_yojson] - - let to_latest = Fn.id - end - end] - end - - module Body = struct - [%%versioned - module Stable = struct - module V2 = struct - type t = - | Payment of { new_accounts : Account_id.Stable.V2.t list } - | Stake_delegation of - { previous_delegate : Public_key.Compressed.Stable.V1.t option } - | Failed - [@@deriving sexp, to_yojson] - - let to_latest = Fn.id - end - end] - end - - [%%versioned - module Stable = struct - module V2 = struct - type t = { common : Common.Stable.V2.t; body : Body.Stable.V2.t } - [@@deriving sexp, to_yojson] - - let to_latest = Fn.id - end - end] - - let new_accounts (t : t) = - match t.body with - | Payment { new_accounts; _ } -> - new_accounts - | Stake_delegation _ | Failed -> - [] - end - - module Zkapp_command_applied = struct - [%%versioned - module Stable = struct - module V1 = struct - type t = - { accounts : - (Account_id.Stable.V2.t * Account.Stable.V2.t option) list - ; command : Zkapp_command.Stable.V1.t With_status.Stable.V2.t - ; new_accounts : Account_id.Stable.V2.t list - } - [@@deriving sexp, to_yojson] - - let to_latest = Fn.id - end - end] - end - - module Command_applied = struct - [%%versioned - module Stable = struct - module V2 = struct - type t = - | Signed_command of Signed_command_applied.Stable.V2.t - | Zkapp_command of Zkapp_command_applied.Stable.V1.t - [@@deriving sexp, to_yojson] - - let to_latest = Fn.id - end - end] - end - - module Fee_transfer_applied = struct - [%%versioned - module Stable = struct - module V2 = struct - type t = - { fee_transfer : Fee_transfer.Stable.V2.t With_status.Stable.V2.t - ; new_accounts : Account_id.Stable.V2.t list - ; burned_tokens : Currency.Amount.Stable.V1.t - } - [@@deriving sexp, to_yojson] - - let to_latest = Fn.id - end - end] - end - - module Coinbase_applied = struct - [%%versioned - module Stable = struct - module V2 = struct - type t = - { coinbase : Coinbase.Stable.V1.t With_status.Stable.V2.t - ; new_accounts : Account_id.Stable.V2.t list - ; burned_tokens : Currency.Amount.Stable.V1.t - } - [@@deriving sexp, to_yojson] - - let to_latest = Fn.id - end - end] - end - - module Varying = struct - [%%versioned - module Stable = struct - module V2 = struct - type t = - | Command of Command_applied.Stable.V2.t - | Fee_transfer of Fee_transfer_applied.Stable.V2.t - | Coinbase of Coinbase_applied.Stable.V2.t - [@@deriving sexp, to_yojson] - - let to_latest = Fn.id - end - end] - end - - [%%versioned - module Stable = struct - module V2 = struct - type t = - { previous_hash : Ledger_hash.Stable.V1.t - ; varying : Varying.Stable.V2.t - } - [@@deriving sexp, to_yojson] - - let to_latest = Fn.id - end - end] - - let burned_tokens : t -> Currency.Amount.t = - fun { varying; _ } -> - match varying with - | Command _ -> - Currency.Amount.zero - | Fee_transfer f -> - f.burned_tokens - | Coinbase c -> - c.burned_tokens - - let new_accounts : t -> Account_id.t list = - fun { varying; _ } -> - match varying with - | Command c -> ( - match c with - | Signed_command sc -> - Signed_command_applied.new_accounts sc - | Zkapp_command zc -> - zc.new_accounts ) - | Fee_transfer f -> - f.new_accounts - | Coinbase c -> - c.new_accounts - - let supply_increase : - constraint_constants:Genesis_constants.Constraint_constants.t - -> t - -> Currency.Amount.Signed.t Or_error.t = - fun ~constraint_constants t -> - let open Or_error.Let_syntax in - let burned_tokens = Currency.Amount.Signed.of_unsigned (burned_tokens t) in - let account_creation_fees = - let account_creation_fee_int = - constraint_constants.account_creation_fee - |> Currency.Fee.to_nanomina_int - in - let num_accounts_created = List.length @@ new_accounts t in - (* int type is OK, no danger of overflow *) - Currency.Amount.( - Signed.of_unsigned - @@ of_nanomina_int_exn (account_creation_fee_int * num_accounts_created)) - in - let txn : Transaction.t = - match t.varying with - | Command - (Signed_command { common = { user_command = { data; _ }; _ }; _ }) -> - Command (Signed_command data) - | Command (Zkapp_command c) -> - Command (Zkapp_command c.command.data) - | Fee_transfer f -> - Fee_transfer f.fee_transfer.data - | Coinbase c -> - Coinbase c.coinbase.data - in - let%bind expected_supply_increase = - Transaction.expected_supply_increase txn - in - let rec process_decreases total = function - | [] -> - Some total - | amt :: amts -> - let%bind.Option sum = - Currency.Amount.Signed.(add @@ negate amt) total - in - process_decreases sum amts - in - let total = - process_decreases - (Currency.Amount.Signed.of_unsigned expected_supply_increase) - [ burned_tokens; account_creation_fees ] - in - Option.value_map total ~default:(Or_error.error_string "overflow") - ~f:(fun v -> Ok v) - - let transaction_with_status : t -> Transaction.t With_status.t = - fun { varying; _ } -> - match varying with - | Command (Signed_command uc) -> - With_status.map uc.common.user_command ~f:(fun cmd -> - Transaction.Command (User_command.Signed_command cmd) ) - | Command (Zkapp_command s) -> - With_status.map s.command ~f:(fun c -> - Transaction.Command (User_command.Zkapp_command c) ) - | Fee_transfer f -> - With_status.map f.fee_transfer ~f:(fun f -> Transaction.Fee_transfer f) - | Coinbase c -> - With_status.map c.coinbase ~f:(fun c -> Transaction.Coinbase c) - - let transaction_status : t -> Transaction_status.t = - fun { varying; _ } -> - match varying with - | Command - (Signed_command { common = { user_command = { status; _ }; _ }; _ }) -> - status - | Command (Zkapp_command c) -> - c.command.status - | Fee_transfer f -> - f.fee_transfer.status - | Coinbase c -> - c.coinbase.status -end - module type S = sig type ledger @@ -2939,3 +2694,5 @@ module For_tests = struct failwithf "gen_zkapp_command_from_test_spec: expected one spec, got %d" (List.length specs) () end + +module Transaction_applied = Transaction_applied diff --git a/src/lib/transaction_logic/transaction_applied.ml b/src/lib/transaction_logic/transaction_applied.ml new file mode 100644 index 00000000000..aa62f240ce2 --- /dev/null +++ b/src/lib/transaction_logic/transaction_applied.ml @@ -0,0 +1,241 @@ +open Core_kernel +open Mina_base +open Signature_lib +open Mina_transaction +module UC = Signed_command + +module Signed_command_applied = struct + module Common = struct + [%%versioned + module Stable = struct + module V2 = struct + type t = + { user_command : Signed_command.Stable.V2.t With_status.Stable.V2.t } + [@@deriving sexp, to_yojson] + + let to_latest = Fn.id + end + end] + end + + module Body = struct + [%%versioned + module Stable = struct + module V2 = struct + type t = + | Payment of { new_accounts : Account_id.Stable.V2.t list } + | Stake_delegation of + { previous_delegate : Public_key.Compressed.Stable.V1.t option } + | Failed + [@@deriving sexp, to_yojson] + + let to_latest = Fn.id + end + end] + end + + [%%versioned + module Stable = struct + module V2 = struct + type t = { common : Common.Stable.V2.t; body : Body.Stable.V2.t } + [@@deriving sexp, to_yojson] + + let to_latest = Fn.id + end + end] + + let new_accounts (t : t) = + match t.body with + | Payment { new_accounts; _ } -> + new_accounts + | Stake_delegation _ | Failed -> + [] +end + +module Zkapp_command_applied = struct + [%%versioned + module Stable = struct + module V1 = struct + type t = + { accounts : (Account_id.Stable.V2.t * Account.Stable.V2.t option) list + ; command : Zkapp_command.Stable.V1.t With_status.Stable.V2.t + ; new_accounts : Account_id.Stable.V2.t list + } + [@@deriving sexp, to_yojson] + + let to_latest = Fn.id + end + end] +end + +module Command_applied = struct + [%%versioned + module Stable = struct + module V2 = struct + type t = + | Signed_command of Signed_command_applied.Stable.V2.t + | Zkapp_command of Zkapp_command_applied.Stable.V1.t + [@@deriving sexp, to_yojson] + + let to_latest = Fn.id + end + end] +end + +module Fee_transfer_applied = struct + [%%versioned + module Stable = struct + module V2 = struct + type t = + { fee_transfer : Fee_transfer.Stable.V2.t With_status.Stable.V2.t + ; new_accounts : Account_id.Stable.V2.t list + ; burned_tokens : Currency.Amount.Stable.V1.t + } + [@@deriving sexp, to_yojson] + + let to_latest = Fn.id + end + end] +end + +module Coinbase_applied = struct + [%%versioned + module Stable = struct + module V2 = struct + type t = + { coinbase : Coinbase.Stable.V1.t With_status.Stable.V2.t + ; new_accounts : Account_id.Stable.V2.t list + ; burned_tokens : Currency.Amount.Stable.V1.t + } + [@@deriving sexp, to_yojson] + + let to_latest = Fn.id + end + end] +end + +module Varying = struct + [%%versioned + module Stable = struct + module V2 = struct + type t = + | Command of Command_applied.Stable.V2.t + | Fee_transfer of Fee_transfer_applied.Stable.V2.t + | Coinbase of Coinbase_applied.Stable.V2.t + [@@deriving sexp, to_yojson] + + let to_latest = Fn.id + end + end] +end + +[%%versioned +module Stable = struct + module V2 = struct + type t = + { previous_hash : Ledger_hash.Stable.V1.t; varying : Varying.Stable.V2.t } + [@@deriving sexp, to_yojson] + + let to_latest = Fn.id + end +end] + +let burned_tokens : t -> Currency.Amount.t = + fun { varying; _ } -> + match varying with + | Command _ -> + Currency.Amount.zero + | Fee_transfer f -> + f.burned_tokens + | Coinbase c -> + c.burned_tokens + +let new_accounts : t -> Account_id.t list = + fun { varying; _ } -> + match varying with + | Command c -> ( + match c with + | Signed_command sc -> + Signed_command_applied.new_accounts sc + | Zkapp_command zc -> + zc.new_accounts ) + | Fee_transfer f -> + f.new_accounts + | Coinbase c -> + c.new_accounts + +let supply_increase : + constraint_constants:Genesis_constants.Constraint_constants.t + -> t + -> Currency.Amount.Signed.t Or_error.t = + fun ~constraint_constants t -> + let open Or_error.Let_syntax in + let burned_tokens = Currency.Amount.Signed.of_unsigned (burned_tokens t) in + let account_creation_fees = + let account_creation_fee_int = + constraint_constants.account_creation_fee |> Currency.Fee.to_nanomina_int + in + let num_accounts_created = List.length @@ new_accounts t in + (* int type is OK, no danger of overflow *) + Currency.Amount.( + Signed.of_unsigned + @@ of_nanomina_int_exn (account_creation_fee_int * num_accounts_created)) + in + let txn : Transaction.t = + match t.varying with + | Command (Signed_command { common = { user_command = { data; _ }; _ }; _ }) + -> + Command (Signed_command data) + | Command (Zkapp_command c) -> + Command (Zkapp_command c.command.data) + | Fee_transfer f -> + Fee_transfer f.fee_transfer.data + | Coinbase c -> + Coinbase c.coinbase.data + in + let%bind expected_supply_increase = + Transaction.expected_supply_increase txn + in + let rec process_decreases total = function + | [] -> + Some total + | amt :: amts -> + let%bind.Option sum = + Currency.Amount.Signed.(add @@ negate amt) total + in + process_decreases sum amts + in + let total = + process_decreases + (Currency.Amount.Signed.of_unsigned expected_supply_increase) + [ burned_tokens; account_creation_fees ] + in + Option.value_map total ~default:(Or_error.error_string "overflow") + ~f:(fun v -> Ok v) + +let transaction_with_status : t -> Transaction.t With_status.t = + fun { varying; _ } -> + match varying with + | Command (Signed_command uc) -> + With_status.map uc.common.user_command ~f:(fun cmd -> + Transaction.Command (User_command.Signed_command cmd) ) + | Command (Zkapp_command s) -> + With_status.map s.command ~f:(fun c -> + Transaction.Command (User_command.Zkapp_command c) ) + | Fee_transfer f -> + With_status.map f.fee_transfer ~f:(fun f -> Transaction.Fee_transfer f) + | Coinbase c -> + With_status.map c.coinbase ~f:(fun c -> Transaction.Coinbase c) + +let transaction_status : t -> Transaction_status.t = + fun { varying; _ } -> + match varying with + | Command (Signed_command { common = { user_command = { status; _ }; _ }; _ }) + -> + status + | Command (Zkapp_command c) -> + c.command.status + | Fee_transfer f -> + f.fee_transfer.status + | Coinbase c -> + c.coinbase.status From 4433ee935f863049b6f1f22c3f988d35eed64981 Mon Sep 17 00:00:00 2001 From: georgeee Date: Mon, 21 Oct 2024 14:07:40 +0000 Subject: [PATCH 28/47] Do not over-expose Transaction_applied Transaction_applied is exposed in both the top-level module and a `Make` functor. It's cleaner to just expose it once. --- src/lib/mina_ledger/sparse_ledger.ml | 4 +- .../snark_profiler_lib/snark_profiler_lib.ml | 10 +- src/lib/staged_ledger/pre_diff_info.ml | 2 +- src/lib/staged_ledger/staged_ledger.ml | 15 +- src/lib/staged_ledger/staged_ledger.mli | 4 +- .../mina_transaction_logic.ml | 140 ++++-------------- src/lib/transaction_logic/test/helpers.ml | 4 +- src/lib/transaction_logic/test/predicates.ml | 4 +- src/lib/transaction_logic/test/zkapp_logic.ml | 2 +- .../test/account_timing/account_timing.ml | 2 +- .../transaction_union/transaction_union.ml | 4 +- src/lib/transaction_snark/test/util.ml | 11 +- .../transaction_validator.ml | 4 +- .../transaction_validator.mli | 2 +- .../transaction_snark_scan_state.ml | 16 +- .../transaction_snark_scan_state.mli | 8 +- 16 files changed, 74 insertions(+), 158 deletions(-) diff --git a/src/lib/mina_ledger/sparse_ledger.ml b/src/lib/mina_ledger/sparse_ledger.ml index fec47bf2266..e89b1082d5a 100644 --- a/src/lib/mina_ledger/sparse_ledger.ml +++ b/src/lib/mina_ledger/sparse_ledger.ml @@ -179,7 +179,9 @@ let apply_zkapp_second_pass_unchecked_with_states ~init ledger c = |> Result.map ~f:(fun (account_update_applied, rev_states) -> let module LS = Mina_transaction_logic.Zkapp_command_logic.Local_state in - let module Applied = T.Transaction_applied.Zkapp_command_applied in + let module Applied = + Mina_transaction_logic.Transaction_applied.Zkapp_command_applied + in let states = match rev_states with | [] -> diff --git a/src/lib/snark_profiler_lib/snark_profiler_lib.ml b/src/lib/snark_profiler_lib/snark_profiler_lib.ml index f40acc5112a..9c55ff5aadb 100644 --- a/src/lib/snark_profiler_lib/snark_profiler_lib.ml +++ b/src/lib/snark_profiler_lib/snark_profiler_lib.ml @@ -563,7 +563,7 @@ let profile_user_command (module T : Transaction_snark.S) ~genesis_constants (target_ledger, applied) -> let txn = With_status.data - @@ Mina_ledger.Ledger.Transaction_applied.transaction applied + @@ Mina_ledger.Ledger.transaction_of_applied applied in (* the txn was already valid before apply, we are just recasting it here after application *) let (`If_this_is_used_it_should_have_a_comment_justifying_it @@ -782,7 +782,7 @@ let check_base_snarks ~genesis_constants ~constraint_constants sparse_ledger0 ~f:(fun source_ledger (target_ledger, applied_txn) -> let txn = With_status.data - @@ Mina_ledger.Ledger.Transaction_applied.transaction applied_txn + @@ Mina_ledger.Ledger.transaction_of_applied applied_txn in (* the txn was already valid before apply, we are just recasting it here after application *) let (`If_this_is_used_it_should_have_a_comment_justifying_it @@ -793,7 +793,7 @@ let check_base_snarks ~genesis_constants ~constraint_constants sparse_ledger0 pending_coinbase_stack_target txn Pending_coinbase.Stack.empty in let supply_increase = - Mina_ledger.Ledger.Transaction_applied.supply_increase + Mina_transaction_logic.Transaction_applied.supply_increase ~constraint_constants applied_txn |> Or_error.ok_exn in @@ -844,7 +844,7 @@ let generate_base_snarks_witness ~genesis_constants ~constraint_constants ~f:(fun source_ledger (target_ledger, applied_txn) -> let txn = With_status.data - @@ Mina_ledger.Ledger.Transaction_applied.transaction applied_txn + @@ Mina_ledger.Ledger.transaction_of_applied applied_txn in (* the txn was already valid before apply, we are just recasting it here after application *) let (`If_this_is_used_it_should_have_a_comment_justifying_it @@ -855,7 +855,7 @@ let generate_base_snarks_witness ~genesis_constants ~constraint_constants pending_coinbase_stack_target txn Pending_coinbase.Stack.empty in let supply_increase = - Mina_ledger.Ledger.Transaction_applied.supply_increase + Mina_transaction_logic.Transaction_applied.supply_increase ~constraint_constants applied_txn |> Or_error.ok_exn in diff --git a/src/lib/staged_ledger/pre_diff_info.ml b/src/lib/staged_ledger/pre_diff_info.ml index 571d72ccafb..57a9d89ce6e 100644 --- a/src/lib/staged_ledger/pre_diff_info.ml +++ b/src/lib/staged_ledger/pre_diff_info.ml @@ -361,7 +361,7 @@ let compute_statuses let split_transaction_statuses txns_with_statuses = List.partition_map txns_with_statuses ~f:(fun txn_applied -> let { With_status.data = txn; status } = - Mina_ledger.Ledger.Transaction_applied.transaction txn_applied + Mina_ledger.Ledger.transaction_of_applied txn_applied in match txn with | Transaction.Command cmd -> diff --git a/src/lib/staged_ledger/staged_ledger.ml b/src/lib/staged_ledger/staged_ledger.ml index af202350850..3f06cc63ad8 100644 --- a/src/lib/staged_ledger/staged_ledger.ml +++ b/src/lib/staged_ledger/staged_ledger.ml @@ -580,20 +580,17 @@ module T = struct let second_pass_ledger_target_hash = Ledger.merkle_root ledger in let%bind supply_increase = to_staged_ledger_or_error - (Ledger.Transaction_applied.supply_increase ~constraint_constants - applied_txn ) + (Mina_transaction_logic.Transaction_applied.supply_increase + ~constraint_constants applied_txn ) in let%map () = - let actual_status = - Ledger.Transaction_applied.transaction_status applied_txn - in + let actual_status = Ledger.status_of_applied applied_txn in if Transaction_status.equal pre_stmt.expected_status actual_status then return () else let txn_with_expected_status = { With_status.data = - With_status.data - (Ledger.Transaction_applied.transaction applied_txn) + With_status.data (Ledger.transaction_of_applied applied_txn) ; status = pre_stmt.expected_status } in @@ -789,9 +786,7 @@ module T = struct List.fold_right ~init:(Ok []) data ~f:(fun (d : Scan_state.Transaction_with_witness.t) acc -> let%map.Or_error acc = acc in - let t = - d.transaction_with_info |> Ledger.Transaction_applied.transaction - in + let t = d.transaction_with_info |> Ledger.transaction_of_applied in t :: acc ) in let total_fee_excess txns = diff --git a/src/lib/staged_ledger/staged_ledger.mli b/src/lib/staged_ledger/staged_ledger.mli index 805314908cb..e94589bbea3 100644 --- a/src/lib/staged_ledger/staged_ledger.mli +++ b/src/lib/staged_ledger/staged_ledger.mli @@ -93,7 +93,7 @@ module Scan_state : sig -> apply_second_pass: ( Ledger.t -> Ledger.Transaction_partially_applied.t - -> Ledger.Transaction_applied.t Or_error.t ) + -> Mina_transaction_logic.Transaction_applied.t Or_error.t ) -> apply_first_pass_sparse_ledger: ( global_slot:Mina_numbers.Global_slot_since_genesis.t -> txn_state_view:Mina_base.Zkapp_precondition.Protocol_state.View.t @@ -121,7 +121,7 @@ module Scan_state : sig -> apply_second_pass: ( Ledger.t -> Ledger.Transaction_partially_applied.t - -> Ledger.Transaction_applied.t Or_error.t ) + -> Mina_transaction_logic.Transaction_applied.t Or_error.t ) -> apply_first_pass_sparse_ledger: ( global_slot:Mina_numbers.Global_slot_since_genesis.t -> txn_state_view:Mina_base.Zkapp_precondition.Protocol_state.View.t diff --git a/src/lib/transaction_logic/mina_transaction_logic.ml b/src/lib/transaction_logic/mina_transaction_logic.ml index 381b06b85ea..40c4c9f5806 100644 --- a/src/lib/transaction_logic/mina_transaction_logic.ml +++ b/src/lib/transaction_logic/mina_transaction_logic.ml @@ -256,87 +256,10 @@ module type S = sig type location - module Transaction_applied : sig - module Signed_command_applied : sig - module Common : sig - type t = Transaction_applied.Signed_command_applied.Common.t = - { user_command : Signed_command.t With_status.t } - [@@deriving sexp] - end - - module Body : sig - type t = Transaction_applied.Signed_command_applied.Body.t = - | Payment of { new_accounts : Account_id.t list } - | Stake_delegation of - { previous_delegate : Public_key.Compressed.t option } - | Failed - [@@deriving sexp] - end - - type t = Transaction_applied.Signed_command_applied.t = - { common : Common.t; body : Body.t } - [@@deriving sexp] - end - - module Zkapp_command_applied : sig - type t = Transaction_applied.Zkapp_command_applied.t = - { accounts : (Account_id.t * Account.t option) list - ; command : Zkapp_command.t With_status.t - ; new_accounts : Account_id.t list - } - [@@deriving sexp] - end + val transaction_of_applied : + Transaction_applied.t -> Transaction.t With_status.t - module Command_applied : sig - type t = Transaction_applied.Command_applied.t = - | Signed_command of Signed_command_applied.t - | Zkapp_command of Zkapp_command_applied.t - [@@deriving sexp] - end - - module Fee_transfer_applied : sig - type t = Transaction_applied.Fee_transfer_applied.t = - { fee_transfer : Fee_transfer.t With_status.t - ; new_accounts : Account_id.t list - ; burned_tokens : Currency.Amount.t - } - [@@deriving sexp] - end - - module Coinbase_applied : sig - type t = Transaction_applied.Coinbase_applied.t = - { coinbase : Coinbase.t With_status.t - ; new_accounts : Account_id.t list - ; burned_tokens : Currency.Amount.t - } - [@@deriving sexp] - end - - module Varying : sig - type t = Transaction_applied.Varying.t = - | Command of Command_applied.t - | Fee_transfer of Fee_transfer_applied.t - | Coinbase of Coinbase_applied.t - [@@deriving sexp] - end - - type t = Transaction_applied.t = - { previous_hash : Ledger_hash.t; varying : Varying.t } - [@@deriving sexp] - - val burned_tokens : t -> Currency.Amount.t - - val supply_increase : - constraint_constants:Genesis_constants.Constraint_constants.t - -> t - -> Currency.Amount.Signed.t Or_error.t - - val transaction : t -> Transaction.t With_status.t - - val transaction_status : t -> Transaction_status.t - - val new_accounts : t -> Account_id.t list - end + val status_of_applied : Transaction_applied.t -> Transaction_status.t module Global_state : sig type t = @@ -760,38 +683,33 @@ module Make (L : Ledger_intf.S) : transaction expiry slot %{sexp: Global_slot_since_genesis.t}" current_global_slot valid_until - module Transaction_applied = struct - include Transaction_applied - - let transaction : t -> Transaction.t With_status.t = - fun { varying; _ } -> - match varying with - | Command (Signed_command uc) -> - With_status.map uc.common.user_command ~f:(fun cmd -> - Transaction.Command (User_command.Signed_command cmd) ) - | Command (Zkapp_command s) -> - With_status.map s.command ~f:(fun c -> - Transaction.Command (User_command.Zkapp_command c) ) - | Fee_transfer f -> - With_status.map f.fee_transfer ~f:(fun f -> - Transaction.Fee_transfer f ) - | Coinbase c -> - With_status.map c.coinbase ~f:(fun c -> Transaction.Coinbase c) + let transaction_of_applied : + Transaction_applied.t -> Transaction.t With_status.t = + fun { varying; _ } -> + match varying with + | Command (Signed_command uc) -> + With_status.map uc.common.user_command ~f:(fun cmd -> + Transaction.Command (User_command.Signed_command cmd) ) + | Command (Zkapp_command s) -> + With_status.map s.command ~f:(fun c -> + Transaction.Command (User_command.Zkapp_command c) ) + | Fee_transfer f -> + With_status.map f.fee_transfer ~f:(fun f -> Transaction.Fee_transfer f) + | Coinbase c -> + With_status.map c.coinbase ~f:(fun c -> Transaction.Coinbase c) - let transaction_status : t -> Transaction_status.t = - fun { varying; _ } -> - match varying with - | Command - (Signed_command { common = { user_command = { status; _ }; _ }; _ }) - -> - status - | Command (Zkapp_command c) -> - c.command.status - | Fee_transfer f -> - f.fee_transfer.status - | Coinbase c -> - c.coinbase.status - end + let status_of_applied : Transaction_applied.t -> Transaction_status.t = + fun { varying; _ } -> + match varying with + | Command + (Signed_command { common = { user_command = { status; _ }; _ }; _ }) -> + status + | Command (Zkapp_command c) -> + c.command.status + | Fee_transfer f -> + f.fee_transfer.status + | Coinbase c -> + c.coinbase.status let get_new_accounts action pk = if Ledger_intf.equal_account_state action `Added then [ pk ] else [] diff --git a/src/lib/transaction_logic/test/helpers.ml b/src/lib/transaction_logic/test/helpers.ml index 5299de0b91f..536087f71e2 100644 --- a/src/lib/transaction_logic/test/helpers.ml +++ b/src/lib/transaction_logic/test/helpers.ml @@ -3,8 +3,8 @@ module Transaction_logic = Mina_transaction_logic.Make (Ledger) module Zk_cmd_result = struct type t = - Transaction_logic.Transaction_applied.Zkapp_command_applied.t * Ledger.t + Mina_transaction_logic.Transaction_applied.Zkapp_command_applied.t * Ledger.t let sexp_of_t (txn, _) = - Transaction_logic.Transaction_applied.Zkapp_command_applied.sexp_of_t txn + Mina_transaction_logic.Transaction_applied.Zkapp_command_applied.sexp_of_t txn end diff --git a/src/lib/transaction_logic/test/predicates.ml b/src/lib/transaction_logic/test/predicates.ml index 15fc4c95616..e5d69c64a16 100644 --- a/src/lib/transaction_logic/test/predicates.ml +++ b/src/lib/transaction_logic/test/predicates.ml @@ -13,7 +13,7 @@ let result ?(with_error = Fn.const false) ~f result = let verify_account_updates ~(ledger : Helpers.Ledger.t) ~(txn : - Helpers.Transaction_logic.Transaction_applied.Zkapp_command_applied.t ) + Mina_transaction_logic.Transaction_applied.Zkapp_command_applied.t ) ~(f : Amount.Signed.t -> Account.t option * Account.t option -> bool) (account : Test_account.t) = let open Helpers in @@ -81,7 +81,7 @@ let verify_balance_changes ~txn ~ledger accounts = let verify_balances_unchanged ~(ledger : Helpers.Ledger.t) ~(txn : - Helpers.Transaction_logic.Transaction_applied.Zkapp_command_applied.t ) + Mina_transaction_logic.Transaction_applied.Zkapp_command_applied.t ) (accounts : Test_account.t list) = let is_fee_payer account = Public_key.Compressed.equal account.Test_account.pk diff --git a/src/lib/transaction_logic/test/zkapp_logic.ml b/src/lib/transaction_logic/test/zkapp_logic.ml index 1ee7ad57f88..cf2b585b9b9 100644 --- a/src/lib/transaction_logic/test/zkapp_logic.ml +++ b/src/lib/transaction_logic/test/zkapp_logic.ml @@ -27,7 +27,7 @@ let balance_to_fee = Fn.compose Amount.to_fee Balance.to_amount validate them. *) let%test_module "Test transaction logic." = ( module struct - open Transaction_logic.Transaction_applied.Zkapp_command_applied + open Mina_transaction_logic.Transaction_applied.Zkapp_command_applied let run_zkapp_cmd ~fee_payer ~fee ~accounts txns = let open Result.Let_syntax in diff --git a/src/lib/transaction_snark/test/account_timing/account_timing.ml b/src/lib/transaction_snark/test/account_timing/account_timing.ml index 1d9a6ac594a..13b1464a6ba 100644 --- a/src/lib/transaction_snark/test/account_timing/account_timing.ml +++ b/src/lib/transaction_snark/test/account_timing/account_timing.ml @@ -362,7 +362,7 @@ let%test_module "account timing check" = stack_with_state in let supply_increase = - Mina_ledger.Ledger.Transaction_applied.supply_increase + Mina_transaction_logic.Transaction_applied.supply_increase ~constraint_constants txn_applied |> Or_error.ok_exn in diff --git a/src/lib/transaction_snark/test/transaction_union/transaction_union.ml b/src/lib/transaction_snark/test/transaction_union/transaction_union.ml index c5f9b64705f..3305327afc4 100644 --- a/src/lib/transaction_snark/test/transaction_union/transaction_union.ml +++ b/src/lib/transaction_snark/test/transaction_union/transaction_union.ml @@ -143,7 +143,7 @@ let%test_module "Transaction union tests" = |> Or_error.ok_exn in let supply_increase = - Mina_ledger.Ledger.Transaction_applied.supply_increase + Mina_transaction_logic.Transaction_applied.supply_increase ~constraint_constants applied_transaction |> Or_error.ok_exn in @@ -518,7 +518,7 @@ let%test_module "Transaction union tests" = ( Ledger.apply_user_command ~constraint_constants ledger ~txn_global_slot:current_global_slot t1 |> Or_error.ok_exn - : Ledger.Transaction_applied.Signed_command_applied.t ) ; + : Mina_transaction_logic.Transaction_applied.Signed_command_applied.t ) ; [%test_eq: Frozen_ledger_hash.t] (Ledger.merkle_root ledger) (Sparse_ledger.merkle_root sparse_ledger) ; diff --git a/src/lib/transaction_snark/test/util.ml b/src/lib/transaction_snark/test/util.ml index 6c87b202b14..f4711161da8 100644 --- a/src/lib/transaction_snark/test/util.ml +++ b/src/lib/transaction_snark/test/util.ml @@ -173,7 +173,7 @@ let check_zkapp_command_with_merges_exn ?(logger = logger_null) let open Async.Deferred.Let_syntax in let applied, statement_opt = if ignore_outside_snark then - ( Ledger.Transaction_applied.Varying.Command + ( Mina_transaction_logic.Transaction_applied.Varying.Command (Zkapp_command { command = { With_status.status = Applied @@ -213,8 +213,8 @@ let check_zkapp_command_with_merges_exn ?(logger = logger_null) ; connecting_ledger_right = connecting_ledger ; fee_excess = Zkapp_command.fee_excess zkapp_command ; supply_increase = - Ledger.Transaction_applied.supply_increase - ~constraint_constants applied_txn + Mina_transaction_logic.Transaction_applied + .supply_increase ~constraint_constants applied_txn |> Or_error.ok_exn ; sok_digest = () } @@ -620,7 +620,7 @@ let test_transaction_union ?expected_failure ?txn_global_slot ledger txn = with | Ok res -> ( if Option.is_some expected_failure then - match Ledger.Transaction_applied.transaction_status res with + match Ledger.status_of_applied res with | Applied -> failwith (sprintf "Expected Ledger.apply_transaction to fail with %s" @@ -665,7 +665,8 @@ let test_transaction_union ?expected_failure ?txn_global_slot ledger txn = let supply_increase = Option.value_map applied_transaction ~default:Amount.Signed.zero ~f:(fun txn -> - Ledger.Transaction_applied.supply_increase ~constraint_constants txn + Mina_transaction_logic.Transaction_applied.supply_increase + ~constraint_constants txn |> Or_error.ok_exn ) in match diff --git a/src/lib/transaction_snark/transaction_validator.ml b/src/lib/transaction_snark/transaction_validator.ml index f44091193e6..776221812fb 100644 --- a/src/lib/transaction_snark/transaction_validator.ml +++ b/src/lib/transaction_snark/transaction_validator.ml @@ -16,7 +16,9 @@ let apply_user_command ~constraint_constants ~txn_global_slot l uc = within_mask l ~f:(fun l' -> Result.map ~f:(fun applied_txn -> - applied_txn.Ledger.Transaction_applied.Signed_command_applied.common + applied_txn + .Mina_transaction_logic.Transaction_applied.Signed_command_applied + .common .user_command .status ) (Ledger.apply_user_command l' ~constraint_constants ~txn_global_slot uc) ) diff --git a/src/lib/transaction_snark/transaction_validator.mli b/src/lib/transaction_snark/transaction_validator.mli index d9b3c60e7db..b4555fd4115 100644 --- a/src/lib/transaction_snark/transaction_validator.mli +++ b/src/lib/transaction_snark/transaction_validator.mli @@ -15,7 +15,7 @@ val apply_transactions : -> txn_state_view:Zkapp_precondition.Protocol_state.View.t -> Mina_ledger.Ledger.t -> Transaction.t list - -> Mina_ledger.Ledger.Transaction_applied.t list Or_error.t + -> Mina_transaction_logic.Transaction_applied.t list Or_error.t val apply_transaction_first_pass : constraint_constants:Genesis_constants.Constraint_constants.t diff --git a/src/lib/transaction_snark_scan_state/transaction_snark_scan_state.ml b/src/lib/transaction_snark_scan_state/transaction_snark_scan_state.ml index 47418185c32..a87774932bd 100644 --- a/src/lib/transaction_snark_scan_state/transaction_snark_scan_state.ml +++ b/src/lib/transaction_snark_scan_state/transaction_snark_scan_state.ml @@ -222,7 +222,7 @@ let create_expected_statement ~constraint_constants @@ Sparse_ledger.merkle_root second_pass_ledger_witness in let { With_status.data = transaction; status = _ } = - Ledger.Transaction_applied.transaction transaction_with_info + Ledger.transaction_of_applied transaction_with_info in let%bind protocol_state = get_state (fst state_hash) in let state_view = Mina_state.Protocol_state.Body.view protocol_state.body in @@ -248,8 +248,8 @@ let create_expected_statement ~constraint_constants |> Frozen_ledger_hash.of_ledger_hash in let%map supply_increase = - Ledger.Transaction_applied.supply_increase ~constraint_constants - applied_transaction + Mina_transaction_logic.Transaction_applied.supply_increase + ~constraint_constants applied_transaction in ( target_first_pass_merkle_root , target_second_pass_merkle_root @@ -701,7 +701,7 @@ module Transactions_ordered = struct (txn_with_witness : Transaction_with_witness.t) -> let txn = - Ledger.Transaction_applied.transaction + Ledger.transaction_of_applied txn_with_witness.transaction_with_info in let target_first_pass_ledger = @@ -770,8 +770,7 @@ end let extract_txn_and_global_slot (txn_with_witness : Transaction_with_witness.t) = let txn = - Ledger.Transaction_applied.transaction - txn_with_witness.transaction_with_info + Ledger.transaction_of_applied txn_with_witness.transaction_with_info in let state_hash = fst txn_with_witness.state_hash in let global_slot = txn_with_witness.block_global_slot in @@ -917,7 +916,7 @@ let apply_ordered_txns_stepwise ?(stop_at_first_pass = false) ordered_txns k () | (expected_status, partially_applied_txn) :: partially_applied_txns' -> let%bind res = apply_second_pass ledger partially_applied_txn in - let status = Ledger.Transaction_applied.transaction_status res in + let status = Ledger.status_of_applied res in if Transaction_status.equal expected_status status then Ok (`Continue @@ -1041,8 +1040,7 @@ let apply_ordered_txns_stepwise ?(stop_at_first_pass = false) ordered_txns Previous_incomplete_txns.Unapplied (List.filter txns ~f:(fun txn -> match - (Ledger.Transaction_applied.transaction - txn.transaction_with_info ) + (Ledger.transaction_of_applied txn.transaction_with_info) .data with | Command (Zkapp_command _) -> diff --git a/src/lib/transaction_snark_scan_state/transaction_snark_scan_state.mli b/src/lib/transaction_snark_scan_state/transaction_snark_scan_state.mli index e343e90d419..a896086a228 100644 --- a/src/lib/transaction_snark_scan_state/transaction_snark_scan_state.mli +++ b/src/lib/transaction_snark_scan_state/transaction_snark_scan_state.mli @@ -16,7 +16,7 @@ end] module Transaction_with_witness : sig (* TODO: The statement is redundant here - it can be computed from the witness and the transaction *) type t = - { transaction_with_info : Ledger.Transaction_applied.t + { transaction_with_info : Mina_transaction_logic.Transaction_applied.t ; state_hash : State_hash.t * State_body_hash.t ; statement : Transaction_snark.Statement.t ; init_stack : Transaction_snark.Pending_coinbase_stack_state.Init_stack.t @@ -137,7 +137,7 @@ val get_snarked_ledger_sync : -> apply_second_pass: ( Ledger.t -> Ledger.Transaction_partially_applied.t - -> Ledger.Transaction_applied.t Or_error.t ) + -> Mina_transaction_logic.Transaction_applied.t Or_error.t ) -> apply_first_pass_sparse_ledger: ( global_slot:Mina_numbers.Global_slot_since_genesis.t -> txn_state_view:Mina_base.Zkapp_precondition.Protocol_state.View.t @@ -162,7 +162,7 @@ val get_snarked_ledger_async : -> apply_second_pass: ( Ledger.t -> Ledger.Transaction_partially_applied.t - -> Ledger.Transaction_applied.t Or_error.t ) + -> Mina_transaction_logic.Transaction_applied.t Or_error.t ) -> apply_first_pass_sparse_ledger: ( global_slot:Mina_numbers.Global_slot_since_genesis.t -> txn_state_view:Mina_base.Zkapp_precondition.Protocol_state.View.t @@ -194,7 +194,7 @@ val get_staged_ledger_async : -> apply_second_pass: ( Ledger.t -> Ledger.Transaction_partially_applied.t - -> Ledger.Transaction_applied.t Or_error.t ) + -> Mina_transaction_logic.Transaction_applied.t Or_error.t ) -> apply_first_pass_sparse_ledger: ( global_slot:Mina_numbers.Global_slot_since_genesis.t -> txn_state_view:Mina_base.Zkapp_precondition.Protocol_state.View.t From 7895d6f37fcf8295c9edf4d8f5fb008088f6340d Mon Sep 17 00:00:00 2001 From: georgeee Date: Tue, 22 Oct 2024 10:38:57 +0000 Subject: [PATCH 29/47] Remove unused Mina_block.payments function --- src/lib/mina_block/block.ml | 11 ----------- src/lib/mina_block/block.mli | 2 -- 2 files changed, 13 deletions(-) diff --git a/src/lib/mina_block/block.ml b/src/lib/mina_block/block.ml index ce044757af2..ef25ea79b59 100644 --- a/src/lib/mina_block/block.ml +++ b/src/lib/mina_block/block.ml @@ -102,17 +102,6 @@ let transactions ~constraint_constants block = |> Result.map_error ~f:Staged_ledger.Pre_diff_info.Error.to_error |> Or_error.ok_exn -let payments block = - block |> body |> Staged_ledger_diff.Body.staged_ledger_diff - |> Staged_ledger_diff.commands - |> List.filter_map ~f:(function - | { data = Signed_command ({ payload = { body = Payment _; _ }; _ } as c) - ; status - } -> - Some { With_status.data = c; status } - | _ -> - None ) - let account_ids_accessed ~constraint_constants t = let transactions = transactions ~constraint_constants t in List.map transactions ~f:(fun { data = txn; status } -> diff --git a/src/lib/mina_block/block.mli b/src/lib/mina_block/block.mli index 2fc84e2877f..dbfe00a144f 100644 --- a/src/lib/mina_block/block.mli +++ b/src/lib/mina_block/block.mli @@ -35,8 +35,6 @@ val transactions : -> t -> Transaction.t With_status.t list -val payments : t -> Signed_command.t With_status.t list - val account_ids_accessed : constraint_constants:Genesis_constants.Constraint_constants.t -> t From 3e0597b086a509e315c61f1c4d961417156cd88a Mon Sep 17 00:00:00 2001 From: georgeee Date: Thu, 24 Oct 2024 19:03:19 +0000 Subject: [PATCH 30/47] Use network_id from runtime config in Graphql Problem: value from compile time config was always used, although such a field existed in runtime config as well and was expected to be used. Solution: use value from runtime config when present. --- src/lib/mina_graphql/mina_graphql.ml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/lib/mina_graphql/mina_graphql.ml b/src/lib/mina_graphql/mina_graphql.ml index f39abf50342..c0454234370 100644 --- a/src/lib/mina_graphql/mina_graphql.ml +++ b/src/lib/mina_graphql/mina_graphql.ml @@ -2672,7 +2672,13 @@ module Queries = struct ~args:Arg.[] ~resolve:(fun { ctx = mina; _ } () -> let cfg = Mina_lib.config mina in - "mina:" ^ cfg.compile_config.network_id ) + let runtime_cfg = Mina_lib.runtime_config mina in + let network_id = + Option.value ~default:cfg.compile_config.network_id + @@ let%bind.Option daemon = runtime_cfg.daemon in + daemon.network_id + in + "mina:" ^ network_id ) let signature_kind = field "signatureKind" From 4ef596a551d1af3dbdc2094090a76383e5d7a0d6 Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Mon, 28 Oct 2024 11:22:08 +0000 Subject: [PATCH 31/47] Reformat --- src/lib/transaction_logic/test/helpers.ml | 6 ++++-- src/lib/transaction_logic/test/predicates.ml | 6 ++---- .../test/transaction_union/transaction_union.ml | 4 +++- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/src/lib/transaction_logic/test/helpers.ml b/src/lib/transaction_logic/test/helpers.ml index 536087f71e2..3ed25abdf71 100644 --- a/src/lib/transaction_logic/test/helpers.ml +++ b/src/lib/transaction_logic/test/helpers.ml @@ -3,8 +3,10 @@ module Transaction_logic = Mina_transaction_logic.Make (Ledger) module Zk_cmd_result = struct type t = - Mina_transaction_logic.Transaction_applied.Zkapp_command_applied.t * Ledger.t + Mina_transaction_logic.Transaction_applied.Zkapp_command_applied.t + * Ledger.t let sexp_of_t (txn, _) = - Mina_transaction_logic.Transaction_applied.Zkapp_command_applied.sexp_of_t txn + Mina_transaction_logic.Transaction_applied.Zkapp_command_applied.sexp_of_t + txn end diff --git a/src/lib/transaction_logic/test/predicates.ml b/src/lib/transaction_logic/test/predicates.ml index e5d69c64a16..0fbb9fe2d7b 100644 --- a/src/lib/transaction_logic/test/predicates.ml +++ b/src/lib/transaction_logic/test/predicates.ml @@ -12,8 +12,7 @@ let result ?(with_error = Fn.const false) ~f result = match Result.bind result ~f with Ok b -> b | Error e -> with_error e let verify_account_updates ~(ledger : Helpers.Ledger.t) - ~(txn : - Mina_transaction_logic.Transaction_applied.Zkapp_command_applied.t ) + ~(txn : Mina_transaction_logic.Transaction_applied.Zkapp_command_applied.t) ~(f : Amount.Signed.t -> Account.t option * Account.t option -> bool) (account : Test_account.t) = let open Helpers in @@ -80,8 +79,7 @@ let verify_balance_changes ~txn ~ledger accounts = false ) ) let verify_balances_unchanged ~(ledger : Helpers.Ledger.t) - ~(txn : - Mina_transaction_logic.Transaction_applied.Zkapp_command_applied.t ) + ~(txn : Mina_transaction_logic.Transaction_applied.Zkapp_command_applied.t) (accounts : Test_account.t list) = let is_fee_payer account = Public_key.Compressed.equal account.Test_account.pk diff --git a/src/lib/transaction_snark/test/transaction_union/transaction_union.ml b/src/lib/transaction_snark/test/transaction_union/transaction_union.ml index 3305327afc4..2b9dabd86ee 100644 --- a/src/lib/transaction_snark/test/transaction_union/transaction_union.ml +++ b/src/lib/transaction_snark/test/transaction_union/transaction_union.ml @@ -518,7 +518,9 @@ let%test_module "Transaction union tests" = ( Ledger.apply_user_command ~constraint_constants ledger ~txn_global_slot:current_global_slot t1 |> Or_error.ok_exn - : Mina_transaction_logic.Transaction_applied.Signed_command_applied.t ) ; + : Mina_transaction_logic.Transaction_applied + .Signed_command_applied + .t ) ; [%test_eq: Frozen_ledger_hash.t] (Ledger.merkle_root ledger) (Sparse_ledger.merkle_root sparse_ledger) ; From 690470dfb4405b9ce136633df44bd4ca57171b54 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 28 Oct 2024 12:28:00 +0100 Subject: [PATCH 32/47] readme things --- .../kimchi_bindings/js/bindings/README.md | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 src/lib/crypto/kimchi_bindings/js/bindings/README.md diff --git a/src/lib/crypto/kimchi_bindings/js/bindings/README.md b/src/lib/crypto/kimchi_bindings/js/bindings/README.md new file mode 100644 index 00000000000..c6c2ee2a8d6 --- /dev/null +++ b/src/lib/crypto/kimchi_bindings/js/bindings/README.md @@ -0,0 +1,88 @@ +**Despite popular belief, the bindings files are not auto-generated, they just look funny.** + +In OCaml, we sometimes call out to foreign functions (that is usually indicated by the `external` keyword), here's an example: + +```ml +module FunnyLittleModule = struct + external do_cool_thingies : unit -> unit = "caml_do_cool_thingies" +end +``` + +This way, when calling the function `FunnyLittleModule.do_cool_thingies`, we tell OCaml that the implementation for `do_cool_thingies` is actually somewhere else, and not in OCaml directly. That other place can, as in our case, be in Rust! So whenever we call `FunnyLittleModule.do_cool_thingies`, we tell OCaml under the hood to look for an external function, in our case somewhere in the Rust bindings, that is called `caml_do_cool_thingies`, and executes it. + +We use this for many things. Many things in the code base rely of implementations in Rust. For example, we use Kimchi to generate proofs! So in order to tell OCaml to generate a Kimchi proof, we need to point it to the correct function that's living in the Rust proof-systems repository. + +The other side of the `external` keyword is somewhere in the Rust bindings layer, more specifically somewhere in `src/lib/crypto/kimchi_bindings/wasm/src` - in our case where we want to establish bindings between OCaml that has been compiled to JavaScript using JSOO and Rust (compiled to WASM). + +For example, the implementation of `caml_do_cool_thingies` could look like this: + +```rs +#[wasm_bindgen] +pub fn caml_do_cool_thingies() { + do_more_funny_things(); +} +``` + +`#[wasm_bindgen]` indicates Rust that we want to compile the code to WASM and use the function there. +`pub fn caml_do_cool_thingies()` is the name of our "external" function that we are looking for in our OCaml module. + +There's one step left! Since we are compiling OCaml to JavaScript using JSOO, we need to tell JSOO how to connect these `external` functions and where to look for them. That's where all these funny little bindings files come in. When compiling OCaml, we tell JSOO to "look at these functions for their correct implementation" - this means we have to write these bindings files to "proxy" OCaml's `external` functions to their implementation. These implementations can be in JavaScript directly, for example something like this + +```js +// Provides: caml_do_cool_thingies +function caml_do_cool_thingies() { + assert(1 + 1 === 2); +} +``` + +The comment above the function actually tells JSOO what `external` function it _provides_! This way JSOO knows how to connect `external` functions to their implementation. The comments used here have their own little syntax, I would recommend you to check it out in the JSOO docs. + +In our case, however, the implementation of the function isn't directly in JavaScript - it is in Rust compiled to WASM! So what we have to do is use these bindings files to point the implementation to WASM, we usually do this by injecting a WASM object or proxy into our bindings layer (see `../web/web_backend.js` and `../node_js/node_backend.js` for their web and node implementations respectively). + +We then use this WASM object and "inject" it into our proxy in order to use it. + +```js +// Provides: caml_do_cool_thingies +// Requires: plonk_wasm +function caml_do_cool_thingies() { + plonk_wasm.caml_do_cool_thingies(); +} +``` + +So now instead of using the implementation in JavaScript, we directly call into the Rust implementation that has been compiled to WASM! This means, whenever something in OCaml invokes `FunnyLittleModule.do_cool_thingies` it automatically resolves to `caml_do_cool_thingies` in Rust compiled to WASM. + +Previously, these bindings were in one single file `bindings.js` which made it hard to understand. Now, bindings are split into separate files, each with their own responsibilities. + +Sometimes, these "proxy" functions actually don't call into WASM directly, but do some pre-computation, like this example: + +```js +// Provides: caml_pasta_fp_plonk_proof_create +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fp_plonk_proof_create = function ( + index, + witness_cols, + caml_runtime_tables, + prev_challenges, + prev_sgs +) { + var w = new plonk_wasm.WasmVecVecFp(witness_cols.length - 1); + for (var i = 1; i < witness_cols.length; i++) { + w.push(tsRustConversion.fp.vectorToRust(witness_cols[i])); + } + witness_cols = w; + prev_challenges = tsRustConversion.fp.vectorToRust(prev_challenges); + var wasm_runtime_tables = + tsRustConversion.fp.runtimeTablesToRust(caml_runtime_tables); + prev_sgs = tsRustConversion.fp.pointsToRust(prev_sgs); + var proof = plonk_wasm.caml_pasta_fp_plonk_proof_create( + index, + witness_cols, + wasm_runtime_tables, + prev_challenges, + prev_sgs + ); + return tsRustConversion.fp.proofFromRust(proof); +}; +``` + +So just keep in mind that sometimes it's not as easy to just forward the implementation to WASM and occasionally some more work needs to be done :) From 4b83686cf89f79f2533230c1ce60aa43d3178f62 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 28 Oct 2024 12:29:06 +0100 Subject: [PATCH 33/47] fix original readme --- README.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b848104e1d8..fdfe7bbfdd4 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,24 @@ ### Build status +<<<<<<< HEAD | Develop | Berkeley | Compatible | | ------- | -------- | ---------- | | [![Build status - develop](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=develop)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - berkeley](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=berkeley)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - compatible](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=compatible)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) +======= +| Develop | Compatible | Master | +| ------- | ---------- | ---------- | +| [![Build status - develop](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=develop)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - compatible](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=compatible)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - master](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=master)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) + +
+
+
+>>>>>>> 3a2561d10c (update badges and logo for README.md) - Mina logo + Mina logo -# Mina + Mina is the first cryptocurrency with a lightweight, constant-sized blockchain. This is the main source code repository for the Mina project and contains code for the OCaml protocol implementation, the [Mina Protocol website](https://minaprotocol.com), and wallet. Enjoy! From b2e1c207940717d3131b1ccd72e526ff67ea300c Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 28 Oct 2024 12:33:36 +0100 Subject: [PATCH 34/47] Revert "fix original readme" This reverts commit 4b83686cf89f79f2533230c1ce60aa43d3178f62. --- README.md | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index fdfe7bbfdd4..b848104e1d8 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,14 @@ ### Build status -<<<<<<< HEAD | Develop | Berkeley | Compatible | | ------- | -------- | ---------- | | [![Build status - develop](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=develop)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - berkeley](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=berkeley)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - compatible](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=compatible)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) -======= -| Develop | Compatible | Master | -| ------- | ---------- | ---------- | -| [![Build status - develop](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=develop)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - compatible](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=compatible)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - master](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=master)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) - -
-
-
->>>>>>> 3a2561d10c (update badges and logo for README.md) - Mina logo + Mina logo - +# Mina Mina is the first cryptocurrency with a lightweight, constant-sized blockchain. This is the main source code repository for the Mina project and contains code for the OCaml protocol implementation, the [Mina Protocol website](https://minaprotocol.com), and wallet. Enjoy! From 392626d4b9fbf553cef439b71860ca9fe2df20f5 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 28 Oct 2024 12:36:02 +0100 Subject: [PATCH 35/47] fix original readme for real --- src/lib/crypto/kimchi_bindings/js/README.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/lib/crypto/kimchi_bindings/js/README.md b/src/lib/crypto/kimchi_bindings/js/README.md index 8ead124f985..d0c32139c67 100644 --- a/src/lib/crypto/kimchi_bindings/js/README.md +++ b/src/lib/crypto/kimchi_bindings/js/README.md @@ -1,11 +1,10 @@ This library provides a wrapper around the WebAssembly prover code, which allows `js_of_ocaml` to compile the mina project against the WebAssembly -backend. +backend. This means that `external` OCaml functions now know what implementation to point to. See `./bindings/README.md` for more details. The different versions of the backend are generated in subdirectories; e.g. the NodeJS backend is generated in `node_js/` and the Web backend is generated -in `web/`. To use a backend, run `dune build backend/plonk_wasm.js` and copy -`backend/plonk_wasm*` to the project directory. +in `web/`. To use a backend, run `dune build **backend**/plonk_wasm.js` (where `**backend**` is either `web` or `node_js`) and copy `**backend**/plonk_wasm*` to the project directory. Note that the backend code is not automatically compiled while linking against the backend library. You should always manually issue a build command for the @@ -14,15 +13,15 @@ For example, to run the nodejs tests in the `test/nodejs` directory you will need to run ``` -dune build src/lib/marlin_plonk_bindings/js/test/nodejs/nodejs_test.bc.js -src/lib/marlin_plonk_bindings/js/test/nodejs/copy_over.sh +dune build src/lib/crypto/kimchi_bindings/js/test/nodejs/nodejs_test.bc.js +src/lib/crypto/kimchi_bindings/js/test/nodejs/copy_over.sh ``` Similarly, to run the web tests in `test/web`, you can run ``` -dune build src/lib/marlin_plonk_bindings/js/test/web/web_test.bc.js -src/lib/marlin_plonk_bindings/js/test/web/copy_over.sh +dune build src/lib/crypto/kimchi_bindings/js/test/web/web_test.bc.js +src/lib/crypto/kimchi_bindings/js/test/web/copy_over.sh ``` and then visit `http://localhost:8000` from a browser. From 92331547da690472d09ddb9501c50f9e99be6371 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 28 Oct 2024 12:43:12 +0100 Subject: [PATCH 36/47] move already isolated files --- .../js/{bindings-bigint256.js => bindings/bigint256.js} | 0 .../js/{bindings-curve.js => bindings/curve.js} | 0 .../js/{bindings-field.js => bindings/field.js} | 0 .../js/{bindings-vector.js => bindings/vector.js} | 0 src/lib/crypto/kimchi_bindings/js/dune | 8 ++++---- 5 files changed, 4 insertions(+), 4 deletions(-) rename src/lib/crypto/kimchi_bindings/js/{bindings-bigint256.js => bindings/bigint256.js} (100%) rename src/lib/crypto/kimchi_bindings/js/{bindings-curve.js => bindings/curve.js} (100%) rename src/lib/crypto/kimchi_bindings/js/{bindings-field.js => bindings/field.js} (100%) rename src/lib/crypto/kimchi_bindings/js/{bindings-vector.js => bindings/vector.js} (100%) diff --git a/src/lib/crypto/kimchi_bindings/js/bindings-bigint256.js b/src/lib/crypto/kimchi_bindings/js/bindings/bigint256.js similarity index 100% rename from src/lib/crypto/kimchi_bindings/js/bindings-bigint256.js rename to src/lib/crypto/kimchi_bindings/js/bindings/bigint256.js diff --git a/src/lib/crypto/kimchi_bindings/js/bindings-curve.js b/src/lib/crypto/kimchi_bindings/js/bindings/curve.js similarity index 100% rename from src/lib/crypto/kimchi_bindings/js/bindings-curve.js rename to src/lib/crypto/kimchi_bindings/js/bindings/curve.js diff --git a/src/lib/crypto/kimchi_bindings/js/bindings-field.js b/src/lib/crypto/kimchi_bindings/js/bindings/field.js similarity index 100% rename from src/lib/crypto/kimchi_bindings/js/bindings-field.js rename to src/lib/crypto/kimchi_bindings/js/bindings/field.js diff --git a/src/lib/crypto/kimchi_bindings/js/bindings-vector.js b/src/lib/crypto/kimchi_bindings/js/bindings/vector.js similarity index 100% rename from src/lib/crypto/kimchi_bindings/js/bindings-vector.js rename to src/lib/crypto/kimchi_bindings/js/bindings/vector.js diff --git a/src/lib/crypto/kimchi_bindings/js/dune b/src/lib/crypto/kimchi_bindings/js/dune index f8746ce705d..664453fa8dc 100644 --- a/src/lib/crypto/kimchi_bindings/js/dune +++ b/src/lib/crypto/kimchi_bindings/js/dune @@ -4,10 +4,10 @@ (js_of_ocaml (javascript_files bindings.js - bindings-bigint256.js - bindings-field.js - bindings-curve.js - bindings-vector.js)) + bindings/bigint256.js + bindings/field.js + bindings/curve.js + bindings/vector.js)) (instrumentation (backend bisect_ppx)) (preprocess From 26f9dc2acee16498c0d64ff21ecae1d4b10f13b1 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 28 Oct 2024 12:47:36 +0100 Subject: [PATCH 37/47] move gate-vector --- src/lib/crypto/kimchi_bindings/js/bindings.js | 120 ----------------- .../js/bindings/gate-vector.js | 122 ++++++++++++++++++ src/lib/crypto/kimchi_bindings/js/dune | 3 +- 3 files changed, 124 insertions(+), 121 deletions(-) create mode 100644 src/lib/crypto/kimchi_bindings/js/bindings/gate-vector.js diff --git a/src/lib/crypto/kimchi_bindings/js/bindings.js b/src/lib/crypto/kimchi_bindings/js/bindings.js index 32907233043..adf33395ae5 100644 --- a/src/lib/crypto/kimchi_bindings/js/bindings.js +++ b/src/lib/crypto/kimchi_bindings/js/bindings.js @@ -273,128 +273,8 @@ var caml_fq_srs_h = function (t) { // Requires: tsSrs var caml_fq_srs_add_lagrange_basis = tsSrs.fq.addLagrangeBasis; -// gate vector - -// Provides: caml_pasta_fp_plonk_gate_vector_create -// Requires: plonk_wasm, free_on_finalize -var caml_pasta_fp_plonk_gate_vector_create = function () { - return free_on_finalize(plonk_wasm.caml_pasta_fp_plonk_gate_vector_create()); -}; - -// Provides: caml_pasta_fp_plonk_gate_vector_add -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fp_plonk_gate_vector_add = function (v, x) { - return plonk_wasm.caml_pasta_fp_plonk_gate_vector_add( - v, - tsRustConversion.fp.gateToRust(x) - ); -}; - -// Provides: caml_pasta_fp_plonk_gate_vector_get -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fp_plonk_gate_vector_get = function (v, i) { - return tsRustConversion.fp.gateFromRust( - plonk_wasm.caml_pasta_fp_plonk_gate_vector_get(v, i) - ); -}; - -// Provides: caml_pasta_fp_plonk_gate_vector_len -// Requires: plonk_wasm -var caml_pasta_fp_plonk_gate_vector_len = function (v) { - return plonk_wasm.caml_pasta_fp_plonk_gate_vector_len(v); -}; - -// Provides: caml_pasta_fp_plonk_gate_vector_wrap -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fp_plonk_gate_vector_wrap = function (v, x, y) { - return plonk_wasm.caml_pasta_fp_plonk_gate_vector_wrap( - v, - tsRustConversion.wireToRust(x), - tsRustConversion.wireToRust(y) - ); -}; - -// Provides: caml_pasta_fp_plonk_gate_vector_digest -// Requires: plonk_wasm, caml_bytes_of_uint8array -var caml_pasta_fp_plonk_gate_vector_digest = function ( - public_input_size, - gate_vector -) { - var uint8array = plonk_wasm.caml_pasta_fp_plonk_gate_vector_digest( - public_input_size, - gate_vector - ); - return caml_bytes_of_uint8array(uint8array); -}; - -// Provides: caml_pasta_fp_plonk_circuit_serialize -// Requires: plonk_wasm, caml_string_of_jsstring -var caml_pasta_fp_plonk_circuit_serialize = function ( - public_input_size, - gate_vector -) { - return caml_string_of_jsstring( - plonk_wasm.caml_pasta_fp_plonk_circuit_serialize( - public_input_size, - gate_vector - ) - ); -}; - // prover index -// Provides: caml_pasta_fq_plonk_gate_vector_create -// Requires: plonk_wasm, free_on_finalize -var caml_pasta_fq_plonk_gate_vector_create = function () { - return free_on_finalize(plonk_wasm.caml_pasta_fq_plonk_gate_vector_create()); -}; - -// Provides: caml_pasta_fq_plonk_gate_vector_add -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fq_plonk_gate_vector_add = function (v, x) { - return plonk_wasm.caml_pasta_fq_plonk_gate_vector_add( - v, - tsRustConversion.fq.gateToRust(x) - ); -}; - -// Provides: caml_pasta_fq_plonk_gate_vector_get -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fq_plonk_gate_vector_get = function (v, i) { - return tsRustConversion.fq.gateFromRust( - plonk_wasm.caml_pasta_fq_plonk_gate_vector_get(v, i) - ); -}; - -// Provides: caml_pasta_fq_plonk_gate_vector_len -// Requires: plonk_wasm -var caml_pasta_fq_plonk_gate_vector_len = function (v) { - return plonk_wasm.caml_pasta_fq_plonk_gate_vector_len(v); -}; - -// Provides: caml_pasta_fq_plonk_gate_vector_wrap -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fq_plonk_gate_vector_wrap = function (v, x, y) { - return plonk_wasm.caml_pasta_fq_plonk_gate_vector_wrap( - v, - tsRustConversion.wireToRust(x), - tsRustConversion.wireToRust(y) - ); -}; - -// Provides: caml_pasta_fq_plonk_gate_vector_digest -// Requires: plonk_wasm, caml_bytes_of_uint8array -var caml_pasta_fq_plonk_gate_vector_digest = function ( - public_input_size, - gate_vector -) { - var uint8array = plonk_wasm.caml_pasta_fq_plonk_gate_vector_digest( - public_input_size, - gate_vector - ); - return caml_bytes_of_uint8array(uint8array); -}; - // Provides: caml_pasta_fq_plonk_circuit_serialize // Requires: plonk_wasm, caml_string_of_jsstring var caml_pasta_fq_plonk_circuit_serialize = function ( diff --git a/src/lib/crypto/kimchi_bindings/js/bindings/gate-vector.js b/src/lib/crypto/kimchi_bindings/js/bindings/gate-vector.js new file mode 100644 index 00000000000..4a4d0d04c37 --- /dev/null +++ b/src/lib/crypto/kimchi_bindings/js/bindings/gate-vector.js @@ -0,0 +1,122 @@ +/* eslint-disable no-unused-vars */ +/* global plonk_wasm, caml_string_of_jsstring, + free_on_finalize, tsRustConversion, caml_bytes_of_uint8array +*/ + +// Provides: caml_pasta_fp_plonk_gate_vector_create +// Requires: plonk_wasm, free_on_finalize +var caml_pasta_fp_plonk_gate_vector_create = function () { + return free_on_finalize(plonk_wasm.caml_pasta_fp_plonk_gate_vector_create()); +}; + +// Provides: caml_pasta_fp_plonk_gate_vector_add +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fp_plonk_gate_vector_add = function (v, x) { + return plonk_wasm.caml_pasta_fp_plonk_gate_vector_add( + v, + tsRustConversion.fp.gateToRust(x) + ); +}; + +// Provides: caml_pasta_fp_plonk_gate_vector_get +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fp_plonk_gate_vector_get = function (v, i) { + return tsRustConversion.fp.gateFromRust( + plonk_wasm.caml_pasta_fp_plonk_gate_vector_get(v, i) + ); +}; + +// Provides: caml_pasta_fp_plonk_gate_vector_len +// Requires: plonk_wasm +var caml_pasta_fp_plonk_gate_vector_len = function (v) { + return plonk_wasm.caml_pasta_fp_plonk_gate_vector_len(v); +}; + +// Provides: caml_pasta_fp_plonk_gate_vector_wrap +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fp_plonk_gate_vector_wrap = function (v, x, y) { + return plonk_wasm.caml_pasta_fp_plonk_gate_vector_wrap( + v, + tsRustConversion.wireToRust(x), + tsRustConversion.wireToRust(y) + ); +}; + +// Provides: caml_pasta_fp_plonk_gate_vector_digest +// Requires: plonk_wasm, caml_bytes_of_uint8array +var caml_pasta_fp_plonk_gate_vector_digest = function ( + public_input_size, + gate_vector +) { + var uint8array = plonk_wasm.caml_pasta_fp_plonk_gate_vector_digest( + public_input_size, + gate_vector + ); + return caml_bytes_of_uint8array(uint8array); +}; + +// Provides: caml_pasta_fp_plonk_circuit_serialize +// Requires: plonk_wasm, caml_string_of_jsstring +var caml_pasta_fp_plonk_circuit_serialize = function ( + public_input_size, + gate_vector +) { + return caml_string_of_jsstring( + plonk_wasm.caml_pasta_fp_plonk_circuit_serialize( + public_input_size, + gate_vector + ) + ); +}; + +// Provides: caml_pasta_fq_plonk_gate_vector_create +// Requires: plonk_wasm, free_on_finalize +var caml_pasta_fq_plonk_gate_vector_create = function () { + return free_on_finalize(plonk_wasm.caml_pasta_fq_plonk_gate_vector_create()); +}; + +// Provides: caml_pasta_fq_plonk_gate_vector_add +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fq_plonk_gate_vector_add = function (v, x) { + return plonk_wasm.caml_pasta_fq_plonk_gate_vector_add( + v, + tsRustConversion.fq.gateToRust(x) + ); +}; + +// Provides: caml_pasta_fq_plonk_gate_vector_get +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fq_plonk_gate_vector_get = function (v, i) { + return tsRustConversion.fq.gateFromRust( + plonk_wasm.caml_pasta_fq_plonk_gate_vector_get(v, i) + ); +}; + +// Provides: caml_pasta_fq_plonk_gate_vector_len +// Requires: plonk_wasm +var caml_pasta_fq_plonk_gate_vector_len = function (v) { + return plonk_wasm.caml_pasta_fq_plonk_gate_vector_len(v); +}; + +// Provides: caml_pasta_fq_plonk_gate_vector_wrap +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fq_plonk_gate_vector_wrap = function (v, x, y) { + return plonk_wasm.caml_pasta_fq_plonk_gate_vector_wrap( + v, + tsRustConversion.wireToRust(x), + tsRustConversion.wireToRust(y) + ); +}; + +// Provides: caml_pasta_fq_plonk_gate_vector_digest +// Requires: plonk_wasm, caml_bytes_of_uint8array +var caml_pasta_fq_plonk_gate_vector_digest = function ( + public_input_size, + gate_vector +) { + var uint8array = plonk_wasm.caml_pasta_fq_plonk_gate_vector_digest( + public_input_size, + gate_vector + ); + return caml_bytes_of_uint8array(uint8array); +}; diff --git a/src/lib/crypto/kimchi_bindings/js/dune b/src/lib/crypto/kimchi_bindings/js/dune index 664453fa8dc..38ec61f81cc 100644 --- a/src/lib/crypto/kimchi_bindings/js/dune +++ b/src/lib/crypto/kimchi_bindings/js/dune @@ -7,7 +7,8 @@ bindings/bigint256.js bindings/field.js bindings/curve.js - bindings/vector.js)) + bindings/vector.js + bindings/gate-vector.js)) (instrumentation (backend bisect_ppx)) (preprocess From a942d121919f7ca11b18e7e7968e4209e98b18a7 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 28 Oct 2024 12:51:19 +0100 Subject: [PATCH 38/47] move oracle/poseidon --- src/lib/crypto/kimchi_bindings/js/bindings.js | 104 ----------------- .../kimchi_bindings/js/bindings/oracles.js | 105 ++++++++++++++++++ src/lib/crypto/kimchi_bindings/js/dune | 3 +- 3 files changed, 107 insertions(+), 105 deletions(-) create mode 100644 src/lib/crypto/kimchi_bindings/js/bindings/oracles.js diff --git a/src/lib/crypto/kimchi_bindings/js/bindings.js b/src/lib/crypto/kimchi_bindings/js/bindings.js index adf33395ae5..616e863b787 100644 --- a/src/lib/crypto/kimchi_bindings/js/bindings.js +++ b/src/lib/crypto/kimchi_bindings/js/bindings.js @@ -784,110 +784,6 @@ var caml_pasta_fq_plonk_proof_deep_copy = function (proof) { ); }; -// oracles - -// Provides: fp_oracles_create -// Requires: plonk_wasm, tsRustConversion -var fp_oracles_create = function (lgr_comm, verifier_index, proof) { - return tsRustConversion.fp.oraclesFromRust( - plonk_wasm.fp_oracles_create( - tsRustConversion.fp.polyCommsToRust(lgr_comm), - tsRustConversion.fp.verifierIndexToRust(verifier_index), - tsRustConversion.fp.proofToRust(proof) - ) - ); -}; - -// Provides: fp_oracles_create_no_public -// Requires: fp_oracles_create -var fp_oracles_create_no_public = function (lgr_comm, verifier_index, proof) { - return fp_oracles_create(lgr_comm, verifier_index, [0, 0, proof]); -}; - -// Provides: fp_oracles_dummy -// Requires: plonk_wasm, tsRustConversion -var fp_oracles_dummy = function () { - return tsRustConversion.fp.oraclesFromRust(plonk_wasm.fp_oracles_dummy()); -}; - -// Provides: fp_oracles_deep_copy -// Requires: plonk_wasm, tsRustConversion -var fp_oracles_deep_copy = function (x) { - return tsRustConversion.fp.oraclesFromRust( - plonk_wasm.fp_oracles_deep_copy(tsRustConversion.fp.oraclesToRust(x)) - ); -}; - -// Provides: fq_oracles_create -// Requires: plonk_wasm, tsRustConversion -var fq_oracles_create = function (lgr_comm, verifier_index, proof) { - return tsRustConversion.fq.oraclesFromRust( - plonk_wasm.fq_oracles_create( - tsRustConversion.fq.polyCommsToRust(lgr_comm), - tsRustConversion.fq.verifierIndexToRust(verifier_index), - tsRustConversion.fq.proofToRust(proof) - ) - ); -}; - -// Provides: fq_oracles_create_no_public -// Requires: fq_oracles_create -var fq_oracles_create_no_public = function (lgr_comm, verifier_index, proof) { - return fq_oracles_create(lgr_comm, verifier_index, [0, 0, proof]); -}; - -// Provides: fq_oracles_dummy -// Requires: plonk_wasm, tsRustConversion -var fq_oracles_dummy = function () { - return tsRustConversion.fq.oraclesFromRust(plonk_wasm.fq_oracles_dummy()); -}; - -// Provides: fq_oracles_deep_copy -// Requires: plonk_wasm, tsRustConversion -var fq_oracles_deep_copy = function (x) { - return tsRustConversion.fq.oraclesFromRust( - plonk_wasm.fq_oracles_deep_copy(tsRustConversion.fq.oraclesToRust(x)) - ); -}; - -// This is fake -- parameters are only needed on the Rust side, so no need to return something meaningful -// Provides: caml_pasta_fp_poseidon_params_create -function caml_pasta_fp_poseidon_params_create() { - return [0]; -} -// Provides: caml_pasta_fq_poseidon_params_create -function caml_pasta_fq_poseidon_params_create() { - return [0]; -} - -// Provides: caml_pasta_fp_poseidon_block_cipher -// Requires: plonk_wasm, tsRustConversion, tsRustConversion -function caml_pasta_fp_poseidon_block_cipher(_fake_params, fp_vector) { - // 1. get permuted field vector from rust - var wasm_flat_vector = plonk_wasm.caml_pasta_fp_poseidon_block_cipher( - tsRustConversion.fp.vectorToRust(fp_vector) - ); - var new_fp_vector = tsRustConversion.fp.vectorFromRust(wasm_flat_vector); - // 2. write back modified field vector to original one - new_fp_vector.forEach(function (a, i) { - fp_vector[i] = a; - }); -} - -// Provides: caml_pasta_fq_poseidon_block_cipher -// Requires: plonk_wasm, tsRustConversion, tsRustConversion -function caml_pasta_fq_poseidon_block_cipher(_fake_params, fq_vector) { - // 1. get permuted field vector from rust - var wasm_flat_vector = plonk_wasm.caml_pasta_fq_poseidon_block_cipher( - tsRustConversion.fq.vectorToRust(fq_vector) - ); - var new_fq_vector = tsRustConversion.fq.vectorFromRust(wasm_flat_vector); - // 2. write back modified field vector to original one - new_fq_vector.forEach(function (a, i) { - fq_vector[i] = a; - }); -} - // Provides: caml_pasta_fp_plonk_proof_example_with_lookup function caml_pasta_fp_plonk_proof_example_with_lookup() { // This is only used in the pickles unit tests diff --git a/src/lib/crypto/kimchi_bindings/js/bindings/oracles.js b/src/lib/crypto/kimchi_bindings/js/bindings/oracles.js new file mode 100644 index 00000000000..1411fd73613 --- /dev/null +++ b/src/lib/crypto/kimchi_bindings/js/bindings/oracles.js @@ -0,0 +1,105 @@ +/* global plonk_wasm, tsRustConversion, + + */ + +// Provides: fp_oracles_create +// Requires: plonk_wasm, tsRustConversion +var fp_oracles_create = function (lgr_comm, verifier_index, proof) { + return tsRustConversion.fp.oraclesFromRust( + plonk_wasm.fp_oracles_create( + tsRustConversion.fp.polyCommsToRust(lgr_comm), + tsRustConversion.fp.verifierIndexToRust(verifier_index), + tsRustConversion.fp.proofToRust(proof) + ) + ); +}; + +// Provides: fp_oracles_create_no_public +// Requires: fp_oracles_create +var fp_oracles_create_no_public = function (lgr_comm, verifier_index, proof) { + return fp_oracles_create(lgr_comm, verifier_index, [0, 0, proof]); +}; + +// Provides: fp_oracles_dummy +// Requires: plonk_wasm, tsRustConversion +var fp_oracles_dummy = function () { + return tsRustConversion.fp.oraclesFromRust(plonk_wasm.fp_oracles_dummy()); +}; + +// Provides: fp_oracles_deep_copy +// Requires: plonk_wasm, tsRustConversion +var fp_oracles_deep_copy = function (x) { + return tsRustConversion.fp.oraclesFromRust( + plonk_wasm.fp_oracles_deep_copy(tsRustConversion.fp.oraclesToRust(x)) + ); +}; + +// Provides: fq_oracles_create +// Requires: plonk_wasm, tsRustConversion +var fq_oracles_create = function (lgr_comm, verifier_index, proof) { + return tsRustConversion.fq.oraclesFromRust( + plonk_wasm.fq_oracles_create( + tsRustConversion.fq.polyCommsToRust(lgr_comm), + tsRustConversion.fq.verifierIndexToRust(verifier_index), + tsRustConversion.fq.proofToRust(proof) + ) + ); +}; + +// Provides: fq_oracles_create_no_public +// Requires: fq_oracles_create +var fq_oracles_create_no_public = function (lgr_comm, verifier_index, proof) { + return fq_oracles_create(lgr_comm, verifier_index, [0, 0, proof]); +}; + +// Provides: fq_oracles_dummy +// Requires: plonk_wasm, tsRustConversion +var fq_oracles_dummy = function () { + return tsRustConversion.fq.oraclesFromRust(plonk_wasm.fq_oracles_dummy()); +}; + +// Provides: fq_oracles_deep_copy +// Requires: plonk_wasm, tsRustConversion +var fq_oracles_deep_copy = function (x) { + return tsRustConversion.fq.oraclesFromRust( + plonk_wasm.fq_oracles_deep_copy(tsRustConversion.fq.oraclesToRust(x)) + ); +}; + +// This is fake -- parameters are only needed on the Rust side, so no need to return something meaningful +// Provides: caml_pasta_fp_poseidon_params_create +function caml_pasta_fp_poseidon_params_create() { + return [0]; +} +// Provides: caml_pasta_fq_poseidon_params_create +function caml_pasta_fq_poseidon_params_create() { + return [0]; +} + +// Provides: caml_pasta_fp_poseidon_block_cipher +// Requires: plonk_wasm, tsRustConversion, tsRustConversion +function caml_pasta_fp_poseidon_block_cipher(_fake_params, fp_vector) { + // 1. get permuted field vector from rust + var wasm_flat_vector = plonk_wasm.caml_pasta_fp_poseidon_block_cipher( + tsRustConversion.fp.vectorToRust(fp_vector) + ); + var new_fp_vector = tsRustConversion.fp.vectorFromRust(wasm_flat_vector); + // 2. write back modified field vector to original one + new_fp_vector.forEach(function (a, i) { + fp_vector[i] = a; + }); +} + +// Provides: caml_pasta_fq_poseidon_block_cipher +// Requires: plonk_wasm, tsRustConversion, tsRustConversion +function caml_pasta_fq_poseidon_block_cipher(_fake_params, fq_vector) { + // 1. get permuted field vector from rust + var wasm_flat_vector = plonk_wasm.caml_pasta_fq_poseidon_block_cipher( + tsRustConversion.fq.vectorToRust(fq_vector) + ); + var new_fq_vector = tsRustConversion.fq.vectorFromRust(wasm_flat_vector); + // 2. write back modified field vector to original one + new_fq_vector.forEach(function (a, i) { + fq_vector[i] = a; + }); +} diff --git a/src/lib/crypto/kimchi_bindings/js/dune b/src/lib/crypto/kimchi_bindings/js/dune index 38ec61f81cc..20e5c8aa7a9 100644 --- a/src/lib/crypto/kimchi_bindings/js/dune +++ b/src/lib/crypto/kimchi_bindings/js/dune @@ -8,7 +8,8 @@ bindings/field.js bindings/curve.js bindings/vector.js - bindings/gate-vector.js)) + bindings/gate-vector.js + bindings/oracles.js)) (instrumentation (backend bisect_ppx)) (preprocess From e37e3ee121f6903ccc704b1becbcdd4b95232e7b Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 28 Oct 2024 12:54:41 +0100 Subject: [PATCH 39/47] move test functions --- src/lib/crypto/kimchi_bindings/js/bindings.js | 48 ------------------- .../js/bindings/pickles-test.js | 47 ++++++++++++++++++ src/lib/crypto/kimchi_bindings/js/dune | 3 +- 3 files changed, 49 insertions(+), 49 deletions(-) create mode 100644 src/lib/crypto/kimchi_bindings/js/bindings/pickles-test.js diff --git a/src/lib/crypto/kimchi_bindings/js/bindings.js b/src/lib/crypto/kimchi_bindings/js/bindings.js index 616e863b787..72a4440ea71 100644 --- a/src/lib/crypto/kimchi_bindings/js/bindings.js +++ b/src/lib/crypto/kimchi_bindings/js/bindings.js @@ -784,14 +784,6 @@ var caml_pasta_fq_plonk_proof_deep_copy = function (proof) { ); }; -// Provides: caml_pasta_fp_plonk_proof_example_with_lookup -function caml_pasta_fp_plonk_proof_example_with_lookup() { - // This is only used in the pickles unit tests - throw new Error( - 'Unimplemented caml_pasta_fp_plonk_proof_example_with_lookup' - ); -} - // Provides: prover_to_json // Requires: plonk_wasm var prover_to_json = plonk_wasm.prover_to_json; @@ -802,43 +794,3 @@ function integers_uint64_of_uint32(i) { // Same as integers_uint64_of_int return new UInt64(caml_int64_of_int32(i)); } - -///////////////////////////////////////////////////////////////////////////// -// The *_example_* functions below are only used in the pickles unit tests // -///////////////////////////////////////////////////////////////////////////// - -// Provides: caml_pasta_fp_plonk_proof_example_with_ffadd -function caml_pasta_fp_plonk_proof_example_with_ffadd() { - throw new Error('Unimplemented caml_pasta_fp_plonk_proof_example_with_ffadd'); -} - -// Provides: caml_pasta_fp_plonk_proof_example_with_foreign_field_mul -function caml_pasta_fp_plonk_proof_example_with_foreign_field_mul() { - throw new Error( - 'Unimplemented caml_pasta_fp_plonk_proof_example_with_foreign_field_mul' - ); -} - -// Provides: caml_pasta_fp_plonk_proof_example_with_range_check -function caml_pasta_fp_plonk_proof_example_with_range_check() { - throw new Error( - 'Unimplemented caml_pasta_fp_plonk_proof_example_with_range_check' - ); -} - -// Provides: caml_pasta_fp_plonk_proof_example_with_range_check0 -function caml_pasta_fp_plonk_proof_example_with_range_check0() { - throw new Error( - 'Unimplemented caml_pasta_fp_plonk_proof_example_with_range_check0' - ); -} - -// Provides: caml_pasta_fp_plonk_proof_example_with_rot -function caml_pasta_fp_plonk_proof_example_with_rot() { - throw new Error('Unimplemented caml_pasta_fp_plonk_proof_example_with_rot'); -} - -// Provides: caml_pasta_fp_plonk_proof_example_with_xor -function caml_pasta_fp_plonk_proof_example_with_xor() { - throw new Error('Unimplemented caml_pasta_fp_plonk_proof_example_with_xor'); -} diff --git a/src/lib/crypto/kimchi_bindings/js/bindings/pickles-test.js b/src/lib/crypto/kimchi_bindings/js/bindings/pickles-test.js new file mode 100644 index 00000000000..0a68e6f9f46 --- /dev/null +++ b/src/lib/crypto/kimchi_bindings/js/bindings/pickles-test.js @@ -0,0 +1,47 @@ +///////////////////////////////////////////////////////////////////////////// +// The *_example_* functions below are only used in the pickles unit tests // +///////////////////////////////////////////////////////////////////////////// + +// Provides: caml_pasta_fp_plonk_proof_example_with_ffadd +function caml_pasta_fp_plonk_proof_example_with_ffadd() { + throw new Error('Unimplemented caml_pasta_fp_plonk_proof_example_with_ffadd'); +} + +// Provides: caml_pasta_fp_plonk_proof_example_with_foreign_field_mul +function caml_pasta_fp_plonk_proof_example_with_foreign_field_mul() { + throw new Error( + 'Unimplemented caml_pasta_fp_plonk_proof_example_with_foreign_field_mul' + ); +} + +// Provides: caml_pasta_fp_plonk_proof_example_with_range_check +function caml_pasta_fp_plonk_proof_example_with_range_check() { + throw new Error( + 'Unimplemented caml_pasta_fp_plonk_proof_example_with_range_check' + ); +} + +// Provides: caml_pasta_fp_plonk_proof_example_with_range_check0 +function caml_pasta_fp_plonk_proof_example_with_range_check0() { + throw new Error( + 'Unimplemented caml_pasta_fp_plonk_proof_example_with_range_check0' + ); +} + +// Provides: caml_pasta_fp_plonk_proof_example_with_rot +function caml_pasta_fp_plonk_proof_example_with_rot() { + throw new Error('Unimplemented caml_pasta_fp_plonk_proof_example_with_rot'); +} + +// Provides: caml_pasta_fp_plonk_proof_example_with_xor +function caml_pasta_fp_plonk_proof_example_with_xor() { + throw new Error('Unimplemented caml_pasta_fp_plonk_proof_example_with_xor'); +} + +// Provides: caml_pasta_fp_plonk_proof_example_with_lookup +function caml_pasta_fp_plonk_proof_example_with_lookup() { + // This is only used in the pickles unit tests + throw new Error( + 'Unimplemented caml_pasta_fp_plonk_proof_example_with_lookup' + ); +} diff --git a/src/lib/crypto/kimchi_bindings/js/dune b/src/lib/crypto/kimchi_bindings/js/dune index 20e5c8aa7a9..ca3d4a8bf78 100644 --- a/src/lib/crypto/kimchi_bindings/js/dune +++ b/src/lib/crypto/kimchi_bindings/js/dune @@ -9,7 +9,8 @@ bindings/curve.js bindings/vector.js bindings/gate-vector.js - bindings/oracles.js)) + bindings/oracles.js + bindings/pickles-test.js)) (instrumentation (backend bisect_ppx)) (preprocess From 3acd295c8340c1d2dc5e88d3d9fb1c29c1f11577 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 28 Oct 2024 13:00:38 +0100 Subject: [PATCH 40/47] move proof --- src/lib/crypto/kimchi_bindings/js/bindings.js | 143 ------------------ .../kimchi_bindings/js/bindings/proof.js | 142 +++++++++++++++++ src/lib/crypto/kimchi_bindings/js/dune | 3 +- 3 files changed, 144 insertions(+), 144 deletions(-) create mode 100644 src/lib/crypto/kimchi_bindings/js/bindings/proof.js diff --git a/src/lib/crypto/kimchi_bindings/js/bindings.js b/src/lib/crypto/kimchi_bindings/js/bindings.js index 72a4440ea71..6a81a12db0b 100644 --- a/src/lib/crypto/kimchi_bindings/js/bindings.js +++ b/src/lib/crypto/kimchi_bindings/js/bindings.js @@ -645,149 +645,6 @@ var caml_pasta_fq_plonk_verifier_index_deep_copy = function (x) { ) ); }; - -// proof - -// Provides: caml_pasta_fp_plonk_proof_create -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fp_plonk_proof_create = function ( - index, - witness_cols, - caml_runtime_tables, - prev_challenges, - prev_sgs -) { - var w = new plonk_wasm.WasmVecVecFp(witness_cols.length - 1); - for (var i = 1; i < witness_cols.length; i++) { - w.push(tsRustConversion.fp.vectorToRust(witness_cols[i])); - } - witness_cols = w; - prev_challenges = tsRustConversion.fp.vectorToRust(prev_challenges); - var wasm_runtime_tables = - tsRustConversion.fp.runtimeTablesToRust(caml_runtime_tables); - prev_sgs = tsRustConversion.fp.pointsToRust(prev_sgs); - var proof = plonk_wasm.caml_pasta_fp_plonk_proof_create( - index, - witness_cols, - wasm_runtime_tables, - prev_challenges, - prev_sgs - ); - return tsRustConversion.fp.proofFromRust(proof); -}; - -// Provides: caml_pasta_fp_plonk_proof_verify -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fp_plonk_proof_verify = function (index, proof) { - index = tsRustConversion.fp.verifierIndexToRust(index); - proof = tsRustConversion.fp.proofToRust(proof); - return plonk_wasm.caml_pasta_fp_plonk_proof_verify(index, proof); -}; - -// Provides: caml_pasta_fp_plonk_proof_batch_verify -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fp_plonk_proof_batch_verify = function (indexes, proofs) { - indexes = tsRustConversion.mapMlArrayToRustVector( - indexes, - tsRustConversion.fp.verifierIndexToRust - ); - proofs = tsRustConversion.mapMlArrayToRustVector( - proofs, - tsRustConversion.fp.proofToRust - ); - return plonk_wasm.caml_pasta_fp_plonk_proof_batch_verify(indexes, proofs); -}; - -// Provides: caml_pasta_fp_plonk_proof_dummy -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fp_plonk_proof_dummy = function () { - return tsRustConversion.fp.proofFromRust( - plonk_wasm.caml_pasta_fp_plonk_proof_dummy() - ); -}; - -// Provides: caml_pasta_fp_plonk_proof_deep_copy -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fp_plonk_proof_deep_copy = function (proof) { - return tsRustConversion.fp.proofFromRust( - plonk_wasm.caml_pasta_fp_plonk_proof_deep_copy( - tsRustConversion.fp.proofToRust(proof) - ) - ); -}; - -// Provides: caml_pasta_fq_plonk_proof_create -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fq_plonk_proof_create = function ( - index, - witness_cols, - caml_runtime_tables, - prev_challenges, - prev_sgs -) { - var w = new plonk_wasm.WasmVecVecFq(witness_cols.length - 1); - for (var i = 1; i < witness_cols.length; i++) { - w.push(tsRustConversion.fq.vectorToRust(witness_cols[i])); - } - witness_cols = w; - prev_challenges = tsRustConversion.fq.vectorToRust(prev_challenges); - var wasm_runtime_tables = - tsRustConversion.fq.runtimeTablesToRust(caml_runtime_tables); - prev_sgs = tsRustConversion.fq.pointsToRust(prev_sgs); - var proof = plonk_wasm.caml_pasta_fq_plonk_proof_create( - index, - witness_cols, - wasm_runtime_tables, - prev_challenges, - prev_sgs - ); - return tsRustConversion.fq.proofFromRust(proof); -}; - -// Provides: caml_pasta_fq_plonk_proof_verify -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fq_plonk_proof_verify = function (index, proof) { - index = tsRustConversion.fq.verifierIndexToRust(index); - proof = tsRustConversion.fq.proofToRust(proof); - return plonk_wasm.caml_pasta_fq_plonk_proof_verify(index, proof); -}; - -// Provides: caml_pasta_fq_plonk_proof_batch_verify -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fq_plonk_proof_batch_verify = function (indexes, proofs) { - indexes = tsRustConversion.mapMlArrayToRustVector( - indexes, - tsRustConversion.fq.verifierIndexToRust - ); - proofs = tsRustConversion.mapMlArrayToRustVector( - proofs, - tsRustConversion.fq.proofToRust - ); - return plonk_wasm.caml_pasta_fq_plonk_proof_batch_verify(indexes, proofs); -}; - -// Provides: caml_pasta_fq_plonk_proof_dummy -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fq_plonk_proof_dummy = function () { - return tsRustConversion.fq.proofFromRust( - plonk_wasm.caml_pasta_fq_plonk_proof_dummy() - ); -}; - -// Provides: caml_pasta_fq_plonk_proof_deep_copy -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fq_plonk_proof_deep_copy = function (proof) { - return tsRustConversion.fq.proofFromRust( - plonk_wasm.caml_pasta_fq_plonk_proof_deep_copy( - tsRustConversion.fq.proofToRust(proof) - ) - ); -}; - -// Provides: prover_to_json -// Requires: plonk_wasm -var prover_to_json = plonk_wasm.prover_to_json; - // Provides: integers_uint64_of_uint32 // Requires: UInt64, caml_int64_of_int32 function integers_uint64_of_uint32(i) { diff --git a/src/lib/crypto/kimchi_bindings/js/bindings/proof.js b/src/lib/crypto/kimchi_bindings/js/bindings/proof.js new file mode 100644 index 00000000000..b0de9eddc60 --- /dev/null +++ b/src/lib/crypto/kimchi_bindings/js/bindings/proof.js @@ -0,0 +1,142 @@ +/* global plonk_wasm, tsRustConversion + */ + +// Provides: caml_pasta_fp_plonk_proof_create +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fp_plonk_proof_create = function ( + index, + witness_cols, + caml_runtime_tables, + prev_challenges, + prev_sgs +) { + var w = new plonk_wasm.WasmVecVecFp(witness_cols.length - 1); + for (var i = 1; i < witness_cols.length; i++) { + w.push(tsRustConversion.fp.vectorToRust(witness_cols[i])); + } + witness_cols = w; + prev_challenges = tsRustConversion.fp.vectorToRust(prev_challenges); + var wasm_runtime_tables = + tsRustConversion.fp.runtimeTablesToRust(caml_runtime_tables); + prev_sgs = tsRustConversion.fp.pointsToRust(prev_sgs); + var proof = plonk_wasm.caml_pasta_fp_plonk_proof_create( + index, + witness_cols, + wasm_runtime_tables, + prev_challenges, + prev_sgs + ); + return tsRustConversion.fp.proofFromRust(proof); +}; + +// Provides: caml_pasta_fp_plonk_proof_verify +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fp_plonk_proof_verify = function (index, proof) { + index = tsRustConversion.fp.verifierIndexToRust(index); + proof = tsRustConversion.fp.proofToRust(proof); + return plonk_wasm.caml_pasta_fp_plonk_proof_verify(index, proof); +}; + +// Provides: caml_pasta_fp_plonk_proof_batch_verify +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fp_plonk_proof_batch_verify = function (indexes, proofs) { + indexes = tsRustConversion.mapMlArrayToRustVector( + indexes, + tsRustConversion.fp.verifierIndexToRust + ); + proofs = tsRustConversion.mapMlArrayToRustVector( + proofs, + tsRustConversion.fp.proofToRust + ); + return plonk_wasm.caml_pasta_fp_plonk_proof_batch_verify(indexes, proofs); +}; + +// Provides: caml_pasta_fp_plonk_proof_dummy +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fp_plonk_proof_dummy = function () { + return tsRustConversion.fp.proofFromRust( + plonk_wasm.caml_pasta_fp_plonk_proof_dummy() + ); +}; + +// Provides: caml_pasta_fp_plonk_proof_deep_copy +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fp_plonk_proof_deep_copy = function (proof) { + return tsRustConversion.fp.proofFromRust( + plonk_wasm.caml_pasta_fp_plonk_proof_deep_copy( + tsRustConversion.fp.proofToRust(proof) + ) + ); +}; + +// Provides: caml_pasta_fq_plonk_proof_create +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fq_plonk_proof_create = function ( + index, + witness_cols, + caml_runtime_tables, + prev_challenges, + prev_sgs +) { + var w = new plonk_wasm.WasmVecVecFq(witness_cols.length - 1); + for (var i = 1; i < witness_cols.length; i++) { + w.push(tsRustConversion.fq.vectorToRust(witness_cols[i])); + } + witness_cols = w; + prev_challenges = tsRustConversion.fq.vectorToRust(prev_challenges); + var wasm_runtime_tables = + tsRustConversion.fq.runtimeTablesToRust(caml_runtime_tables); + prev_sgs = tsRustConversion.fq.pointsToRust(prev_sgs); + var proof = plonk_wasm.caml_pasta_fq_plonk_proof_create( + index, + witness_cols, + wasm_runtime_tables, + prev_challenges, + prev_sgs + ); + return tsRustConversion.fq.proofFromRust(proof); +}; + +// Provides: caml_pasta_fq_plonk_proof_verify +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fq_plonk_proof_verify = function (index, proof) { + index = tsRustConversion.fq.verifierIndexToRust(index); + proof = tsRustConversion.fq.proofToRust(proof); + return plonk_wasm.caml_pasta_fq_plonk_proof_verify(index, proof); +}; + +// Provides: caml_pasta_fq_plonk_proof_batch_verify +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fq_plonk_proof_batch_verify = function (indexes, proofs) { + indexes = tsRustConversion.mapMlArrayToRustVector( + indexes, + tsRustConversion.fq.verifierIndexToRust + ); + proofs = tsRustConversion.mapMlArrayToRustVector( + proofs, + tsRustConversion.fq.proofToRust + ); + return plonk_wasm.caml_pasta_fq_plonk_proof_batch_verify(indexes, proofs); +}; + +// Provides: caml_pasta_fq_plonk_proof_dummy +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fq_plonk_proof_dummy = function () { + return tsRustConversion.fq.proofFromRust( + plonk_wasm.caml_pasta_fq_plonk_proof_dummy() + ); +}; + +// Provides: caml_pasta_fq_plonk_proof_deep_copy +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fq_plonk_proof_deep_copy = function (proof) { + return tsRustConversion.fq.proofFromRust( + plonk_wasm.caml_pasta_fq_plonk_proof_deep_copy( + tsRustConversion.fq.proofToRust(proof) + ) + ); +}; + +// Provides: prover_to_json +// Requires: plonk_wasm +var prover_to_json = plonk_wasm.prover_to_json; diff --git a/src/lib/crypto/kimchi_bindings/js/dune b/src/lib/crypto/kimchi_bindings/js/dune index ca3d4a8bf78..a79d1f139fb 100644 --- a/src/lib/crypto/kimchi_bindings/js/dune +++ b/src/lib/crypto/kimchi_bindings/js/dune @@ -10,7 +10,8 @@ bindings/vector.js bindings/gate-vector.js bindings/oracles.js - bindings/pickles-test.js)) + bindings/pickles-test.js + bindings/proof.js)) (instrumentation (backend bisect_ppx)) (preprocess From 5c25270a2db52edca6987f1ab0ad67264532e5d2 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 28 Oct 2024 13:04:01 +0100 Subject: [PATCH 41/47] move prover index --- src/lib/crypto/kimchi_bindings/js/bindings.js | 221 ----------------- .../js/bindings/prover-index.js | 222 ++++++++++++++++++ src/lib/crypto/kimchi_bindings/js/dune | 3 +- 3 files changed, 224 insertions(+), 222 deletions(-) create mode 100644 src/lib/crypto/kimchi_bindings/js/bindings/prover-index.js diff --git a/src/lib/crypto/kimchi_bindings/js/bindings.js b/src/lib/crypto/kimchi_bindings/js/bindings.js index 6a81a12db0b..0f6a5144962 100644 --- a/src/lib/crypto/kimchi_bindings/js/bindings.js +++ b/src/lib/crypto/kimchi_bindings/js/bindings.js @@ -273,227 +273,6 @@ var caml_fq_srs_h = function (t) { // Requires: tsSrs var caml_fq_srs_add_lagrange_basis = tsSrs.fq.addLagrangeBasis; -// prover index - -// Provides: caml_pasta_fq_plonk_circuit_serialize -// Requires: plonk_wasm, caml_string_of_jsstring -var caml_pasta_fq_plonk_circuit_serialize = function ( - public_input_size, - gate_vector -) { - return caml_string_of_jsstring( - plonk_wasm.caml_pasta_fq_plonk_circuit_serialize( - public_input_size, - gate_vector - ) - ); -}; - -// Provides: caml_pasta_fp_plonk_index_create -// Requires: plonk_wasm, free_on_finalize, tsRustConversion -var caml_pasta_fp_plonk_index_create = function ( - gates, - public_inputs, - caml_lookup_tables, - caml_runtime_table_cfgs, - prev_challenges, - urs -) { - var wasm_lookup_tables = - tsRustConversion.fp.lookupTablesToRust(caml_lookup_tables); - var wasm_runtime_table_cfgs = tsRustConversion.fp.runtimeTableCfgsToRust( - caml_runtime_table_cfgs - ); - - var t = plonk_wasm.caml_pasta_fp_plonk_index_create( - gates, - public_inputs, - wasm_lookup_tables, - wasm_runtime_table_cfgs, - prev_challenges, - urs - ); - return free_on_finalize(t); -}; - -// Provides: caml_pasta_fp_plonk_index_create_bytecode -// Requires: caml_pasta_fp_plonk_index_create -var caml_pasta_fp_plonk_index_create_bytecode = function ( - gates, - public_inputs, - caml_lookup_tables, - caml_runtime_table_cfgs, - prev_challenges, - urs -) { - return caml_pasta_fp_plonk_index_create( - gates, - public_inputs, - caml_lookup_tables, - caml_runtime_table_cfgs, - prev_challenges, - urs - ); -}; - -// Provides: caml_pasta_fp_plonk_index_max_degree -// Requires: plonk_wasm -var caml_pasta_fp_plonk_index_max_degree = - plonk_wasm.caml_pasta_fp_plonk_index_max_degree; - -// Provides: caml_pasta_fp_plonk_index_public_inputs -// Requires: plonk_wasm -var caml_pasta_fp_plonk_index_public_inputs = - plonk_wasm.caml_pasta_fp_plonk_index_public_inputs; - -// Provides: caml_pasta_fp_plonk_index_domain_d1_size -// Requires: plonk_wasm -var caml_pasta_fp_plonk_index_domain_d1_size = - plonk_wasm.caml_pasta_fp_plonk_index_domain_d1_size; - -// Provides: caml_pasta_fp_plonk_index_domain_d4_size -// Requires: plonk_wasm -var caml_pasta_fp_plonk_index_domain_d4_size = - plonk_wasm.caml_pasta_fp_plonk_index_domain_d4_size; - -// Provides: caml_pasta_fp_plonk_index_domain_d8_size -// Requires: plonk_wasm -var caml_pasta_fp_plonk_index_domain_d8_size = - plonk_wasm.caml_pasta_fp_plonk_index_domain_d8_size; - -// Provides: caml_pasta_fp_plonk_index_read -// Requires: plonk_wasm, caml_jsstring_of_string -var caml_pasta_fp_plonk_index_read = function (offset, urs, path) { - if (offset === 0) { - offset = undefined; - } else { - offset = offset[1]; - } - return plonk_wasm.caml_pasta_fp_plonk_index_read( - offset, - urs, - caml_jsstring_of_string(path) - ); -}; - -// Provides: caml_pasta_fp_plonk_index_write -// Requires: plonk_wasm, caml_jsstring_of_string -var caml_pasta_fp_plonk_index_write = function (append, t, path) { - if (append === 0) { - append = undefined; - } else { - append = append[1]; - } - return plonk_wasm.caml_pasta_fp_plonk_index_write( - append, - t, - caml_jsstring_of_string(path) - ); -}; - -// Provides: caml_pasta_fq_plonk_index_create -// Requires: plonk_wasm, free_on_finalize, tsRustConversion -var caml_pasta_fq_plonk_index_create = function ( - gates, - public_inputs, - caml_lookup_tables, - caml_runtime_table_cfgs, - prev_challenges, - urs -) { - var wasm_lookup_tables = - tsRustConversion.fq.lookupTablesToRust(caml_lookup_tables); - var wasm_runtime_table_cfgs = tsRustConversion.fq.runtimeTableCfgsToRust( - caml_runtime_table_cfgs - ); - - return free_on_finalize( - plonk_wasm.caml_pasta_fq_plonk_index_create( - gates, - public_inputs, - wasm_lookup_tables, - wasm_runtime_table_cfgs, - prev_challenges, - urs - ) - ); -}; - -// Provides: caml_pasta_fq_plonk_index_create_bytecode -// Requires: caml_pasta_fq_plonk_index_create -var caml_pasta_fq_plonk_index_create_bytecode = function ( - gates, - public_inputs, - caml_lookup_tables, - caml_runtime_table_cfgs, - prev_challenges, - urs -) { - return caml_pasta_fq_plonk_index_create( - gates, - public_inputs, - caml_lookup_tables, - caml_runtime_table_cfgs, - prev_challenges, - urs - ); -}; - -// Provides: caml_pasta_fq_plonk_index_max_degree -// Requires: plonk_wasm -var caml_pasta_fq_plonk_index_max_degree = - plonk_wasm.caml_pasta_fq_plonk_index_max_degree; - -// Provides: caml_pasta_fq_plonk_index_public_inputs -// Requires: plonk_wasm -var caml_pasta_fq_plonk_index_public_inputs = - plonk_wasm.caml_pasta_fq_plonk_index_public_inputs; - -// Provides: caml_pasta_fq_plonk_index_domain_d1_size -// Requires: plonk_wasm -var caml_pasta_fq_plonk_index_domain_d1_size = - plonk_wasm.caml_pasta_fq_plonk_index_domain_d1_size; - -// Provides: caml_pasta_fq_plonk_index_domain_d4_size -// Requires: plonk_wasm -var caml_pasta_fq_plonk_index_domain_d4_size = - plonk_wasm.caml_pasta_fq_plonk_index_domain_d4_size; - -// Provides: caml_pasta_fq_plonk_index_domain_d8_size -// Requires: plonk_wasm -var caml_pasta_fq_plonk_index_domain_d8_size = - plonk_wasm.caml_pasta_fq_plonk_index_domain_d8_size; - -// Provides: caml_pasta_fq_plonk_index_read -// Requires: plonk_wasm, caml_jsstring_of_string -var caml_pasta_fq_plonk_index_read = function (offset, urs, path) { - if (offset === 0) { - offset = undefined; - } else { - offset = offset[1]; - } - return plonk_wasm.caml_pasta_fq_plonk_index_read( - offset, - urs, - caml_jsstring_of_string(path) - ); -}; - -// Provides: caml_pasta_fq_plonk_index_write -// Requires: plonk_wasm, caml_jsstring_of_string -var caml_pasta_fq_plonk_index_write = function (append, t, path) { - if (append === 0) { - append = undefined; - } else { - append = append[1]; - } - return plonk_wasm.caml_pasta_fq_plonk_index_write( - append, - t, - caml_jsstring_of_string(path) - ); -}; - // verifier index // Provides: caml_opt_of_rust diff --git a/src/lib/crypto/kimchi_bindings/js/bindings/prover-index.js b/src/lib/crypto/kimchi_bindings/js/bindings/prover-index.js new file mode 100644 index 00000000000..163a38dce22 --- /dev/null +++ b/src/lib/crypto/kimchi_bindings/js/bindings/prover-index.js @@ -0,0 +1,222 @@ +/* global plonk_wasm, tsRustConversion, caml_string_of_jsstring, + free_on_finalize, caml_jsstring_of_string + */ + +// Provides: caml_pasta_fq_plonk_circuit_serialize +// Requires: plonk_wasm, caml_string_of_jsstring +var caml_pasta_fq_plonk_circuit_serialize = function ( + public_input_size, + gate_vector +) { + return caml_string_of_jsstring( + plonk_wasm.caml_pasta_fq_plonk_circuit_serialize( + public_input_size, + gate_vector + ) + ); +}; + +// Provides: caml_pasta_fp_plonk_index_create +// Requires: plonk_wasm, free_on_finalize, tsRustConversion +var caml_pasta_fp_plonk_index_create = function ( + gates, + public_inputs, + caml_lookup_tables, + caml_runtime_table_cfgs, + prev_challenges, + urs +) { + var wasm_lookup_tables = + tsRustConversion.fp.lookupTablesToRust(caml_lookup_tables); + var wasm_runtime_table_cfgs = tsRustConversion.fp.runtimeTableCfgsToRust( + caml_runtime_table_cfgs + ); + + var t = plonk_wasm.caml_pasta_fp_plonk_index_create( + gates, + public_inputs, + wasm_lookup_tables, + wasm_runtime_table_cfgs, + prev_challenges, + urs + ); + return free_on_finalize(t); +}; + +// Provides: caml_pasta_fp_plonk_index_create_bytecode +// Requires: caml_pasta_fp_plonk_index_create +var caml_pasta_fp_plonk_index_create_bytecode = function ( + gates, + public_inputs, + caml_lookup_tables, + caml_runtime_table_cfgs, + prev_challenges, + urs +) { + return caml_pasta_fp_plonk_index_create( + gates, + public_inputs, + caml_lookup_tables, + caml_runtime_table_cfgs, + prev_challenges, + urs + ); +}; + +// Provides: caml_pasta_fp_plonk_index_max_degree +// Requires: plonk_wasm +var caml_pasta_fp_plonk_index_max_degree = + plonk_wasm.caml_pasta_fp_plonk_index_max_degree; + +// Provides: caml_pasta_fp_plonk_index_public_inputs +// Requires: plonk_wasm +var caml_pasta_fp_plonk_index_public_inputs = + plonk_wasm.caml_pasta_fp_plonk_index_public_inputs; + +// Provides: caml_pasta_fp_plonk_index_domain_d1_size +// Requires: plonk_wasm +var caml_pasta_fp_plonk_index_domain_d1_size = + plonk_wasm.caml_pasta_fp_plonk_index_domain_d1_size; + +// Provides: caml_pasta_fp_plonk_index_domain_d4_size +// Requires: plonk_wasm +var caml_pasta_fp_plonk_index_domain_d4_size = + plonk_wasm.caml_pasta_fp_plonk_index_domain_d4_size; + +// Provides: caml_pasta_fp_plonk_index_domain_d8_size +// Requires: plonk_wasm +var caml_pasta_fp_plonk_index_domain_d8_size = + plonk_wasm.caml_pasta_fp_plonk_index_domain_d8_size; + +// Provides: caml_pasta_fp_plonk_index_read +// Requires: plonk_wasm, caml_jsstring_of_string +var caml_pasta_fp_plonk_index_read = function (offset, urs, path) { + if (offset === 0) { + offset = undefined; + } else { + offset = offset[1]; + } + return plonk_wasm.caml_pasta_fp_plonk_index_read( + offset, + urs, + caml_jsstring_of_string(path) + ); +}; + +// Provides: caml_pasta_fp_plonk_index_write +// Requires: plonk_wasm, caml_jsstring_of_string +var caml_pasta_fp_plonk_index_write = function (append, t, path) { + if (append === 0) { + append = undefined; + } else { + append = append[1]; + } + return plonk_wasm.caml_pasta_fp_plonk_index_write( + append, + t, + caml_jsstring_of_string(path) + ); +}; + +// Provides: caml_pasta_fq_plonk_index_create +// Requires: plonk_wasm, free_on_finalize, tsRustConversion +var caml_pasta_fq_plonk_index_create = function ( + gates, + public_inputs, + caml_lookup_tables, + caml_runtime_table_cfgs, + prev_challenges, + urs +) { + var wasm_lookup_tables = + tsRustConversion.fq.lookupTablesToRust(caml_lookup_tables); + var wasm_runtime_table_cfgs = tsRustConversion.fq.runtimeTableCfgsToRust( + caml_runtime_table_cfgs + ); + + return free_on_finalize( + plonk_wasm.caml_pasta_fq_plonk_index_create( + gates, + public_inputs, + wasm_lookup_tables, + wasm_runtime_table_cfgs, + prev_challenges, + urs + ) + ); +}; + +// Provides: caml_pasta_fq_plonk_index_create_bytecode +// Requires: caml_pasta_fq_plonk_index_create +var caml_pasta_fq_plonk_index_create_bytecode = function ( + gates, + public_inputs, + caml_lookup_tables, + caml_runtime_table_cfgs, + prev_challenges, + urs +) { + return caml_pasta_fq_plonk_index_create( + gates, + public_inputs, + caml_lookup_tables, + caml_runtime_table_cfgs, + prev_challenges, + urs + ); +}; + +// Provides: caml_pasta_fq_plonk_index_max_degree +// Requires: plonk_wasm +var caml_pasta_fq_plonk_index_max_degree = + plonk_wasm.caml_pasta_fq_plonk_index_max_degree; + +// Provides: caml_pasta_fq_plonk_index_public_inputs +// Requires: plonk_wasm +var caml_pasta_fq_plonk_index_public_inputs = + plonk_wasm.caml_pasta_fq_plonk_index_public_inputs; + +// Provides: caml_pasta_fq_plonk_index_domain_d1_size +// Requires: plonk_wasm +var caml_pasta_fq_plonk_index_domain_d1_size = + plonk_wasm.caml_pasta_fq_plonk_index_domain_d1_size; + +// Provides: caml_pasta_fq_plonk_index_domain_d4_size +// Requires: plonk_wasm +var caml_pasta_fq_plonk_index_domain_d4_size = + plonk_wasm.caml_pasta_fq_plonk_index_domain_d4_size; + +// Provides: caml_pasta_fq_plonk_index_domain_d8_size +// Requires: plonk_wasm +var caml_pasta_fq_plonk_index_domain_d8_size = + plonk_wasm.caml_pasta_fq_plonk_index_domain_d8_size; + +// Provides: caml_pasta_fq_plonk_index_read +// Requires: plonk_wasm, caml_jsstring_of_string +var caml_pasta_fq_plonk_index_read = function (offset, urs, path) { + if (offset === 0) { + offset = undefined; + } else { + offset = offset[1]; + } + return plonk_wasm.caml_pasta_fq_plonk_index_read( + offset, + urs, + caml_jsstring_of_string(path) + ); +}; + +// Provides: caml_pasta_fq_plonk_index_write +// Requires: plonk_wasm, caml_jsstring_of_string +var caml_pasta_fq_plonk_index_write = function (append, t, path) { + if (append === 0) { + append = undefined; + } else { + append = append[1]; + } + return plonk_wasm.caml_pasta_fq_plonk_index_write( + append, + t, + caml_jsstring_of_string(path) + ); +}; diff --git a/src/lib/crypto/kimchi_bindings/js/dune b/src/lib/crypto/kimchi_bindings/js/dune index a79d1f139fb..99ae2a0ee0a 100644 --- a/src/lib/crypto/kimchi_bindings/js/dune +++ b/src/lib/crypto/kimchi_bindings/js/dune @@ -11,7 +11,8 @@ bindings/gate-vector.js bindings/oracles.js bindings/pickles-test.js - bindings/proof.js)) + bindings/proof.js + bindings/prover-index.js)) (instrumentation (backend bisect_ppx)) (preprocess From 284f57ba8f2281d8a4073a43a655dd269bde6622 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 28 Oct 2024 13:08:46 +0100 Subject: [PATCH 42/47] move util functions --- src/lib/crypto/kimchi_bindings/js/bindings.js | 93 +------------------ .../kimchi_bindings/js/bindings/util.js | 93 +++++++++++++++++++ src/lib/crypto/kimchi_bindings/js/dune | 3 +- 3 files changed, 97 insertions(+), 92 deletions(-) create mode 100644 src/lib/crypto/kimchi_bindings/js/bindings/util.js diff --git a/src/lib/crypto/kimchi_bindings/js/bindings.js b/src/lib/crypto/kimchi_bindings/js/bindings.js index 0f6a5144962..837633b4e11 100644 --- a/src/lib/crypto/kimchi_bindings/js/bindings.js +++ b/src/lib/crypto/kimchi_bindings/js/bindings.js @@ -1,94 +1,11 @@ -/* global plonk_wasm, caml_jsstring_of_string, caml_string_of_jsstring, - caml_create_bytes, caml_bytes_unsafe_set, caml_bytes_unsafe_get, caml_ml_bytes_length, - UInt64, caml_int64_of_int32 +/* global plonk_wasm, caml_jsstring_of_string, + tsBindings, tsRustConversion */ -// Provides: tsBindings -var tsBindings = globalThis.__snarkyTsBindings; - -// Provides: tsRustConversion -// Requires: tsBindings, plonk_wasm -var tsRustConversion = tsBindings.rustConversion(plonk_wasm); - // Provides: tsSrs // Requires: tsBindings, plonk_wasm var tsSrs = tsBindings.srs(plonk_wasm); -// Provides: getTsBindings -// Requires: tsBindings -function getTsBindings() { - return tsBindings; -} - -// Provides: caml_bytes_of_uint8array -// Requires: caml_create_bytes, caml_bytes_unsafe_set -var caml_bytes_of_uint8array = function (uint8array) { - var length = uint8array.length; - var ocaml_bytes = caml_create_bytes(length); - for (var i = 0; i < length; i++) { - // No need to convert here: OCaml Char.t is just an int under the hood. - caml_bytes_unsafe_set(ocaml_bytes, i, uint8array[i]); - } - return ocaml_bytes; -}; - -// Provides: caml_bytes_to_uint8array -// Requires: caml_ml_bytes_length, caml_bytes_unsafe_get -var caml_bytes_to_uint8array = function (ocaml_bytes) { - var length = caml_ml_bytes_length(ocaml_bytes); - var bytes = new globalThis.Uint8Array(length); - for (var i = 0; i < length; i++) { - // No need to convert here: OCaml Char.t is just an int under the hood. - bytes[i] = caml_bytes_unsafe_get(ocaml_bytes, i); - } - return bytes; -}; - -// Provides: caml_option_of_maybe_undefined -var caml_option_of_maybe_undefined = function (x) { - if (x === undefined) { - return 0; // None - } else { - return [0, x]; // Some(x) - } -}; - -// Provides: caml_option_to_maybe_undefined -var caml_option_to_maybe_undefined = function (x) { - if (x === 0) { - // None - return undefined; - } else { - return x[1]; - } -}; - -// Provides: free_finalization_registry -var free_finalization_registry = new globalThis.FinalizationRegistry(function ( - instance_representative -) { - instance_representative.free(); -}); - -// Provides: free_on_finalize -// Requires: free_finalization_registry -var free_on_finalize = function (x) { - // This is an unfortunate hack: we're creating a second instance of the - // class to be able to call free on it. We can't pass the value itself, - // since the registry holds a strong reference to the representative value. - // - // However, the class is only really a wrapper around a pointer, with a - // reference to the class' prototype as its __prototype__. - // - // It might seem cleaner to call the destructor here on the pointer - // directly, but unfortunately the destructor name is some mangled internal - // string generated by wasm_bindgen. For now, this is the best, - // least-brittle way to free once the original class instance gets collected. - var instance_representative = x.constructor.__wrap(x.__wbg_ptr); - free_finalization_registry.register(x, instance_representative, x); - return x; -}; - // srs // Provides: caml_fp_srs_create @@ -424,9 +341,3 @@ var caml_pasta_fq_plonk_verifier_index_deep_copy = function (x) { ) ); }; -// Provides: integers_uint64_of_uint32 -// Requires: UInt64, caml_int64_of_int32 -function integers_uint64_of_uint32(i) { - // Same as integers_uint64_of_int - return new UInt64(caml_int64_of_int32(i)); -} diff --git a/src/lib/crypto/kimchi_bindings/js/bindings/util.js b/src/lib/crypto/kimchi_bindings/js/bindings/util.js new file mode 100644 index 00000000000..8ada833be86 --- /dev/null +++ b/src/lib/crypto/kimchi_bindings/js/bindings/util.js @@ -0,0 +1,93 @@ +/* global UInt64, caml_int64_of_int32, caml_create_bytes, + caml_bytes_unsafe_set, caml_bytes_unsafe_get, caml_ml_bytes_length, + plonk_wasm + */ + +// Provides: tsBindings +var tsBindings = globalThis.__snarkyTsBindings; + +// Provides: tsRustConversion +// Requires: tsBindings, plonk_wasm +var tsRustConversion = tsBindings.rustConversion(plonk_wasm); + +// Provides: getTsBindings +// Requires: tsBindings +function getTsBindings() { + return tsBindings; +} + +// Provides: integers_uint64_of_uint32 +// Requires: UInt64, caml_int64_of_int32 +function integers_uint64_of_uint32(i) { + // Same as integers_uint64_of_int + return new UInt64(caml_int64_of_int32(i)); +} + +// Provides: caml_bytes_of_uint8array +// Requires: caml_create_bytes, caml_bytes_unsafe_set +var caml_bytes_of_uint8array = function (uint8array) { + var length = uint8array.length; + var ocaml_bytes = caml_create_bytes(length); + for (var i = 0; i < length; i++) { + // No need to convert here: OCaml Char.t is just an int under the hood. + caml_bytes_unsafe_set(ocaml_bytes, i, uint8array[i]); + } + return ocaml_bytes; +}; + +// Provides: caml_bytes_to_uint8array +// Requires: caml_ml_bytes_length, caml_bytes_unsafe_get +var caml_bytes_to_uint8array = function (ocaml_bytes) { + var length = caml_ml_bytes_length(ocaml_bytes); + var bytes = new globalThis.Uint8Array(length); + for (var i = 0; i < length; i++) { + // No need to convert here: OCaml Char.t is just an int under the hood. + bytes[i] = caml_bytes_unsafe_get(ocaml_bytes, i); + } + return bytes; +}; + +// Provides: caml_option_of_maybe_undefined +var caml_option_of_maybe_undefined = function (x) { + if (x === undefined) { + return 0; // None + } else { + return [0, x]; // Some(x) + } +}; + +// Provides: caml_option_to_maybe_undefined +var caml_option_to_maybe_undefined = function (x) { + if (x === 0) { + // None + return undefined; + } else { + return x[1]; + } +}; + +// Provides: free_finalization_registry +var free_finalization_registry = new globalThis.FinalizationRegistry(function ( + instance_representative +) { + instance_representative.free(); +}); + +// Provides: free_on_finalize +// Requires: free_finalization_registry +var free_on_finalize = function (x) { + // This is an unfortunate hack: we're creating a second instance of the + // class to be able to call free on it. We can't pass the value itself, + // since the registry holds a strong reference to the representative value. + // + // However, the class is only really a wrapper around a pointer, with a + // reference to the class' prototype as its __prototype__. + // + // It might seem cleaner to call the destructor here on the pointer + // directly, but unfortunately the destructor name is some mangled internal + // string generated by wasm_bindgen. For now, this is the best, + // least-brittle way to free once the original class instance gets collected. + var instance_representative = x.constructor.__wrap(x.__wbg_ptr); + free_finalization_registry.register(x, instance_representative, x); + return x; +}; diff --git a/src/lib/crypto/kimchi_bindings/js/dune b/src/lib/crypto/kimchi_bindings/js/dune index 99ae2a0ee0a..dc9aa5e4107 100644 --- a/src/lib/crypto/kimchi_bindings/js/dune +++ b/src/lib/crypto/kimchi_bindings/js/dune @@ -12,7 +12,8 @@ bindings/oracles.js bindings/pickles-test.js bindings/proof.js - bindings/prover-index.js)) + bindings/prover-index.js + bindings/util.js)) (instrumentation (backend bisect_ppx)) (preprocess From bfc685019e69319cc8b134e3c4fbdbc17028e9bc Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 28 Oct 2024 13:11:32 +0100 Subject: [PATCH 43/47] move srs --- src/lib/crypto/kimchi_bindings/js/bindings.js | 194 +----------------- .../crypto/kimchi_bindings/js/bindings/srs.js | 191 +++++++++++++++++ src/lib/crypto/kimchi_bindings/js/dune | 3 +- 3 files changed, 195 insertions(+), 193 deletions(-) create mode 100644 src/lib/crypto/kimchi_bindings/js/bindings/srs.js diff --git a/src/lib/crypto/kimchi_bindings/js/bindings.js b/src/lib/crypto/kimchi_bindings/js/bindings.js index 837633b4e11..2bd98058519 100644 --- a/src/lib/crypto/kimchi_bindings/js/bindings.js +++ b/src/lib/crypto/kimchi_bindings/js/bindings.js @@ -1,195 +1,5 @@ -/* global plonk_wasm, caml_jsstring_of_string, - tsBindings, tsRustConversion -*/ - -// Provides: tsSrs -// Requires: tsBindings, plonk_wasm -var tsSrs = tsBindings.srs(plonk_wasm); - -// srs - -// Provides: caml_fp_srs_create -// Requires: tsSrs -var caml_fp_srs_create = tsSrs.fp.create; - -// Provides: caml_fp_srs_write -// Requires: plonk_wasm, caml_jsstring_of_string -var caml_fp_srs_write = function (append, t, path) { - if (append === 0) { - append = undefined; - } else { - append = append[1]; - } - return plonk_wasm.caml_fp_srs_write(append, t, caml_jsstring_of_string(path)); -}; - -// Provides: caml_fp_srs_read -// Requires: plonk_wasm, caml_jsstring_of_string -var caml_fp_srs_read = function (offset, path) { - if (offset === 0) { - offset = undefined; - } else { - offset = offset[1]; - } - var res = plonk_wasm.caml_fp_srs_read(offset, caml_jsstring_of_string(path)); - if (res) { - return [0, res]; // Some(res) - } else { - return 0; // None - } -}; - -// Provides: caml_fp_srs_lagrange_commitment -// Requires: tsSrs -var caml_fp_srs_lagrange_commitment = tsSrs.fp.lagrangeCommitment; - -// Provides: caml_fp_srs_commit_evaluations -// Requires: plonk_wasm, tsRustConversion -var caml_fp_srs_commit_evaluations = function (t, domain_size, fps) { - var res = plonk_wasm.caml_fp_srs_commit_evaluations( - t, - domain_size, - tsRustConversion.fp.vectorToRust(fps) - ); - return tsRustConversion.fp.polyCommFromRust(res); -}; - -// Provides: caml_fp_srs_b_poly_commitment -// Requires: plonk_wasm, tsRustConversion -var caml_fp_srs_b_poly_commitment = function (srs, chals) { - var res = plonk_wasm.caml_fp_srs_b_poly_commitment( - srs, - tsRustConversion.fieldsToRustFlat(chals) - ); - return tsRustConversion.fp.polyCommFromRust(res); -}; - -// Provides: caml_fp_srs_batch_accumulator_check -// Requires: plonk_wasm, tsRustConversion -var caml_fp_srs_batch_accumulator_check = function (srs, comms, chals) { - var rust_comms = tsRustConversion.fp.pointsToRust(comms); - var rust_chals = tsRustConversion.fp.vectorToRust(chals); - var ok = plonk_wasm.caml_fp_srs_batch_accumulator_check( - srs, - rust_comms, - rust_chals - ); - return ok; -}; - -// Provides: caml_fp_srs_batch_accumulator_generate -// Requires: plonk_wasm, tsRustConversion -var caml_fp_srs_batch_accumulator_generate = function (srs, n_comms, chals) { - var rust_chals = tsRustConversion.fp.vectorToRust(chals); - var rust_comms = plonk_wasm.caml_fp_srs_batch_accumulator_generate( - srs, - n_comms, - rust_chals - ); - return tsRustConversion.fp.pointsFromRust(rust_comms); -}; - -// Provides: caml_fp_srs_h -// Requires: plonk_wasm, tsRustConversion -var caml_fp_srs_h = function (t) { - return tsRustConversion.fp.pointFromRust(plonk_wasm.caml_fp_srs_h(t)); -}; - -// Provides: caml_fp_srs_add_lagrange_basis -// Requires: tsSrs -var caml_fp_srs_add_lagrange_basis = tsSrs.fp.addLagrangeBasis; - -// Provides: caml_fq_srs_create -// Requires: tsSrs -var caml_fq_srs_create = tsSrs.fq.create; - -// Provides: caml_fq_srs_write -// Requires: plonk_wasm, caml_jsstring_of_string -var caml_fq_srs_write = function (append, t, path) { - if (append === 0) { - append = undefined; - } else { - append = append[1]; - } - return plonk_wasm.caml_fq_srs_write(append, t, caml_jsstring_of_string(path)); -}; - -// Provides: caml_fq_srs_read -// Requires: plonk_wasm, caml_jsstring_of_string -var caml_fq_srs_read = function (offset, path) { - if (offset === 0) { - offset = undefined; - } else { - offset = offset[1]; - } - var res = plonk_wasm.caml_fq_srs_read(offset, caml_jsstring_of_string(path)); - if (res) { - return [0, res]; // Some(res) - } else { - return 0; // None - } -}; - -// Provides: caml_fq_srs_lagrange_commitment -// Requires: tsSrs -var caml_fq_srs_lagrange_commitment = tsSrs.fq.lagrangeCommitment; - -// Provides: caml_fq_srs_commit_evaluations -// Requires: plonk_wasm, tsRustConversion -var caml_fq_srs_commit_evaluations = function (t, domain_size, fqs) { - var res = plonk_wasm.caml_fq_srs_commit_evaluations( - t, - domain_size, - tsRustConversion.fq.vectorToRust(fqs) - ); - return tsRustConversion.fq.polyCommFromRust(res); -}; - -// Provides: caml_fq_srs_b_poly_commitment -// Requires: plonk_wasm, tsRustConversion -var caml_fq_srs_b_poly_commitment = function (srs, chals) { - var res = plonk_wasm.caml_fq_srs_b_poly_commitment( - srs, - tsRustConversion.fieldsToRustFlat(chals) - ); - return tsRustConversion.fq.polyCommFromRust(res); -}; - -// Provides: caml_fq_srs_batch_accumulator_check -// Requires: plonk_wasm, tsRustConversion -var caml_fq_srs_batch_accumulator_check = function (srs, comms, chals) { - var rust_comms = tsRustConversion.fq.pointsToRust(comms); - var rust_chals = tsRustConversion.fq.vectorToRust(chals); - var ok = plonk_wasm.caml_fq_srs_batch_accumulator_check( - srs, - rust_comms, - rust_chals - ); - return ok; -}; - -// Provides: caml_fq_srs_batch_accumulator_generate -// Requires: plonk_wasm, tsRustConversion -var caml_fq_srs_batch_accumulator_generate = function (srs, comms, chals) { - var rust_chals = tsRustConversion.fq.vectorToRust(chals); - var rust_comms = plonk_wasm.caml_fq_srs_batch_accumulator_generate( - srs, - comms, - rust_chals - ); - return tsRustConversion.fq.pointsFromRust(rust_comms); -}; - -// Provides: caml_fq_srs_h -// Requires: plonk_wasm, tsRustConversion -var caml_fq_srs_h = function (t) { - return tsRustConversion.fq.pointFromRust(plonk_wasm.caml_fq_srs_h(t)); -}; - -// Provides: caml_fq_srs_add_lagrange_basis -// Requires: tsSrs -var caml_fq_srs_add_lagrange_basis = tsSrs.fq.addLagrangeBasis; - +/* global plonk_wasm, caml_jsstring_of_string, tsRustConversion + */ // verifier index // Provides: caml_opt_of_rust diff --git a/src/lib/crypto/kimchi_bindings/js/bindings/srs.js b/src/lib/crypto/kimchi_bindings/js/bindings/srs.js new file mode 100644 index 00000000000..d61480d422d --- /dev/null +++ b/src/lib/crypto/kimchi_bindings/js/bindings/srs.js @@ -0,0 +1,191 @@ +/* global plonk_wasm, caml_jsstring_of_string, + tsBindings, tsRustConversion +*/ + +// Provides: tsSrs +// Requires: tsBindings, plonk_wasm +var tsSrs = tsBindings.srs(plonk_wasm); + +// srs + +// Provides: caml_fp_srs_create +// Requires: tsSrs +var caml_fp_srs_create = tsSrs.fp.create; + +// Provides: caml_fp_srs_write +// Requires: plonk_wasm, caml_jsstring_of_string +var caml_fp_srs_write = function (append, t, path) { + if (append === 0) { + append = undefined; + } else { + append = append[1]; + } + return plonk_wasm.caml_fp_srs_write(append, t, caml_jsstring_of_string(path)); +}; + +// Provides: caml_fp_srs_read +// Requires: plonk_wasm, caml_jsstring_of_string +var caml_fp_srs_read = function (offset, path) { + if (offset === 0) { + offset = undefined; + } else { + offset = offset[1]; + } + var res = plonk_wasm.caml_fp_srs_read(offset, caml_jsstring_of_string(path)); + if (res) { + return [0, res]; // Some(res) + } else { + return 0; // None + } +}; + +// Provides: caml_fp_srs_lagrange_commitment +// Requires: tsSrs +var caml_fp_srs_lagrange_commitment = tsSrs.fp.lagrangeCommitment; + +// Provides: caml_fp_srs_commit_evaluations +// Requires: plonk_wasm, tsRustConversion +var caml_fp_srs_commit_evaluations = function (t, domain_size, fps) { + var res = plonk_wasm.caml_fp_srs_commit_evaluations( + t, + domain_size, + tsRustConversion.fp.vectorToRust(fps) + ); + return tsRustConversion.fp.polyCommFromRust(res); +}; + +// Provides: caml_fp_srs_b_poly_commitment +// Requires: plonk_wasm, tsRustConversion +var caml_fp_srs_b_poly_commitment = function (srs, chals) { + var res = plonk_wasm.caml_fp_srs_b_poly_commitment( + srs, + tsRustConversion.fieldsToRustFlat(chals) + ); + return tsRustConversion.fp.polyCommFromRust(res); +}; + +// Provides: caml_fp_srs_batch_accumulator_check +// Requires: plonk_wasm, tsRustConversion +var caml_fp_srs_batch_accumulator_check = function (srs, comms, chals) { + var rust_comms = tsRustConversion.fp.pointsToRust(comms); + var rust_chals = tsRustConversion.fp.vectorToRust(chals); + var ok = plonk_wasm.caml_fp_srs_batch_accumulator_check( + srs, + rust_comms, + rust_chals + ); + return ok; +}; + +// Provides: caml_fp_srs_batch_accumulator_generate +// Requires: plonk_wasm, tsRustConversion +var caml_fp_srs_batch_accumulator_generate = function (srs, n_comms, chals) { + var rust_chals = tsRustConversion.fp.vectorToRust(chals); + var rust_comms = plonk_wasm.caml_fp_srs_batch_accumulator_generate( + srs, + n_comms, + rust_chals + ); + return tsRustConversion.fp.pointsFromRust(rust_comms); +}; + +// Provides: caml_fp_srs_h +// Requires: plonk_wasm, tsRustConversion +var caml_fp_srs_h = function (t) { + return tsRustConversion.fp.pointFromRust(plonk_wasm.caml_fp_srs_h(t)); +}; + +// Provides: caml_fp_srs_add_lagrange_basis +// Requires: tsSrs +var caml_fp_srs_add_lagrange_basis = tsSrs.fp.addLagrangeBasis; + +// Provides: caml_fq_srs_create +// Requires: tsSrs +var caml_fq_srs_create = tsSrs.fq.create; + +// Provides: caml_fq_srs_write +// Requires: plonk_wasm, caml_jsstring_of_string +var caml_fq_srs_write = function (append, t, path) { + if (append === 0) { + append = undefined; + } else { + append = append[1]; + } + return plonk_wasm.caml_fq_srs_write(append, t, caml_jsstring_of_string(path)); +}; + +// Provides: caml_fq_srs_read +// Requires: plonk_wasm, caml_jsstring_of_string +var caml_fq_srs_read = function (offset, path) { + if (offset === 0) { + offset = undefined; + } else { + offset = offset[1]; + } + var res = plonk_wasm.caml_fq_srs_read(offset, caml_jsstring_of_string(path)); + if (res) { + return [0, res]; // Some(res) + } else { + return 0; // None + } +}; + +// Provides: caml_fq_srs_lagrange_commitment +// Requires: tsSrs +var caml_fq_srs_lagrange_commitment = tsSrs.fq.lagrangeCommitment; + +// Provides: caml_fq_srs_commit_evaluations +// Requires: plonk_wasm, tsRustConversion +var caml_fq_srs_commit_evaluations = function (t, domain_size, fqs) { + var res = plonk_wasm.caml_fq_srs_commit_evaluations( + t, + domain_size, + tsRustConversion.fq.vectorToRust(fqs) + ); + return tsRustConversion.fq.polyCommFromRust(res); +}; + +// Provides: caml_fq_srs_b_poly_commitment +// Requires: plonk_wasm, tsRustConversion +var caml_fq_srs_b_poly_commitment = function (srs, chals) { + var res = plonk_wasm.caml_fq_srs_b_poly_commitment( + srs, + tsRustConversion.fieldsToRustFlat(chals) + ); + return tsRustConversion.fq.polyCommFromRust(res); +}; + +// Provides: caml_fq_srs_batch_accumulator_check +// Requires: plonk_wasm, tsRustConversion +var caml_fq_srs_batch_accumulator_check = function (srs, comms, chals) { + var rust_comms = tsRustConversion.fq.pointsToRust(comms); + var rust_chals = tsRustConversion.fq.vectorToRust(chals); + var ok = plonk_wasm.caml_fq_srs_batch_accumulator_check( + srs, + rust_comms, + rust_chals + ); + return ok; +}; + +// Provides: caml_fq_srs_batch_accumulator_generate +// Requires: plonk_wasm, tsRustConversion +var caml_fq_srs_batch_accumulator_generate = function (srs, comms, chals) { + var rust_chals = tsRustConversion.fq.vectorToRust(chals); + var rust_comms = plonk_wasm.caml_fq_srs_batch_accumulator_generate( + srs, + comms, + rust_chals + ); + return tsRustConversion.fq.pointsFromRust(rust_comms); +}; + +// Provides: caml_fq_srs_h +// Requires: plonk_wasm, tsRustConversion +var caml_fq_srs_h = function (t) { + return tsRustConversion.fq.pointFromRust(plonk_wasm.caml_fq_srs_h(t)); +}; + +// Provides: caml_fq_srs_add_lagrange_basis +// Requires: tsSrs +var caml_fq_srs_add_lagrange_basis = tsSrs.fq.addLagrangeBasis; diff --git a/src/lib/crypto/kimchi_bindings/js/dune b/src/lib/crypto/kimchi_bindings/js/dune index dc9aa5e4107..f5c99df7e7b 100644 --- a/src/lib/crypto/kimchi_bindings/js/dune +++ b/src/lib/crypto/kimchi_bindings/js/dune @@ -13,7 +13,8 @@ bindings/pickles-test.js bindings/proof.js bindings/prover-index.js - bindings/util.js)) + bindings/util.js + bindings/srs.js)) (instrumentation (backend bisect_ppx)) (preprocess From 3b39824fce9b1c073ba33d9697b0fa256d979d40 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 28 Oct 2024 13:13:37 +0100 Subject: [PATCH 44/47] move verifier-index --- src/lib/crypto/kimchi_bindings/js/bindings.js | 153 ------------------ .../js/bindings/verifier-index.js | 152 +++++++++++++++++ src/lib/crypto/kimchi_bindings/js/dune | 3 +- 3 files changed, 154 insertions(+), 154 deletions(-) create mode 100644 src/lib/crypto/kimchi_bindings/js/bindings/verifier-index.js diff --git a/src/lib/crypto/kimchi_bindings/js/bindings.js b/src/lib/crypto/kimchi_bindings/js/bindings.js index 2bd98058519..e69de29bb2d 100644 --- a/src/lib/crypto/kimchi_bindings/js/bindings.js +++ b/src/lib/crypto/kimchi_bindings/js/bindings.js @@ -1,153 +0,0 @@ -/* global plonk_wasm, caml_jsstring_of_string, tsRustConversion - */ -// verifier index - -// Provides: caml_opt_of_rust -var caml_opt_of_rust = function (value, value_of_rust) { - if (value === undefined) { - return 0; - } else { - return [0, value_of_rust(value)]; - } -}; - -// Provides: caml_opt_to_rust -var caml_opt_to_rust = function (caml_optional_value, to_rust) { - // to_rust expects the parameters of the variant. A `Some vx` is represented - // as [0, vx] - if (caml_optional_value === 0) { - return undefined; - } else { - return to_rust(caml_optional_value[1]); - } -}; - -// Provides: caml_pasta_fp_plonk_verifier_index_create -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fp_plonk_verifier_index_create = function (x) { - var vk = plonk_wasm.caml_pasta_fp_plonk_verifier_index_create(x); - return tsRustConversion.fp.verifierIndexFromRust(vk); -}; - -// Provides: caml_pasta_fp_plonk_verifier_index_read -// Requires: plonk_wasm, caml_jsstring_of_string, tsRustConversion -var caml_pasta_fp_plonk_verifier_index_read = function (offset, urs, path) { - if (offset === 0) { - offset = undefined; - } else { - offset = offset[1]; - } - return tsRustConversion.fp.verifierIndexFromRust( - plonk_wasm.caml_pasta_fp_plonk_verifier_index_read( - offset, - urs, - caml_jsstring_of_string(path) - ) - ); -}; - -// Provides: caml_pasta_fp_plonk_verifier_index_write -// Requires: plonk_wasm, caml_jsstring_of_string, tsRustConversion -var caml_pasta_fp_plonk_verifier_index_write = function (append, t, path) { - if (append === 0) { - append = undefined; - } else { - append = append[1]; - } - return plonk_wasm.caml_pasta_fp_plonk_verifier_index_write( - append, - tsRustConversion.fp.verifierIndexToRust(t), - caml_jsstring_of_string(path) - ); -}; - -// Provides: caml_pasta_fp_plonk_verifier_index_shifts -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fp_plonk_verifier_index_shifts = function (log2_size) { - return tsRustConversion.fp.shiftsFromRust( - plonk_wasm.caml_pasta_fp_plonk_verifier_index_shifts(log2_size) - ); -}; - -// Provides: caml_pasta_fp_plonk_verifier_index_dummy -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fp_plonk_verifier_index_dummy = function () { - var res = plonk_wasm.caml_pasta_fp_plonk_verifier_index_dummy(); - return tsRustConversion.fp.verifierIndexFromRust(res); -}; - -// Provides: caml_pasta_fp_plonk_verifier_index_deep_copy -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fp_plonk_verifier_index_deep_copy = function (x) { - return tsRustConversion.fp.verifierIndexFromRust( - plonk_wasm.caml_pasta_fp_plonk_verifier_index_deep_copy( - tsRustConversion.fp.verifierIndexToRust(x) - ) - ); -}; - -// Provides: caml_pasta_fq_plonk_verifier_index_create -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fq_plonk_verifier_index_create = function (x) { - return tsRustConversion.fq.verifierIndexFromRust( - plonk_wasm.caml_pasta_fq_plonk_verifier_index_create(x) - ); -}; - -// Provides: caml_pasta_fq_plonk_verifier_index_read -// Requires: plonk_wasm, caml_jsstring_of_string, tsRustConversion -var caml_pasta_fq_plonk_verifier_index_read = function (offset, urs, path) { - if (offset === 0) { - offset = undefined; - } else { - offset = offset[1]; - } - return tsRustConversion.fq.verifierIndexFromRust( - plonk_wasm.caml_pasta_fq_plonk_verifier_index_read( - offset, - urs, - caml_jsstring_of_string(path) - ) - ); -}; - -// Provides: caml_pasta_fq_plonk_verifier_index_write -// Requires: plonk_wasm, caml_jsstring_of_string, tsRustConversion -var caml_pasta_fq_plonk_verifier_index_write = function (append, t, path) { - if (append === 0) { - append = undefined; - } else { - append = append[1]; - } - return plonk_wasm.caml_pasta_fq_plonk_verifier_index_write( - append, - tsRustConversion.fq.verifierIndexToRust(t), - caml_jsstring_of_string(path) - ); -}; - -// Provides: caml_pasta_fq_plonk_verifier_index_shifts -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fq_plonk_verifier_index_shifts = function (log2_size) { - return tsRustConversion.fq.shiftsFromRust( - plonk_wasm.caml_pasta_fq_plonk_verifier_index_shifts(log2_size) - ); -}; - -// Provides: caml_pasta_fq_plonk_verifier_index_dummy -// Requires: plonk_wasm, tsRustConversion -var caml_pasta_fq_plonk_verifier_index_dummy = function () { - return tsRustConversion.fq.verifierIndexFromRust( - plonk_wasm.caml_pasta_fq_plonk_verifier_index_dummy() - ); -}; - -// Provides: caml_pasta_fq_plonk_verifier_index_deep_copy -// Requires: plonk_wasm, tsRustConversion, tsRustConversion -var caml_pasta_fq_plonk_verifier_index_deep_copy = function (x) { - return tsRustConversion.fq.verifierIndexFromRust( - plonk_wasm.caml_pasta_fq_plonk_verifier_index_deep_copy( - tsRustConversion.fq.verifierIndexToRust(x) - ) - ); -}; diff --git a/src/lib/crypto/kimchi_bindings/js/bindings/verifier-index.js b/src/lib/crypto/kimchi_bindings/js/bindings/verifier-index.js new file mode 100644 index 00000000000..877d17084f4 --- /dev/null +++ b/src/lib/crypto/kimchi_bindings/js/bindings/verifier-index.js @@ -0,0 +1,152 @@ +/* global plonk_wasm, caml_jsstring_of_string, tsRustConversion + */ + +// Provides: caml_opt_of_rust +var caml_opt_of_rust = function (value, value_of_rust) { + if (value === undefined) { + return 0; + } else { + return [0, value_of_rust(value)]; + } +}; + +// Provides: caml_opt_to_rust +var caml_opt_to_rust = function (caml_optional_value, to_rust) { + // to_rust expects the parameters of the variant. A `Some vx` is represented + // as [0, vx] + if (caml_optional_value === 0) { + return undefined; + } else { + return to_rust(caml_optional_value[1]); + } +}; + +// Provides: caml_pasta_fp_plonk_verifier_index_create +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fp_plonk_verifier_index_create = function (x) { + var vk = plonk_wasm.caml_pasta_fp_plonk_verifier_index_create(x); + return tsRustConversion.fp.verifierIndexFromRust(vk); +}; + +// Provides: caml_pasta_fp_plonk_verifier_index_read +// Requires: plonk_wasm, caml_jsstring_of_string, tsRustConversion +var caml_pasta_fp_plonk_verifier_index_read = function (offset, urs, path) { + if (offset === 0) { + offset = undefined; + } else { + offset = offset[1]; + } + return tsRustConversion.fp.verifierIndexFromRust( + plonk_wasm.caml_pasta_fp_plonk_verifier_index_read( + offset, + urs, + caml_jsstring_of_string(path) + ) + ); +}; + +// Provides: caml_pasta_fp_plonk_verifier_index_write +// Requires: plonk_wasm, caml_jsstring_of_string, tsRustConversion +var caml_pasta_fp_plonk_verifier_index_write = function (append, t, path) { + if (append === 0) { + append = undefined; + } else { + append = append[1]; + } + return plonk_wasm.caml_pasta_fp_plonk_verifier_index_write( + append, + tsRustConversion.fp.verifierIndexToRust(t), + caml_jsstring_of_string(path) + ); +}; + +// Provides: caml_pasta_fp_plonk_verifier_index_shifts +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fp_plonk_verifier_index_shifts = function (log2_size) { + return tsRustConversion.fp.shiftsFromRust( + plonk_wasm.caml_pasta_fp_plonk_verifier_index_shifts(log2_size) + ); +}; + +// Provides: caml_pasta_fp_plonk_verifier_index_dummy +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fp_plonk_verifier_index_dummy = function () { + var res = plonk_wasm.caml_pasta_fp_plonk_verifier_index_dummy(); + return tsRustConversion.fp.verifierIndexFromRust(res); +}; + +// Provides: caml_pasta_fp_plonk_verifier_index_deep_copy +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fp_plonk_verifier_index_deep_copy = function (x) { + return tsRustConversion.fp.verifierIndexFromRust( + plonk_wasm.caml_pasta_fp_plonk_verifier_index_deep_copy( + tsRustConversion.fp.verifierIndexToRust(x) + ) + ); +}; + +// Provides: caml_pasta_fq_plonk_verifier_index_create +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fq_plonk_verifier_index_create = function (x) { + return tsRustConversion.fq.verifierIndexFromRust( + plonk_wasm.caml_pasta_fq_plonk_verifier_index_create(x) + ); +}; + +// Provides: caml_pasta_fq_plonk_verifier_index_read +// Requires: plonk_wasm, caml_jsstring_of_string, tsRustConversion +var caml_pasta_fq_plonk_verifier_index_read = function (offset, urs, path) { + if (offset === 0) { + offset = undefined; + } else { + offset = offset[1]; + } + return tsRustConversion.fq.verifierIndexFromRust( + plonk_wasm.caml_pasta_fq_plonk_verifier_index_read( + offset, + urs, + caml_jsstring_of_string(path) + ) + ); +}; + +// Provides: caml_pasta_fq_plonk_verifier_index_write +// Requires: plonk_wasm, caml_jsstring_of_string, tsRustConversion +var caml_pasta_fq_plonk_verifier_index_write = function (append, t, path) { + if (append === 0) { + append = undefined; + } else { + append = append[1]; + } + return plonk_wasm.caml_pasta_fq_plonk_verifier_index_write( + append, + tsRustConversion.fq.verifierIndexToRust(t), + caml_jsstring_of_string(path) + ); +}; + +// Provides: caml_pasta_fq_plonk_verifier_index_shifts +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fq_plonk_verifier_index_shifts = function (log2_size) { + return tsRustConversion.fq.shiftsFromRust( + plonk_wasm.caml_pasta_fq_plonk_verifier_index_shifts(log2_size) + ); +}; + +// Provides: caml_pasta_fq_plonk_verifier_index_dummy +// Requires: plonk_wasm, tsRustConversion +var caml_pasta_fq_plonk_verifier_index_dummy = function () { + return tsRustConversion.fq.verifierIndexFromRust( + plonk_wasm.caml_pasta_fq_plonk_verifier_index_dummy() + ); +}; + +// Provides: caml_pasta_fq_plonk_verifier_index_deep_copy +// Requires: plonk_wasm, tsRustConversion, tsRustConversion +var caml_pasta_fq_plonk_verifier_index_deep_copy = function (x) { + return tsRustConversion.fq.verifierIndexFromRust( + plonk_wasm.caml_pasta_fq_plonk_verifier_index_deep_copy( + tsRustConversion.fq.verifierIndexToRust(x) + ) + ); +}; diff --git a/src/lib/crypto/kimchi_bindings/js/dune b/src/lib/crypto/kimchi_bindings/js/dune index f5c99df7e7b..6c8243b06ff 100644 --- a/src/lib/crypto/kimchi_bindings/js/dune +++ b/src/lib/crypto/kimchi_bindings/js/dune @@ -14,7 +14,8 @@ bindings/proof.js bindings/prover-index.js bindings/util.js - bindings/srs.js)) + bindings/srs.js + bindings/verifier-index.js)) (instrumentation (backend bisect_ppx)) (preprocess From 2781571c89a2f6db74586a4e1e79511192872978 Mon Sep 17 00:00:00 2001 From: Florian Date: Mon, 28 Oct 2024 13:15:27 +0100 Subject: [PATCH 45/47] remove old bindings.js file --- src/lib/crypto/kimchi_bindings/js/bindings.js | 0 src/lib/crypto/kimchi_bindings/js/dune | 1 - 2 files changed, 1 deletion(-) delete mode 100644 src/lib/crypto/kimchi_bindings/js/bindings.js diff --git a/src/lib/crypto/kimchi_bindings/js/bindings.js b/src/lib/crypto/kimchi_bindings/js/bindings.js deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/src/lib/crypto/kimchi_bindings/js/dune b/src/lib/crypto/kimchi_bindings/js/dune index 6c8243b06ff..7c0eb98fe09 100644 --- a/src/lib/crypto/kimchi_bindings/js/dune +++ b/src/lib/crypto/kimchi_bindings/js/dune @@ -3,7 +3,6 @@ (public_name bindings_js) (js_of_ocaml (javascript_files - bindings.js bindings/bigint256.js bindings/field.js bindings/curve.js From d7bb993e99869b71462c861a530dd786cd933242 Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Mon, 28 Oct 2024 16:37:50 +0000 Subject: [PATCH 46/47] Fixup typo --- buildkite/src/Jobs/Test/PatchArchiveTest.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall index e6b468d843b..a0414c85a4f 100644 --- a/buildkite/src/Jobs/Test/PatchArchiveTest.dhall +++ b/buildkite/src/Jobs/Test/PatchArchiveTest.dhall @@ -28,7 +28,7 @@ in Pipeline.build , spec = JobSpec::{ , dirtyWhen = [ S.strictlyStart (S.contains "src") - , S.exactly "scripts/path-archive-test" "sh" + , S.exactly "scripts/patch-archive-test" "sh" , S.exactly "buildkite/src/Jobs/Test/PatchArchiveTest" "dhall" , S.exactly "buildkite/src/Command/PatchArchiveTest" "dhall" ] From 2cdc48a505ea83250fd7df89d9fe9356e697d7a2 Mon Sep 17 00:00:00 2001 From: mrmr1993 Date: Mon, 28 Oct 2024 16:42:42 +0000 Subject: [PATCH 47/47] Update old, invalid dirtyWhen --- buildkite/src/Constants/DebianVersions.dhall | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildkite/src/Constants/DebianVersions.dhall b/buildkite/src/Constants/DebianVersions.dhall index af12f14134d..4ca7d17a6c3 100644 --- a/buildkite/src/Constants/DebianVersions.dhall +++ b/buildkite/src/Constants/DebianVersions.dhall @@ -92,7 +92,7 @@ let bullseyeDirtyWhen = [ S.strictlyStart (S.contains "src") , S.strictlyStart (S.contains "automation") , S.strictly (S.contains "Makefile") - , S.exactly "buildkite/scripts/connect-to-berkeley" "sh" + , S.exactly "buildkite/scripts/connect-to-testnet" "sh" , S.exactly "buildkite/scripts/connect-to-mainnet-on-compatible" "sh" , S.exactly "buildkite/scripts/rosetta-integration-tests" "sh" , S.exactly "buildkite/scripts/rosetta-integration-tests-full" "sh"