Merge branch 'develop' into merge-back-to-develop-2024-1015

README-ci-failures.md

@@ -46,7 +46,7 @@ In this case, you should:
    [Google Cloud](https://console.cloud.google.com/gcr/images/o1labs-192920/global/mina-toolchain)
    where they have been automatically uploaded. Look for your branch name and
    commit hash in the second column. You should find several images suffixed
-   with `-bullseye` and `-focal` (Debian versions).
+   with `-bullseye` or `-focal` (Debian versions).
 3. For each such image, retrieve its full name and hash by hovering its link and
    clicking the `Copy full image name` tooltip that appears (or retrieve it on
    the image's page).

README.md

@@ -1,14 +1,18 @@
-### Build status
+<img src="https://github.com/MinaProtocol/mina-resources/blob/main/mina.png?raw=true" >
 
-| Develop | Berkeley | Compatible | 
-| ------- | -------- | ---------- |
-| [![Build status - develop](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=develop)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - berkeley](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=berkeley)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - compatible](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=compatible)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies)
+<h1 align="center" > Mina </h1>
 
-<a href="https://minaprotocol.com">
-  <img src="https://github.com/MinaProtocol/docs/blob/main/public/static/img/svg/mina-wordmark-redviolet.svg?raw=true&sanitize=true" width="350" alt="Mina logo">
-</a>
+<div align="center">
 
-# Mina
+  ![GitHub stars](https://img.shields.io/github/stars/minaprotocol/mina)&nbsp;&nbsp;![GitHub forks](https://img.shields.io/github/forks/minaprotocol/mina)
+
+![GitHub contributors](https://img.shields.io/github/contributors/minaprotocol/mina)&nbsp;&nbsp;![GitHub commit activity](https://img.shields.io/github/commit-activity/m/minaprotocol/mina)&nbsp;&nbsp;![GitHub last commit](https://img.shields.io/github/last-commit/minaprotocol/mina)
+
+| Develop[^1] | Compatible[^2] | Master[^3] | 
+| ------- | ---------- | ---------- |
+| [![Build status - develop](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=develop)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - compatible](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=compatible)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies) | [![Build status - master](https://badge.buildkite.com/0c47452f3ea619d3217d388e0de522b218db28c3e161887a9a.svg?branch=master)](https://buildkite.com/o-1-labs-2/mina-end-to-end-nightlies)
+
+</div>
 
 Mina is the first cryptocurrency with a lightweight, constant-sized blockchain. This is the main source code repository for the Mina project and contains code for the OCaml protocol implementation, the [Mina Protocol website](https://minaprotocol.com), and wallet. Enjoy!
 
@@ -60,3 +64,7 @@ The [Node Developers](https://docs.minaprotocol.com/node-developers) docs contai
 [Apache 2.0](LICENSE)
 
 Commits older than 2018-10-03 do not have a [LICENSE](LICENSE) file or this notice, but are distributed under the same terms.
+
+[^1]: Develop is a mainline branch containig code that may be not compatible with current mainnet and may require major upgrade (hardfork).
+[^2]: Compatible is a mainline branch containig code which does not need hardfork in order to apply it to mainnet.
+[^3]: Branch which contains current mainnet code.

automation/services/watchdog/Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.9.16-slim-buster
+FROM python:3.9.16-slim-bullseye
 
 ARG GCLOUDSDK_DOWNLOAD_URL="https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-296.0.1-linux-x86_64.tar.gz"
 ARG WATCHMAN_DEB_URL="http://ftp.us.debian.org/debian/pool/main/w/watchman/watchman_4.9.0-5+b1_amd64.deb"

buildkite/scripts/build-artifact.sh

@@ -23,7 +23,7 @@ make -C src/app/libp2p_helper
 MAINNET_TARGETS=""
 [[ ${MINA_BUILD_MAINNET} ]] && MAINNET_TARGETS="src/app/cli/src/mina_mainnet_signatures.exe src/app/rosetta/rosetta_mainnet_signatures.exe src/app/rosetta/ocaml-signer/signer_mainnet_signatures.exe"
 
-echo "--- Build all major tagets required for packaging"
+echo "--- Build all major targets required for packaging"
 echo "Building from Commit SHA: ${MINA_COMMIT_SHA1}"
 echo "Rust Version: $(rustc --version)"
 dune build "--profile=${DUNE_PROFILE}" $INSTRUMENTED_PARAM \
@@ -44,4 +44,5 @@ dune build "--profile=${DUNE_PROFILE}" $INSTRUMENTED_PARAM \
   src/app/rosetta/indexer_test/indexer_test.exe \
   src/app/rosetta/ocaml-signer/signer_testnet_signatures.exe \
   src/app/test_executive/test_executive.exe  \
-  src/test/command_line_tests/command_line_tests.exe # 2>&1 | tee /tmp/buildocaml.log
+  src/test/command_line_tests/command_line_tests.exe \
+  src/test/archive/patch_archive_test/patch_archive_test.exe

buildkite/scripts/build-release.sh

@@ -9,7 +9,7 @@ source ~/.profile
 ./buildkite/scripts/build-artifact.sh
 
 echo "--- Bundle all packages for Debian ${MINA_DEB_CODENAME}"
-echo " Includes mina daemon, archive-node, rosetta, generate keypair for berkeley"
+echo " Includes mina daemon, archive-node, rosetta, generate keypair for devnet"
 [[ ${MINA_BUILD_MAINNET} ]] && echo " MINA_BUILD_MAINNET is true so this includes the mainnet and devnet packages for mina-daemon as well"
 
 

buildkite/scripts/check-compatibility.sh

@@ -10,7 +10,7 @@ function get_shas {
 
 function image_tag {
     SHA=$1
-    IMAGE_TAG="$SHA-bullseye-berkeley"
+    IMAGE_TAG="$SHA-bullseye-devnet"
 }
 
 function download-docker {
@@ -22,17 +22,25 @@ function download-docker {
 function try_docker_shas {
     DOCKER_SHAS=$1
     GOT_DOCKER=0
-
     for sha in $DOCKER_SHAS; do
-	download-docker $sha
-	if [ $? -eq 0 ] ; then
-	    GOT_DOCKER=1
-	    image_tag $sha
-	    break
-	else
-	    echo "No docker available for SHA=$sha"
-	fi
+
+        set +e
+        download-docker $sha
+
+        if [ $? -eq 0 ] ; then
+            GOT_DOCKER=1
+            image_tag $sha
+            break
+        else
+            echo "No docker available for SHA=$sha"
+        fi
+        set -e
     done
+
+    if [[ $GOT_DOCKER == 0 ]]; then
+        echo "docker cannot be found for given shas: $DOCKER_SHAS"
+        exit 1
+    fi
 }
 
 function image_id {

buildkite/scripts/connect-to-testnet.sh

@@ -7,53 +7,46 @@ if [[ $# -ne 3 ]]; then
     exit 1
 fi
 
-TESTNET_VERSION_NAME="berkeley"
 TESTNET_NAME=$1
 WAIT_BETWEEN_POLLING_GRAPHQL=$2
 WAIT_AFTER_FINAL_CHECK=$3
 
-case "$BUILDKITE_PULL_REQUEST_BASE_BRANCH" in
-  rampup|berkeley|release/2.0.0|develop)
-  ;;
-  *)
-    echo "Not pulling against rampup, not running the connect test"
-    exit 0 ;;
-esac
-
 # Don't prompt for answers during apt-get install
 export DEBIAN_FRONTEND=noninteractive
 
-apt-get update
-apt-get install -y git apt-transport-https ca-certificates tzdata curl
+sudo apt-get update
+sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl libwww-perl jq
 
 git config --global --add safe.directory /workdir
 
 source buildkite/scripts/export-git-env-vars.sh
 
-source buildkite/scripts/debian/install.sh "mina-${TESTNET_VERSION_NAME}"
+source buildkite/scripts/debian/install.sh "mina-${TESTNET_NAME}" 1
 
 # Remove lockfile if present
-rm ~/.mina-config/.mina-lock ||:
+sudo rm ~/.mina-config/.mina-lock ||:
+
+sudo mkdir -p /root/libp2p-keys/
 
-mkdir -p /root/libp2p-keys/
-# Pre-generated random password for this quick test
-export MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf
-mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key
 # Set permissions on the keypair so the daemon doesn't complain
-chmod -R 0700 /root/libp2p-keys/
+sudo chmod -R 0700 /root/libp2p-keys/
+# Pre-generated random password for this quick test
+sudo MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf mina libp2p generate-keypair --privkey-path /root/libp2p-keys/key
 
 # Restart in the background
-mina daemon \
-  --peer-list-url "https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt" \
-  --libp2p-keypair "/root/libp2p-keys/key" \
-& # -background
+sudo MINA_LIBP2P_PASS=eithohShieshichoh8uaJ5iefo1reiRudaekohG7AeCeib4XuneDet2uGhu7lahf \
+    TESTNET_NAME=$TESTNET_NAME \
+  bash -c "mina daemon \
+  --peer-list-url \"https://storage.googleapis.com/seed-lists/${TESTNET_NAME}_seeds.txt\" \
+  --libp2p-keypair \"/root/libp2p-keys/key\" \
+  --seed &" # -background
 
 # Attempt to connect to the GraphQL client every 10s for up to 8 minutes
 num_status_retries=24
 for ((i=1;i<=$num_status_retries;i++)); do
   sleep $WAIT_BETWEEN_POLLING_GRAPHQL
   set +e
-  mina client status
+  sudo mina client status
   status_exit_code=$?
   set -e
   if [ $status_exit_code -eq 0 ]; then
@@ -63,13 +56,18 @@ for ((i=1;i<=$num_status_retries;i++)); do
   fi
 done
 
-# Check that the daemon has connected to peers and is still up after 2 mins
-sleep $WAIT_AFTER_FINAL_CHECK
-mina client status
-if [ $(mina advanced get-peers | wc -l) -gt 0 ]; then
-    echo "Found some peers"
-else
-    echo "No peers found"
-    exit 1
-fi
+peer_retries=10
+for ((i=1;i<=$peer_retries;i++)); do
+  peer_count=$(sudo mina advanced get-peers | wc -l)
+  sudo mina client status
+
+  if [ "$peer_count" -gt 0 ]; then
+      echo "Found some peers"
+      exit 0;      
+  else
+      echo "No peers found"
+  fi
+  sleep $WAIT_AFTER_FINAL_CHECK
+done
 
+exit 1;

buildkite/scripts/debian/install.sh

@@ -34,7 +34,7 @@ else
   debs=(${DEBS//,/ })
   for i in "${debs[@]}"; do
     case $i in
-      mina-berkeley|mina-devnet|mina-mainnet|mina-berkeley-lightnet)
+      mina-devnet|mina-mainnet|mina-devnet-lightnet)
         # Downaload mina-logproc too
         source ./buildkite/scripts/download-artifact-from-cache.sh "mina-logproc*" $MINA_DEB_CODENAME/_build "" $LOCAL_DEB_FOLDER
       ;;

buildkite/scripts/dump-mina-type-shapes.sh

@@ -8,7 +8,7 @@ export DEBIAN_FRONTEND=noninteractive
 sudo apt-get update
 sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl
 
-TESTNET_NAME="berkeley"
+TESTNET_NAME="devnet"
 
 git config --global --add safe.directory /workdir
 source buildkite/scripts/export-git-env-vars.sh

buildkite/scripts/rosetta-integration-tests.sh

@@ -250,10 +250,12 @@ if [[ "$MODE" == "full" ]]; then
   send_zkapp_transactions &
 fi
 
-next_block_time=$(mina client status --json | jq '.next_block_production.timing[1].time' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13)
-sleep_time=$((($next_block_time - $curr_time) / 1000))
+mina client status --json
+
+next_block_time=$(mina client status --json | jq '.next_block_production.timing[1]' | tr -d '"') curr_time=$(date +%s%N | cut -b1-13)
+sleep_time=$((($next_block_time - $curr_time) / 10000000))
 echo "Sleeping for ${sleep_time}s until next block is created..."
-sleep ${sleep_time}
+sleep $sleep_time
 
 # Mina Rosetta Checks (spec construction data perf)
 echo "============ ROSETTA CLI: VALIDATE CONF FILE ${ROSETTA_CONFIGURATION_FILE} =============="

buildkite/scripts/run-snark-transaction-profiler.sh

@@ -8,7 +8,7 @@ export DEBIAN_FRONTEND=noninteractive
 sudo apt-get update
 sudo apt-get install -y git apt-transport-https ca-certificates tzdata curl python3
 
-TESTNET_NAME="berkeley"
+TESTNET_NAME="devnet"
 
 git config --global --add safe.directory /workdir
 source buildkite/scripts/export-git-env-vars.sh

buildkite/scripts/run-test-executive-cloud.sh

@@ -2,7 +2,7 @@
 set -o pipefail -x
 
 TEST_NAME="$1"
-MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-berkeley"
+MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-devnet"
 ARCHIVE_IMAGE="gcr.io/o1labs-192920/mina-archive:$MINA_DOCKER_TAG"
 
 if [[ "${TEST_NAME:0:15}" == "block-prod-prio" ]] && [[ "$RUN_OPT_TESTS" == "" ]]; then

buildkite/scripts/run-test-executive-local.sh

@@ -26,7 +26,7 @@ cleanup
 
 TEST_NAME="$1"
 
-MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-berkeley"
+MINA_IMAGE="gcr.io/o1labs-192920/mina-daemon:$MINA_DOCKER_TAG-devnet"
 ARCHIVE_IMAGE="gcr.io/o1labs-192920/mina-archive:$MINA_DOCKER_TAG"
 
 if [[ "${TEST_NAME:0:15}" == "block-prod-prio" ]] && [[ "$RUN_OPT_TESTS" == "" ]]; then
@@ -42,7 +42,7 @@ rm -f /etc/apt/sources.list.d/hashicorp.list
 apt-get update
 apt-get install -y git apt-transport-https ca-certificates tzdata curl
 
-TESTNET_NAME="berkeley"
+TESTNET_NAME="devnet"
 
 git config --global --add safe.directory /workdir
 

buildkite/scripts/run_promote_build_job.sh

@@ -61,7 +61,7 @@ DHALL_DEBIANS="([] : List $DEBIAN_DHALL_DEF.Type)"
 if [[ -n "$DEBIANS" ]]; then 
     if [[ -z "$CODENAMES" ]]; then usage "Codenames is not set!"; exit 1; fi;
     if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi;
-    if [[ -z "$NETWORK" ]]; then NETWORK="Berkeley"; fi;
+    if [[ -z "$NETWORK" ]]; then NETWORK="Devnet"; fi;
     if [[ -z "$REMOVE_PROFILE_FROM_NAME" ]]; then REMOVE_PROFILE_FROM_NAME=0; fi;
     if [[ -z "$PUBLISH" ]]; then PUBLISH=0; fi;
     if [[ -z "$FROM_CHANNEL" ]]; then FROM_CHANNEL="Unstable"; fi;

buildkite/scripts/run_verify_promoted_build_job.sh

@@ -59,7 +59,7 @@ DHALL_DEBIANS="([] : List $DEBIAN_DHALL_DEF.Type)"
 if [[ -n "$DEBIANS" ]]; then 
     if [[ -z "$CODENAMES" ]]; then usage "Codenames is not set!"; exit 1; fi;
     if [[ -z "$PROFILE" ]]; then PROFILE="Standard"; fi;
-    if [[ -z "$NETWORK" ]]; then NETWORK="Berkeley"; fi;
+    if [[ -z "$NETWORK" ]]; then NETWORK="Devnet"; fi;
     if [[ -z "$REMOVE_PROFILE_FROM_NAME" ]]; then REMOVE_PROFILE_FROM_NAME=0; fi;
     if [[ -z "$PUBLISH" ]]; then PUBLISH=0; fi;
     if [[ -z "$TO_CHANNEL" ]]; then TO_CHANNEL="Unstable"; fi;

buildkite/scripts/single-node-tests.sh

@@ -12,7 +12,7 @@ git config --global --add safe.directory /workdir
 
 source buildkite/scripts/export-git-env-vars.sh
 
-source buildkite/scripts/debian/install.sh "mina-test-suite,mina-berkeley-lightnet" 1
+source buildkite/scripts/debian/install.sh "mina-test-suite,mina-devnet-lightnet" 1
 
 export MINA_LIBP2P_PASS="naughty blue worm"
 export MINA_PRIVKEY_PASS="naughty blue worm"