PR adding support for Zarr V3

.github/workflows/main-cmake.yml

@@ -3,7 +3,7 @@ name: NetCDF-C CMake CI - Windows
 on: [ pull_request, workflow_dispatch]
 
 env:
-  REMOTETESTDOWN: ${{ vars.REMOTETESTDOWN }}
+  REMOTETESTDOWN: no
 
 concurrency:  
   group: ${{ github.workflow }}-${{ github.head_ref }}  
@@ -53,6 +53,12 @@ jobs:
         - name: Dump Matrix Context
           run: echo '${{ toJSON(matrix) }}'
 
+        - name: Check Hard Drive Space
+          shell: bash -el {0}
+          run: |
+            df -h
+            pwd
+
         #- run: echo "CMAKE_PREFIX_PATH=${env.CONDA_PREFIX}/Library" >> $GITHUB_ENV
         #- run: echo "/c/Users/runneradmin/miniconda3/Library/lib:${GITHUB_PATH}" >> $GITHUB_ENV
         #- run: echo ""
@@ -88,6 +94,12 @@ jobs:
             ls $CONDA_PREFIX/Library/include/
           shell: bash -el {0}
 
+        - name: Check Hard Drive Space
+          shell: bash -el {0}
+          run: |
+            df -h
+            pwd
+
         - name: Perform out-of-directory configuration
           shell: bash -el {0}
           run: |
@@ -108,6 +120,12 @@ jobs:
           run: |
             cd build
             cat libnetcdf.settings  
+
+        - name: Check Hard Drive Space Post Summary
+          shell: bash -el {0}
+          run: |
+            df -h
+            pwd
 
         - name: Perform out-of-directory build - libnetcdf
           shell: bash -el {0}
@@ -163,4 +181,6 @@ jobs:
           run: |
             cd build
             PATH=~/tmp/bin:$PATH ctest . --rerun-failed --output-on-failure -VV
+            df -h
+            pwd
           if: ${{ failure() }}    

.github/workflows/run_tests_cdash.yml

@@ -7,7 +7,7 @@ name: Run CDash Ubuntu/Linux netCDF Tests
 on: [workflow_dispatch]
 
 env:
-  REMOTETESTDOWN: ${{ vars.REMOTETESTDOWN }}
+  REMOTETESTDOWN: no
 
 concurrency:  
   group: ${{ github.workflow}}-${{ github.head_ref }}  

.github/workflows/run_tests_ubuntu.yml

@@ -7,7 +7,7 @@ name: Run Ubuntu/Linux netCDF Tests
 on: [pull_request,workflow_dispatch]
 
 env:
-  REMOTETESTDOWN: ${{ vars.REMOTETESTDOWN }}
+  REMOTETESTDOWN: no
 
 concurrency:  
   group: ${{ github.workflow}}-${{ github.head_ref }}  

.github/workflows/run_tests_win_cygwin.yml

@@ -10,7 +10,7 @@ env:
   SHELLOPTS: igncr
   CHERE_INVOKING: 1
   CYGWIN_NOWINPATH: 1
-  REMOTETESTDOWN: ${{ vars.REMOTETESTDOWN }}
+  REMOTETESTDOWN: no
 
 jobs:
   build-and-test-autotools:
@@ -26,6 +26,10 @@ jobs:
         plugin_dir_option: ["", "--without-plugin-dir"]
 
     steps:
+      - name: Set safe directory for Git
+        shell: pwsh 
+        run: git config --global --add safe.directory /cygdrive/d/a/netcdf-c/netcdf-c
+
       - name: Fix line endings
         shell: pwsh
         run: git config --global core.autocrlf input
@@ -55,7 +59,7 @@ jobs:
           /bin/dash ./configure --enable-hdf5 --enable-shared
           --disable-static --enable-dap --disable-dap-remote-tests
           --enable-plugins ${{ matrix.plugin_dir_option }}
-          --disable-nczarr --disable-nczarr-filters
+          --disable-nczarr #--disable-nczarr-filters
           --disable-s3 --with-s3-testing=no
 
 
@@ -103,8 +107,13 @@ jobs:
       run:
         shell: C:/cygwin/bin/bash.exe -eo pipefail -o igncr "{0}"
 
-    steps:
+#        shell: C:/cygwin/bin/bash.exe -eo pipefail -o igncr "{0}"
 
+    steps:
+      - name: Set safe directory for Git
+        shell: pwsh
+        run: git config --global --add safe.directory /cygdrive/d/a/netcdf-c/netcdf-c
+
       - run: git config --global core.autocrlf input
         shell: pwsh
       - uses: actions/checkout@v4

.github/workflows/run_tests_win_mingw.yml

@@ -46,7 +46,7 @@ jobs:
         run: autoreconf -if
 
       - name: (Autotools) Configure Build
-        run: ./configure --enable-hdf5 --enable-dap --disable-dap-remote-tests --disable-static --disable-byterange --disable-dap-remote-tests --disable-logging --enable-plugins --disable-nczarr-filters --disable-s3
+        run: ./configure --enable-hdf5 --enable-dap --disable-dap-remote-tests --disable-static --disable-byterange --disable-dap-remote-tests --disable-logging --enable-plugins --disable-s3 #--disable-nczarr-filters
         if: ${{ success() }}
 
       - name: (Autotools) Look at config.log if error

CMakeLists.txt

@@ -45,7 +45,6 @@ set(PACKAGE "netCDF" CACHE STRING "")
 include(netcdf_functions_macros)
 include(deprecated)
 
-
 # Backport of built-in `PROJECT_IS_TOP_LEVEL` from CMake 3.21
 if (NOT DEFINED NETCDF_IS_TOP_LEVEL)
   set(NETCDF_IS_TOP_LEVEL OFF)
@@ -440,18 +439,18 @@ endif()
 # Format Option checks
 ################################
 
-# As a long term goal, and because it is now the case that
-# NETCDF_ENABLE_NCZARR => USE_NETCDF4, so make the external options
-# NETCDF_ENABLE_NETCDF_4 and NETCDF_ENABLE_NETCDF4 obsolete
-# in favor of NETCDF_ENABLE_HDF5.
+# As a long term goal, and because it is now the case that --enable-nczarr
+# => USE_NETCDF4, make the external options --enable-netcdf-4 and
+# --enable-netcdf4 obsolete in favor of --enable-hdf5
 # We will do the following for one more release cycle.
-# 1. Make NETCDF_ENABLE_NETCDF_4 be an alias for NETCDF_ENABLE_NETCDF4.
-# 2. Make NETCDF_ENABLE_NETCDF4 an alias for NETCDF_ENABLE_HDF5.
-# 3. Internally, convert most (but not all) uses of USE_NETCDF_4 and USE_NETCDF4 to USE_HDF5.
+# 1. Make --enable-netcdf-4 be an alias for --enable-netcdf4.
+# 2. Make --enable-netcdf4 an alias for --enable-hdf5.
+# 3. Internally, convert most uses of USE_NETCDF_4 ad USE_NETCDF4 to USE_HDF5
 
-# Collect the values of NETCDF_ENABLE_NETCDF_4, NETCDF_ENABLE_NETCDF4, and NETCDF_ENABLE_HDF5.
+# Collect the values of -DNETCDF_ENABLE_NETCDF-4, -DNETCDF_ENABLE_NETCDF4, and -DNETCDF_ENABLE_HDF5.
+# Also determine which have been explicitly set on the command line.
 
-# Figure out which options are defined and process options
+# Figure out which options are defined
 if(DEFINED NETCDF_ENABLE_NETCDF_4)
     set(UNDEF_NETCDF_4 OFF CACHE BOOL "") 
     option(NETCDF_ENABLE_NETCDF_4 "" ON)
@@ -493,8 +492,11 @@ if(UNDEF_HDF5)
 set(NETCDF_ENABLE_HDF5 ON CACHE BOOL "" FORCE) 
 endif()
 
-# Turn off NETCDF_ENABLE_NETCDF4 because it will be used
-# as a shorthand for NETCDF_ENABLE_HDF5|NETCDF_ENABLE_HDF4|NETCDF_ENABLE_NCZARR
+# Turn off enable_netcdf_4 because it is no longer needed
+set(NETCDF_ENABLE_NETCDF_4 OFF CACHE BOOL "" FORCE)
+
+# Turn off enable_netcdf4 because it will be used
+# as a shorthand for ENABLE_HDF5|ENABLE_HDF4|ENABLE_NCZARR
 set(NETCDF_ENABLE_NETCDF4 OFF CACHE BOOL "" FORCE)
 option(NETCDF_ENABLE_DAP "Enable DAP2 and DAP4 Client." ON)
 option(NETCDF_ENABLE_NCZARR "Enable NCZarr Client." ON)
@@ -507,10 +509,28 @@ if(NETCDF_ENABLE_HDF4)
   set(USE_HDF4 ON)
 endif()
 
+# Decide default Zarr Format for creation
+OPTION(NETCDF_ENABLE_DEFAULT_ZARR_FORMAT_V3 "Specify the default Zarr format is V3" OFF)
+if(NETCDF_ENABLE_DEFAULT_ZARR_FORMAT_V3)
+SET(DFALTZARRFORMAT 3 CACHE STRING "")
+ELSE()
+SET(DFALTZARRFORMAT 2 CACHE STRING "")
+ENDIF()
+
+IF(NETCDF_ENABLE_NCZARR)
+  SET(NETCDF_ENABLE_NCZARR_V3 ON CACHE BOOL "Enable Zarr V3" FORCE)
+  # V3 is enabled if NCzarr is enabled
+ELSE()
+  SET(NETCDF_ENABLE_NCZARR_V3 OFF CACHE BOOL "Enable Zarr V3" FORCE)
+ENDIF()
+
 # Netcdf-4 support (i.e. libsrc4) is required by more than just HDF5 (e.g. NCZarr)
 # So depending on what above formats are enabled, enable netcdf-4
+# as a short-hand for this boolean expression
 if(NETCDF_ENABLE_HDF5 OR NETCDF_ENABLE_HDF4 OR NETCDF_ENABLE_NCZARR)
   set(NETCDF_ENABLE_NETCDF4 ON CACHE BOOL "Enable netCDF-4 API" FORCE)
+  # For convenience, define USE_NETCDF4
+  set(USE_NETCDF4 ON CACHE BOOL "Enable netCDF-4 API" FORCE)
 endif()
 
 # enable|disable all forms of network access
@@ -534,10 +554,7 @@ endif()
 # Did the user specify a default minimum blocksize for posixio?
 set(NCIO_MINBLOCKSIZE 256 CACHE STRING "Minimum I/O Blocksize for netCDF classic and 64-bit offset format files.")
 
-if(NETCDF_ENABLE_NETCDF4)
-  set(USE_NETCDF4 ON CACHE BOOL "")
-  set(NETCDF_ENABLE_NETCDF4 ON CACHE BOOL "")
-else()
+if(NOT USE_NETCDF4)
   set(USE_HDF4_FILE_TESTS OFF)
   set(USE_HDF4 OFF)
   set(NETCDF_ENABLE_HDF4_FILE_TESTS OFF)
@@ -993,7 +1010,7 @@ endif(NETCDF_ENABLE_ERANGE_FILL)
 # Now NETCDF_ENABLE_ERANGE_FILL is either AUTO, ON, or OFF
 
 # More relaxed coordinate check is now mandatory for all builds.
-set(ENABLE_ZERO_LENGTH_COORD_BOUND ON)
+set(NETCDF_ENABLE_ZERO_LENGTH_COORD_BOUND ON)
 
 # check and conform with PnetCDF settings on ERANGE_FILL and RELAX_COORD_BOUND
 if(USE_PNETCDF)
@@ -1016,7 +1033,7 @@ if(NETCDF_ENABLE_ERANGE_FILL)
   set(M4FLAGS "-DERANGE_FILL" CACHE STRING "")
 endif()
 
-if(ENABLE_ZERO_LENGTH_COORD_BOUND)
+if(NETCDF_ENABLE_ZERO_LENGTH_COORD_BOUND)
   message(STATUS "Enabling a more relaxed check for NC_EINVALCOORDS")
   target_compile_definitions(netcdf PRIVATE RELAX_COORD_BOUND)
 endif()
@@ -1540,6 +1557,12 @@ if(NETCDF_ENABLE_TESTS)
   if(NETCDF_ENABLE_NCZARR)
     include_directories(nczarr_test)
     add_subdirectory(nczarr_test)
+    if(NOT ISCMAKE)
+      # Cannot create target because another target with
+      # the same name already exists.
+      # See documentation for policy CMP0002 for more details.
+      add_subdirectory(v3_nczarr_test)
+     endif()
   endif()
 endif()
 
@@ -1673,10 +1696,10 @@ is_disabled(BUILD_SHARED_LIBS enable_static)
 is_enabled(BUILD_SHARED_LIBS enable_shared)
 
 is_enabled(NETCDF_ENABLE_V2_API HAS_NC2)
-is_enabled(NETCDF_ENABLE_NETCDF4 HAS_NC4)
 is_enabled(NETCDF_ENABLE_HDF4 HAS_HDF4)
 is_enabled(USE_HDF4 HAS_HDF4)
 is_enabled(USE_HDF5 HAS_HDF5)
+is_enabled(USE_NETCDF4 HAS_NC4)
 is_enabled(OFF HAS_BENCHMARKS)
 is_enabled(STATUS_PNETCDF HAS_PNETCDF)
 is_enabled(STATUS_PARALLEL HAS_PARALLEL)
@@ -1735,6 +1758,10 @@ endif()
 # WARNING: this must match the value in configure.ac
 set(S3TESTBUCKET "unidata-zarr-test-data" CACHE STRING "S3 test bucket")
 
+# Additional S3 Test Endpoing
+set(S3ENDPOINT "s3.us-east-1.amazonaws.com" CACHE STRING "S3 endpoint")
+
+
 # The working S3 path tree within the Unidata bucket.
 # WARNING: this must match the value in configure.ac
 set(S3TESTSUBTREE "netcdf-c" CACHE STRING "Working S3 path.")
@@ -1849,7 +1876,7 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/s3cleanup.in ${CMAKE_CURRENT_BINARY_D
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/s3gc.in ${CMAKE_CURRENT_BINARY_DIR}/s3gc.sh @ONLY NEWLINE_STYLE LF)
 
 #####
-# Build and copy nc_test4/findplugin.sh to various places
+# Transfer files from a single source to directories that use it.
 #####
 foreach(CC nc_test4 nczarr_test v3_nczarr_test plugins h5_test examples/C)
     configure_file(${CMAKE_CURRENT_SOURCE_DIR}/plugins/findplugin.in ${CMAKE_CURRENT_BINARY_DIR}/${CC}/findplugin.sh @ONLY NEWLINE_STYLE LF)

Makefile.am

@@ -110,8 +110,11 @@ endif
 
 # Build Cloud Storage if desired.
 if NETCDF_ENABLE_NCZARR
-ZARR_TEST_DIRS = nczarr_test
 ZARR = libnczarr
+ZARR_TEST_DIRS = nczarr_test
+if NETCDF_ENABLE_NCZARR_V3
+ZARR_TEST_DIRS += v3_nczarr_test
+endif
 endif
 
 # Optionally build test plugins
@@ -218,6 +221,9 @@ if NETCDF_ENABLE_S3_TESTALL
 	mv ${abs_top_builddir}/tmp_@[email protected] ${abs_top_builddir}/s3cleanup_@[email protected]
 endif
 
+clean-local:
+	rm -fr alltests_*
+
 if NETCDF_ENABLE_S3_TESTALL
 distclean-local:
 	rm -f ${abs_top_builddir}/s3cleanup_@[email protected]

RELEASE_NOTES.md

@@ -5,6 +5,9 @@ Release Notes       {#RELEASE_NOTES}
 
 This file contains a high-level description of this package's evolution. Releases are in reverse chronological order (most recent first). Note that, as of netcdf 4.2, the `netcdf-c++` and `netcdf-fortran` libraries have been separated into their own libraries.
 
+## 4.9.4 - TBD
+* Add experimental support for the Zarr Version 3 storage format. This code willl change as the Zarr Version 3 Specification evolves. See [Github #3068](https://github.com/Unidata/netcdf-c/pull/3068).
+
 ## 4.9.3 - TBD
 
 * Extend the netcdf API to support programmatic changes to the plugin search path. See [Github #3034](https://github.com/Unidata/netcdf-c/pull/3034) for more information.
@@ -26,8 +29,6 @@ This file contains a high-level description of this package's evolution. Release
 * Convert NCZarr V2 to store all netcdf-4 specific info as attributes. This improves interoperability with other Zarr implementations by no longer using non-standard keys. The price to be paid is that lazy attribute reading cannot be supported. See [Github #2836](https://github.com/Unidata/netcdf-c/pull/2936) for more information.
 * Cleanup the option code for NETCDF_ENABLE_SET_LOG_LEVEL\[_FUNC\] See [Github #2931](https://github.com/Unidata/netcdf-c/pull/2931) for more information.
 
-### Release Candidate 1 - July 26, 2024
-
 * Convert NCZarr V2 to store all netcdf-4 specific info as attributes. This improves interoperability with other Zarr implementations by no longer using non-standard keys. The price to be paid is that lazy attribute reading cannot be supported. See [Github #2836](https://github.com/Unidata/netcdf-c/issues/2936) for more information.
 * Cleanup the option code for NETCDF_ENABLE_SET_LOG_LEVEL\[_FUNC\] See [Github #2931](https://github.com/Unidata/netcdf-c/issues/2931) for more information.
 * Fix duplicate definition when using aws-sdk-cpp. See [Github #2928](https://github.com/Unidata/netcdf-c/issues/2928) for more information.

cmake/netcdf_functions_macros.cmake

@@ -221,7 +221,7 @@ macro(print_conf_summary)
   message("Configuration Summary:")
   message("")
   message(STATUS "Building Shared Libraries:     ${BUILD_SHARED_LIBS}")
-  message(STATUS "Building netCDF-4:             ${NETCDF_ENABLE_NETCDF_4}")
+  message(STATUS "Building netCDF-4:             ${NETCDF_ENABLE_NETCDF4}")
   message(STATUS "Building DAP2 Support:         ${NETCDF_ENABLE_DAP2}")
   message(STATUS "Building DAP4 Support:         ${NETCDF_ENABLE_DAP4}")
   message(STATUS "Building Byte-range Support:   ${NETCDF_ENABLE_BYTERANGE}")
@@ -277,6 +277,39 @@ macro(getdpkg_arch arch)
   execute_process(COMMAND "${NC_DPKG}" "--print-architecture" OUTPUT_VARIABLE "${arch}" OUTPUT_STRIP_TRAILING_WHITESPACE)
 endmacro(getdpkg_arch)
 
+macro(NCZARR_SH_TEST basename src)
+  file(READ ${CMAKE_CURRENT_SOURCE_DIR}/../${src}/tst_${basename}.sh SHSOURCE)
+  # Make sure the order of prepended lines is correct
+  string(PREPEND SHSOURCE "TESTNCZARR=1\n")
+  string(PREPEND SHSOURCE "#!/bin/bash\n")
+  # Replace with FILE(CONFIGURE) when cmake 3.18 is in common use
+  file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/run_${basename}.1 "${SHSOURCE}")
+  configure_file(${CMAKE_CURRENT_BINARY_DIR}/run_${basename}.1 ${CMAKE_CURRENT_BINARY_DIR}/run_${basename}.sh FILE_PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE @ONLY NEWLINE_STYLE LF)
+  file(REMOVE ${CMAKE_CURRENT_BINARY_DIR}/run_${basename}.1)
+endmacro(NCZARR_SH_TEST)
+
+macro(NCZARR_C_TEST basename newname src)
+  file(READ ${CMAKE_CURRENT_SOURCE_DIR}/../${src}/${basename}.c CSOURCE)
+  string(PREPEND CSOURCE "#define TESTNCZARR\n")
+  file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/${newname}.c "${CSOURCE}")
+endmacro(NCZARR_C_TEST)
+
+macro(build_bin_test_with_util_lib F UTIL_LIB)
+  build_bin_test(${F})
+  if(ENABLE_DLL)
+    target_compile_definitions(${F} PUBLIC -DDLL_NETCDF)
+  endif(ENABLE_DLL)
+  target_link_libraries(${F} ${UTIL_LIB} ${ALL_TLL_LIBS})
+endmacro()
+
+macro(add_bin_test_with_util_lib PREFIX F UTIL_LIB)
+  add_bin_test(${PREFIX} ${F})
+  if(ENABLE_DLL)
+    target_compile_definitions(${PREFIX}_${F} PUBLIC -DDLL_NETCDF)
+  endif(ENABLE_DLL)
+  target_link_libraries(${PREFIX}_${F} ${UTIL_LIB} ${ALL_TLL_LIBS})
+endmacro()
+
 ################################
 # Functions
 ################################
@@ -336,4 +369,3 @@ function(getlastdir s ret_val)
   list(GET list -1 last)
   set(${ret_val} "${last}" PARENT_SCOPE)
 endfunction()
-