Add SCOTCH library for geographical domain decomposition (#849)

Co-authored-by: JessicaMeixner-NOAA <[email protected]>
Co-authored-by: Matthew Masarik <[email protected]>
Co-authored-by: Ty Hesser <[email protected]>

.github/workflows/gnu.yml

@@ -7,7 +7,7 @@ concurrency:
   cancel-in-progress: true
 
 env:
-  cache_key: gnu5
+  cache_key: gnu8
   CC: gcc-10
   FC: gfortran-10
   CXX: g++-10

cmake/FindSCOTCH.cmake

@@ -0,0 +1,84 @@
+message(STATUS "Searching for PTSCOTCHparmetis library ...")
+find_library(ptscotchparmetis_lib NAMES libptscotchparmetisv3.a  HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
+find_path(ptscotchparmetis_inc parmetis.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
+
+message(STATUS "Searching for SCOTCH library ...")
+find_library(scotch_lib NAMES libscotch.a HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
+find_path(scotch_inc scotch.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
+
+message(STATUS "Searching for PTSCOTCH library ...")
+find_library(ptscotch_lib NAMES libptscotch.a HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
+find_path(ptscotch_inc ptscotch.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
+
+message(STATUS "Searching for SCOTCHerr library ...")
+find_library(scotcherr_lib NAMES libscotcherr.a HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
+find_path(scotcherr_inc scotch.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
+
+message(STATUS "Searching for PTSCOTCHerr library ...")
+find_library(ptscotcherr_lib NAMES libptscotcherr.a HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
+find_path(ptscotcherr_inc ptscotch.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
+
+add_library(PTSCOTCHparmetis::PTSCOTCHparmetis STATIC IMPORTED)
+add_library(SCOTCH::SCOTCH STATIC IMPORTED)
+add_library(PTSCOTCH::PTSCOTCH STATIC IMPORTED)
+add_library(SCOTCHerr::SCOTCHerr STATIC IMPORTED)
+add_library(PTSCOTCHerr::PTSCOTCHerr STATIC IMPORTED)
+
+set_target_properties(SCOTCH::SCOTCH PROPERTIES
+  IMPORTED_LOCATION "${scotch_lib}"
+  INTERFACE_INCLUDE_DIRECTORIES "${scotch_inc}")
+
+set_target_properties(SCOTCHerr::SCOTCHerr PROPERTIES
+  IMPORTED_LOCATION "${scotcherr_lib}"
+  INTERFACE_INCLUDE_DIRECTORIES "${scotcherr_inc}")
+
+set_target_properties(PTSCOTCH::PTSCOTCH PROPERTIES
+  IMPORTED_LOCATION "${ptscotch_lib}"
+  INTERFACE_INCLUDE_DIRECTORIES "${ptscotch_inc}")
+
+set_target_properties(PTSCOTCHerr::PTSCOTCHerr PROPERTIES
+  IMPORTED_LOCATION "${ptscotcherr_lib}"
+  INTERFACE_INCLUDE_DIRECTORIES "${ptscotcherr_inc}")
+
+set_target_properties(PTSCOTCHparmetis::PTSCOTCHparmetis PROPERTIES
+  IMPORTED_LOCATION "${ptscotchparmetis_lib}"
+  INTERFACE_INCLUDE_DIRECTORIES "${ptscotchparmetis_inc}")
+
+
+## Interfaces and links
+target_link_libraries(PTSCOTCHparmetis::PTSCOTCHparmetis INTERFACE PTSCOTCH::PTSCOTCH PTSCOTCHerr::PTSCOTCHerr  SCOTCH::SCOTCH)
+
+
+## Finalize find_package
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(
+    ${CMAKE_FIND_PACKAGE_NAME}
+    REQUIRED_VARS scotch_lib
+    scotch_inc)
+
+find_package_handle_standard_args(
+    ${CMAKE_FIND_PACKAGE_NAME}
+    REQUIRED_VARS ptscotch_lib
+    ptscotch_inc)
+
+find_package_handle_standard_args(
+    ${CMAKE_FIND_PACKAGE_NAME}
+    REQUIRED_VARS scotcherr_lib
+    scotcherr_inc)
+
+find_package_handle_standard_args(
+    ${CMAKE_FIND_PACKAGE_NAME}
+    REQUIRED_VARS ptscotcherr_lib
+    ptscotcherr_inc)
+
+find_package_handle_standard_args(
+    ${CMAKE_FIND_PACKAGE_NAME}
+    REQUIRED_VARS ptscotchparmetis_lib
+    ptscotchparmetis_inc)
+
+message(STATUS "Found SCOTCH: ${scotch_lib}")
+message(STATUS "Found PTSCOTCH: ${ptscotch_lib}")
+message(STATUS "Found SCOTCHerr: ${scotcherr_lib}")
+message(STATUS "Found PTSCOTCHerr: ${ptscotcherr_lib}")
+message(STATUS "Found PTSCOTCHparmetis: ${ptscotchparmetis_lib}")

model/bin/build_utils.sh

@@ -40,7 +40,7 @@ check_switches()
               dstress s_ice s_is reflection \
               wind windx wcor rwind curr currx mgwind mgprop mggse \
               subsec tdyn dss0 pdif tide refrx ig rotag nnt mprf \
-              cou oasis agcm ogcm igcm trknc setup pdlib memck uost rstwind b4b
+              cou oasis agcm ogcm igcm trknc setup pdlib ddlib memck uost rstwind b4b
   do
 
 # 1.a.1 Group switches by category
@@ -291,6 +291,12 @@ check_switches()
                ID='use pdlib'
                TS='PDLIB'
                OK='PDLIB' ;;
+
+#sort:ddlib:
+      ddlib  ) TY='upto1'
+               ID='domain decomposition library'
+               OK='METIS SCOTCH' ;;
+
 #sort:memck:
       memck  ) TY='upto1'
                ID='check memory use'
@@ -428,6 +434,7 @@ check_switches()
       igcm   ) igcm=$sw ;;
       trknc  ) trknc=$sw ;;
       pdlib  ) pdlib=$sw ;;
+      ddlib  ) ddlib=$sw ;;
       memck  ) memck=$sw ;;
       setup  ) setup=$sw ;;
       uost   ) uost=$sw ;;
@@ -547,6 +554,30 @@ check_switches()
       echo ' ' ; exit 17
   fi
 
+  if [ "$pdlib" = 'PDLIB' ] && [ "$ddlib" != 'SCOTCH' ]
+      then
+         if [ "$ddlib" != 'METIS' ]
+         then
+            echo ' '
+            echo "   *** For PDLIB, we need either SCOTCH or METIS, not both."
+            echo ' ' ; exit 18
+         fi
+  fi
+
+
+  if [ "$ddlib" = 'METIS' ] && [ "$pdlib" != 'PDLIB' ]
+  then
+      echo ' '
+      echo "   *** For METIS, we need to have PDLIB as well."
+      echo ' ' ; exit 19
+  fi
+
+  if [ "$ddlib" = 'SCOTCH' ] && [ "$pdlib" != 'PDLIB' ]
+  then
+      echo ' '
+      echo "   *** For SCOTCH, we need to have PDLIB as well."
+      echo ' ' ; exit 20
+  fi
 
 } #end of check_switches
 

model/bin/link.tmpl

@@ -117,9 +117,23 @@
     libs="$libs `$NETCDF_CONFIG --flibs` `$NETCDF_CONFIG --libs`"
   fi
 
+  # scotch library 
+  if [ "$prog" = 'ww3_shel' ] || [ "$prog" = 'ww3_multi' ] || [ "$prog" = 'ww3_sbs1' ] ; then
+    if [ "$scotch_mod" = 'yes' ] ; then
+      if [ -z "$(env | grep SCOTCH_PATH)" ] ; then
+        echo ''
+        echo "[ERROR] SCOTCH_PATH is not defined"
+        exit 1
+      fi
+      echo "link with scotch" 
+     libs="$libs $SCOTCH_PATH/lib/libptscotchparmetisv3.a $SCOTCH_PATH/lib/libptscotch.a  $SCOTCH_PATH/lib/libptscotcherr.a $SCOTCH_PATH/lib/libscotch.a"
+    fi
+  fi
+
+
   # parmetis library
   if [ "$prog" = 'ww3_shel' ] || [ "$prog" = 'ww3_multi' ] || [ "$prog" = 'ww3_sbs1' ] ; then
-    if [ "$pdlib_mod" = 'yes' ] ; then
+    if [ "$metis_mod" = 'yes' ] ; then
       if [ -z "$(env | grep METIS_PATH)" ] ; then
         echo ''
         echo "[ERROR] METIS_PATH is not defined"

model/bin/switch_Ifremer2_pdlib

@@ -1 +1 @@
-NOGRB PDLIB SCRIP SCRIPNC TRKNC DIST MPI PR3 UQ FLX0 LN1 ST4 STAB0 NL1 BT4 DB1 MLIM TR0 BS0 IC2 IS2 REF1 IG1 WNT2 WNX1 RWND CRT1 CRX1 TIDE O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7
+NOGRB PDLIB METIS SCRIP SCRIPNC TRKNC DIST MPI PR3 UQ FLX0 LN1 ST4 STAB0 NL1 BT4 DB1 MLIM TR0 BS0 IC2 IS2 REF1 IG1 WNT2 WNX1 RWND CRT1 CRX1 TIDE O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7

model/bin/switch_USACE_1

@@ -1 +1 @@
-NOGRB TRKNC DIST MPI MLIM PR3 UQ FLX1 PDLIB LN1 ST1 STAB0 NL1 BT4 DB0 TR0 BS0 IS0 IC0 REF0 WNT2 WNX1 CRT1 CRX1 O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7
+NOGRB TRKNC DIST MPI MLIM PR3 UQ FLX1 PDLIB SCOTCH LN1 ST1 STAB0 NL1 BT4 DB0 TR0 BS0 IS0 IC0 REF0 WNT2 WNX1 CRT1 CRX1 O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7

model/bin/switch_USACE_2

@@ -1 +1 @@
-NOGRB PDLIB TRKNC DIST MPI PR3 UQ FLX0 LN1 ST4 STAB0 NL1 BT1 DB1 MLIM TR0 BS0 WNT1 WNX1 IC0 IS0 REF0 RWND CRT1 CRX1 O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7
+NOGRB PDLIB SCOTCH TRKNC DIST MPI PR3 UQ FLX0 LN1 ST4 STAB0 NL1 BT1 DB1 MLIM TR0 BS0 WNT1 WNX1 IC0 IS0 REF0 RWND CRT1 CRX1 O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7

model/bin/switch_ite_pdlib

@@ -1 +1 @@
-NOGRB PDLIB TRKNC DIST MPI PR3 UQ FLX0 LN1 ST4 STAB0 NL1 BT4 DB1 TR0 BS0 IC0 IS0 REF1 WNT2 WNX1 RWND CRT1 CRX1 TIDE O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7
+NOGRB PDLIB SCOTCH TRKNC DIST MPI PR3 UQ FLX0 LN1 ST4 STAB0 NL1 BT4 DB1 TR0 BS0 IC0 IS0 REF1 WNT2 WNX1 RWND CRT1 CRX1 TIDE O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7

model/bin/switch_ugdev2

@@ -1 +1 @@
-NOGRB TRKNC DIST MPI SCRIP MLIM PR3 UQ FLX0 PDLIB LN1 ST4 STAB0 NL1 BT4 DB1 TR0 BS0 IS0 IC0 REF0 WNT2 WNX1 RWND CRT1 CRX1 O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7
+NOGRB TRKNC DIST MPI SCRIP MLIM PR3 UQ FLX0 PDLIB SCOTCH LN1 ST4 STAB0 NL1 BT4 DB1 TR0 BS0 IS0 IC0 REF0 WNT2 WNX1 RWND CRT1 CRX1 O0 O1 O2 O2a O2b O2c O3 O4 O5 O6 O7

model/bin/w3_automake

@@ -305,6 +305,8 @@
     ./sort_switch -s -r switch
     sed -e 's/DIST/SHRD/g' \
         -e 's/PDLIB //g'    \
+        -e 's/METIS //g'    \
+        -e 's/SCOTCH //g'    \
         -e 's/OMPG //g'    \
         -e 's/PDLIB //g'    \
         -e 's/OMPH //g'    \

model/bin/w3_make

@@ -498,13 +498,28 @@ EOF
       export oasis_mod
 
       pdlib_mod=no
+      metis_mod=no
+      scotch_mod=no
+
       esmfpdlib=''
       if [ -n "`grep PDLIB $switch_file`" ]
       then
         export pdlib_mod='yes'
-        esmfpdlib="$METIS_PATH/lib/libparmetis.a $METIS_PATH/lib/libmetis.a"
+        if [ -n "`grep METIS $switch_file`" ]
+        then
+             export metis_mod='yes'
+            esmfpdlib="$METIS_PATH/lib/libparmetis.a $METIS_PATH/lib/libmetis.a"
+        fi
+        if [ -n "`grep SCOTCH $switch_file`" ]
+        then
+            export scotch_mod='yes' 
+            esmfpdlib="$SCOTCH_PATH/lib/libptscotchparmetisv3.a $SCOTCH_PATH/lib/libptscotch.a \
+                       $SCOTCH_PATH/lib/libptscotcherr.a $SCOTCH_PATH/lib/libscotch.a"
+        fi
       fi
       export pdlib_mod
+      export metis_mod
+      export scotch_mod
 
       # NCEP GRIB
       export ncep_grib_compile="no"

model/ci/README.md

@@ -1,4 +1,4 @@
 # Continuous Integration
 WW3 uses Github Actions for continuous integration.
 
-This directory contains a Spack environment file to build library dependencies such as NetCDF, ESMF, ParMETIS, and g2.
+This directory contains a Spack environment file to build library dependencies such as NetCDF, ESMF, ParMETIS, SCOTCH and g2.

model/ci/spack_gnu.yaml

@@ -6,6 +6,7 @@ spack:
   specs:
   - [email protected]~shared
   - [email protected]~shared
+  - [email protected]+mpi+metis~shared
   - [email protected]~dap
   - [email protected]
   - [email protected]
@@ -15,4 +16,4 @@ spack:
   - [email protected]~pio~pnetcdf~xerces
   view: true
   concretizer:
-    unify: true
+    unify: when_possible

model/ci/spack_intel.yaml

@@ -7,6 +7,7 @@ spack:
   specs:
   - [email protected]~shared
   - [email protected]~shared
+  - [email protected]+mpi+metis~shared
   - [email protected]~dap
   - [email protected]
   - [email protected]
@@ -17,4 +18,4 @@ spack:
   - intel-oneapi-mpi %intel
   view: true
   concretizer:
-    unify: true
+    unify: when_possible

model/src/CMakeLists.txt

@@ -157,9 +157,17 @@ endif()
 
 # Handle PDLIB, SCRIP, SCRIPNC build files directly instead of through configuration file
 if("PDLIB" IN_LIST switches)
+  if("SCOTCH" IN_LIST switches)
+  find_package(SCOTCH REQUIRED)
+  target_sources(ww3_lib PRIVATE ${pdlib_src})
+  target_link_libraries(ww3_lib PUBLIC PTSCOTCHparmetis::PTSCOTCHparmetis)
+elseif("METIS"  IN_LIST switches)
   find_package(ParMETIS REQUIRED)
   target_sources(ww3_lib PRIVATE ${pdlib_src})
   target_link_libraries(ww3_lib PUBLIC ParMETIS::ParMETIS)
+  else()
+    message(FATAL_ERROR "PDLIB requires METIS or SCOTCH library for domain decomposition")
+ endif()
 endif()
 
 if("SCRIP" IN_LIST switches)

model/src/PDLIB/yowpdlibmain.F90

@@ -436,6 +436,7 @@ subroutine runParmetis(MNP)
     ! parmetis need single precision
     real(4), allocatable :: xyz(:), tpwgts(:), ubvec(:)
     integer :: IP_glob, itmp
+    integer :: ref
     logical :: lexist = .false.
 
     ! Node to domain mapping.
@@ -627,12 +628,24 @@ subroutine runParmetis(MNP)
     endif
 
     !if(debugParmetis) write(710+myrank,*) "Run ParMETIS now..."
+#ifdef W3_SCOTCH
+    call SCOTCH_ParMETIS_V3_PartGeomKway(vtxdist, xadj, adjncy, &
+         vwgt, & !vwgt - ignore weights
+         adjwgt, & ! adjwgt - ignore weights
+         wgtflag, &
+         numflag,ndims,xyz,ncon,nparts,tpwgts,ubvec,options, &
+         edgecut,part, comm,ref)
+#endif
+
+#ifdef W3_METIS
     call ParMETIS_V3_PartGeomKway(vtxdist, xadj, adjncy, &
          vwgt, & !vwgt - ignore weights
          adjwgt, & ! adjwgt - ignore weights
          wgtflag, &
          numflag,ndims,xyz,ncon,nparts,tpwgts,ubvec,options, &
          edgecut,part, comm)
+#endif
+
 
     CALL REAL_MPI_BARRIER_PDLIB(comm, "runParmetis, step 9")
 

model/src/cmake/switches.json

@@ -749,10 +749,25 @@
         "name": "pdlib",
         "num_switches": "upto1",
         "description": "use pdlib",
-        "valid-options": [
+        "valid-options":[
             {
                 "name": "PDLIB",
                 "requires": ["MPI"]
+       	    }
+        ]
+    },
+    {   
+        "name": "ddlib", 
+        "num_switches": "upto1",
+        "description": "domain decomposition library",
+        "valid-options": [
+            {   
+                "name": "METIS",
+                "requires": ["PDLIB"]
+            },  
+            {   
+                "name": "SCOTCH",
+                "requires": ["PDLIB"]
             }
         ]
     },

regtests/bin/matrix.base

@@ -2047,6 +2047,7 @@
    if [ "$pdlib" = 'y' ]
    then
      echo "$rtst -s MPI -s PDLIB -w work_b  -g b        -f -p $mpi -n $np $ww3 ww3_tp2.21" >> matrix.body
+     echo "$rtst -s MPI -s PDLIB_METIS -w work_b_metis -g b -f -p $mpi -n $np $ww3 ww3_tp2.21" >> matrix.body
      echo "$rtst -s MPI -s PDLIB -w work_mb -m grdset_b -f -p $mpi -n $np $ww3 ww3_tp2.21" >> matrix.body
    fi
  fi