ww3 scotch

cmake/FindPTSCOTCH.cmake

@@ -0,0 +1,20 @@
+message(STATUS "Searching for PTSCOTCH library ...")
+
+find_library(ptscotch_lib NAMES libptscotch.a HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
+find_path(ptscotch_inc ptscotch.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
+
+add_library(PTSCOTCH::PTSCOTCH STATIC IMPORTED)
+
+set_target_properties(PTSCOTCH::PTSCOTCH PROPERTIES
+  IMPORTED_LOCATION "${ptscotch_lib}"
+  INTERFACE_INCLUDE_DIRECTORIES "${ptscotch_inc}")
+
+## Finalize find_package
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(
+    ${CMAKE_FIND_PACKAGE_NAME}
+    REQUIRED_VARS ptscotch_lib
+    ptscotch_inc)
+
+message(STATUS "Found PTSCOTCH: ${ptscotch_lib}")

cmake/FindPTSCOTCHerr.cmake

@@ -0,0 +1,20 @@
+message(STATUS "Searching for PTSCOTCHerr library ...")
+
+find_library(ptscotcherr_lib NAMES libptscotcherr.a HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
+find_path(ptscotcherr_inc ptscotch.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
+
+add_library(PTSCOTCHerr::PTSCOTCHerr STATIC IMPORTED)
+
+set_target_properties(PTSCOTCHerr::PTSCOTCHerr PROPERTIES
+  IMPORTED_LOCATION "${ptscotcherr_lib}"
+  INTERFACE_INCLUDE_DIRECTORIES "${ptscotcherr_inc}")
+
+## Finalize find_package
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(
+    ${CMAKE_FIND_PACKAGE_NAME}
+    REQUIRED_VARS ptscotcherr_lib
+    ptscotcherr_inc)
+
+message(STATUS "Found PTSCOTCHerr: ${ptscotcherr_lib}")

cmake/FindPTSCOTCHparmetis.cmake

@@ -0,0 +1,20 @@
+message(STATUS "Searching for PTSCOTCHparmetis library ...")
+find_library(ptscotchparmetis_lib NAMES libptscotchparmetisv3.a  HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
+find_path(ptscotchparmetis_inc parmetis.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
+
+add_library(PTSCOTCHparmetis::PTSCOTCHparmetis STATIC IMPORTED)
+
+set_target_properties(PTSCOTCHparmetis::PTSCOTCHparmetis PROPERTIES
+  IMPORTED_LOCATION "${ptscotchparmetis_lib}"
+  INTERFACE_INCLUDE_DIRECTORIES "${ptscotchparmetis_inc}")
+
+## Finalize find_package
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(
+    ${CMAKE_FIND_PACKAGE_NAME}
+    REQUIRED_VARS ptscotchparmetis_lib
+    ptscotchparmetis_inc)
+
+message(STATUS "Found PTSCOTCHparmetis: ${ptscotchparmetis_lib}")
+

cmake/FindSCOTCH.cmake

@@ -0,0 +1,21 @@
+message(STATUS "Searching for SCOTCH library ...")
+
+find_library(scotch_lib NAMES libscotch.a HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
+find_path(scotch_inc scotch.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
+
+add_library(SCOTCH::SCOTCH STATIC IMPORTED)
+
+set_target_properties(SCOTCH::SCOTCH PROPERTIES
+  IMPORTED_LOCATION "${scotch_lib}"
+  INTERFACE_INCLUDE_DIRECTORIES "${scotch_inc}")
+
+## Finalize find_package
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(
+    ${CMAKE_FIND_PACKAGE_NAME}
+    REQUIRED_VARS scotch_lib
+    scotch_inc)
+
+message(STATUS "Found SCOTCH: ${scotch_lib}")
+

cmake/FindSCOTCHerr.cmake

@@ -0,0 +1,22 @@
+message(STATUS "Searching for SCOTCHerr library ...")
+
+find_library(scotcherr_lib NAMES libscotcherr.a HINTS ENV SCOTCH_PATH PATH_SUFFIXES lib)
+find_path(scotcherr_inc scotch.h HINTS ENV SCOTCH_PATH PATH_SUFFIXES include)
+
+add_library(SCOTCHerr::SCOTCHerr STATIC IMPORTED)
+
+
+set_target_properties(SCOTCHerr::SCOTCHerr PROPERTIES
+  IMPORTED_LOCATION "${scotcherr_lib}"
+  INTERFACE_INCLUDE_DIRECTORIES "${scotcherr_inc}")
+
+
+## Finalize find_package
+include(FindPackageHandleStandardArgs)
+
+find_package_handle_standard_args(
+    ${CMAKE_FIND_PACKAGE_NAME}
+    REQUIRED_VARS scotcherr_lib
+    scotcherr_inc)
+
+message(STATUS "Found SCOTCHerr: ${scotcherr_lib}")

model/bin/build_utils.sh

@@ -40,7 +40,7 @@ check_switches()
               dstress s_ice s_is reflection \
               wind windx wcor rwind curr currx mgwind mgprop mggse \
               subsec tdyn dss0 pdif tide refrx ig rotag nnt mprf \
-              cou oasis agcm ogcm igcm trknc setup pdlib memck uost rstwind b4b
+              cou oasis agcm ogcm igcm trknc setup pdlib ddlib memck uost rstwind b4b
   do
 
 # 1.a.1 Group switches by category
@@ -291,6 +291,12 @@ check_switches()
                ID='use pdlib'
                TS='PDLIB'
                OK='PDLIB' ;;
+
+#sort:ddlib:
+      ddlib  ) TY='upto1'
+               ID='domain decomposition library'
+               OK='METIS SCOTCH' ;;
+
 #sort:memck:
       memck  ) TY='upto1'
                ID='check memory use'
@@ -428,6 +434,7 @@ check_switches()
       igcm   ) igcm=$sw ;;
       trknc  ) trknc=$sw ;;
       pdlib  ) pdlib=$sw ;;
+      ddlib  ) ddlib=$sw ;;
       memck  ) memck=$sw ;;
       setup  ) setup=$sw ;;
       uost   ) uost=$sw ;;
@@ -547,6 +554,30 @@ check_switches()
       echo ' ' ; exit 17
   fi
 
+  if [ "$pdlib" = 'PDLIB' ] && [ "$ddlib" != 'SCOTCH' ]
+      then
+         if [ "$ddlib" != 'METIS' ]
+         then
+            echo ' '
+            echo "   *** For PDLIB, we need either SCOTCH or METIS, not both."
+            echo ' ' ; exit 18
+         fi
+  fi
+
+
+  if [ "$ddlib" = 'METIS' ] && [ "$pdlib" != 'PDLIB' ]
+  then
+      echo ' '
+      echo "   *** For METIS, we need to have PDLIB as well."
+      echo ' ' ; exit 19
+  fi
+
+  if [ "$ddlib" = 'SCOTCH' ] && [ "$pdlib" != 'PDLIB' ]
+  then
+      echo ' '
+      echo "   *** For SCOTCH, we need to have PDLIB as well."
+      echo ' ' ; exit 20
+  fi
 
 } #end of check_switches
 

model/bin/link.tmpl

@@ -117,9 +117,23 @@
     libs="$libs `$NETCDF_CONFIG --flibs` `$NETCDF_CONFIG --libs`"
   fi
 
+  # scotch library 
+  if [ "$prog" = 'ww3_shel' ] || [ "$prog" = 'ww3_multi' ] || [ "$prog" = 'ww3_sbs1' ] ; then
+    if [ "$scotch_mod" = 'yes' ] ; then
+      if [ -z "$(env | grep SCOTCH_PATH)" ] ; then
+        echo ''
+        echo "[ERROR] SCOTCH_PATH is not defined"
+        exit 1
+      fi
+      echo "link with scotch" 
+     libs="$libs $SCOTCH_PATH/lib/libptscotchparmetisv3.a $SCOTCH_PATH/lib/libptscotch.a  $SCOTCH_PATH/lib/libptscotcherr.a $SCOTCH_PATH/lib/libscotch.a"
+    fi
+  fi
+
+
   # parmetis library
   if [ "$prog" = 'ww3_shel' ] || [ "$prog" = 'ww3_multi' ] || [ "$prog" = 'ww3_sbs1' ] ; then
-    if [ "$pdlib_mod" = 'yes' ] ; then
+    if [ "$metis_mod" = 'yes' ] ; then
       if [ -z "$(env | grep METIS_PATH)" ] ; then
         echo ''
         echo "[ERROR] METIS_PATH is not defined"

model/bin/w3_automake

@@ -305,6 +305,8 @@
     ./sort_switch -s -r switch
     sed -e 's/DIST/SHRD/g' \
         -e 's/PDLIB //g'    \
+        -e 's/METIS //g'    \
+        -e 's/SCOTCH //g'    \
         -e 's/OMPG //g'    \
         -e 's/PDLIB //g'    \
         -e 's/OMPH //g'    \

model/bin/w3_make

@@ -498,13 +498,28 @@ EOF
       export oasis_mod
 
       pdlib_mod=no
+      metis_mod=no
+      scotch_mod=no
+
       esmfpdlib=''
       if [ -n "`grep PDLIB $switch_file`" ]
       then
         export pdlib_mod='yes'
-        esmfpdlib="$METIS_PATH/lib/libparmetis.a $METIS_PATH/lib/libmetis.a"
+        if [ -n "`grep METIS $switch_file`" ]
+        then
+             export metis_mod='yes'
+            esmfpdlib="$METIS_PATH/lib/libparmetis.a $METIS_PATH/lib/libmetis.a"
+        fi
+        if [ -n "`grep SCOTCH $switch_file`" ]
+        then
+            export scotch_mod='yes' 
+            esmfpdlib="$SCOTCH_PATH/lib/libptscotchparmetisv3.a $SCOTCH_PATH/lib/libptscotch.a \
+                       $SCOTCH_PATH/lib/libptscotcherr.a $SCOTCH_PATH/lib/libscotch.a"
+        fi
       fi
       export pdlib_mod
+      export metis_mod
+      export scotch_mod
 
       # NCEP GRIB
       export ncep_grib_compile="no"

model/ci/README.md

@@ -1,4 +1,4 @@
 # Continuous Integration
 WW3 uses Github Actions for continuous integration.
 
-This directory contains a Spack environment file to build library dependencies such as NetCDF, ESMF, ParMETIS, and g2.
+This directory contains a Spack environment file to build library dependencies such as NetCDF, ESMF, ParMETIS, SCOTCH and g2.

model/ci/spack.yaml

@@ -5,6 +5,7 @@ spack:
   specs:
   - [email protected]~shared
   - [email protected]~shared
+  - [email protected]~shared
   - [email protected]~dap
   - [email protected]
   - [email protected]

model/src/CMakeLists.txt

@@ -157,9 +157,24 @@ endif()
 
 # Handle PDLIB, SCRIP, SCRIPNC build files directly instead of through configuration file
 if("PDLIB" IN_LIST switches)
+  if("SCOTCH" IN_LIST switches)
+  find_package(PTSCOTCHparmetis REQUIRED)
+  find_package(PTSCOTCH REQUIRED)
+  find_package(PTSCOTCHerr REQUIRED)
+  find_package(SCOTCHerr REQUIRED)
+  find_package(SCOTCH REQUIRED)
+  target_sources(ww3_lib PRIVATE ${pdlib_src})
+  target_link_libraries(PTSCOTCHparmetis::PTSCOTCHparmetis INTERFACE PTSCOTCHerr::PTSCOTCHerr SCOTCHerr::SCOTCHerr PTSCOTCH::PTSCOTCH SCOTCH::SCOTCH)
+  target_link_libraries(ww3_lib PUBLIC PTSCOTCHparmetis::PTSCOTCHparmetis SCOTCHerr::SCOTCHerr)
+ endif()
+endif()
+
+if("PDLIB" IN_LIST switches)
+ if ("METIS"  IN_LIST switches)
   find_package(ParMETIS REQUIRED)
   target_sources(ww3_lib PRIVATE ${pdlib_src})
   target_link_libraries(ww3_lib PUBLIC ParMETIS::ParMETIS)
+ endif()
 endif()
 
 if("SCRIP" IN_LIST switches)

model/src/PDLIB/yowpdlibmain.F90

@@ -486,6 +486,7 @@ subroutine runParmetis(MNP)
     ! parmetis need single precision
     real(4), allocatable :: xyz(:), tpwgts(:), ubvec(:)
     integer :: IP_glob, itmp
+    integer :: ref
     logical :: lexist = .false.
 
     ! Node to domain mapping.
@@ -677,12 +678,24 @@ subroutine runParmetis(MNP)
     endif
 
     !if(debugParmetis) write(710+myrank,*) "Run ParMETIS now..."
+#ifdef W3_SCOTCH
+    call SCOTCH_ParMETIS_V3_PartGeomKway(vtxdist, xadj, adjncy, &
+         vwgt, & !vwgt - ignore weights
+         adjwgt, & ! adjwgt - ignore weights
+         wgtflag, &
+         numflag,ndims,xyz,ncon,nparts,tpwgts,ubvec,options, &
+         edgecut,part, comm,ref)
+#endif
+
+#ifdef W3_METIS
     call ParMETIS_V3_PartGeomKway(vtxdist, xadj, adjncy, &
          vwgt, & !vwgt - ignore weights
          adjwgt, & ! adjwgt - ignore weights
          wgtflag, &
          numflag,ndims,xyz,ncon,nparts,tpwgts,ubvec,options, &
          edgecut,part, comm)
+#endif
+
 
     CALL REAL_MPI_BARRIER_PDLIB(comm, "runParmetis, step 9")
 

model/src/cmake/switches.json

@@ -749,10 +749,25 @@
         "name": "pdlib",
         "num_switches": "upto1",
         "description": "use pdlib",
-        "valid-options": [
+        "valid-options":[
             {
                 "name": "PDLIB",
                 "requires": ["MPI"]
+       	    }
+        ]
+    },
+    {   
+        "name": "ddlib", 
+        "num_switches": "upto1",
+        "description": "domain decomposition library",
+        "valid-options": [
+            {   
+                "name": "METIS",
+                "requires": ["PDLIB"]
+            },  
+            {   
+                "name": "SCOTCH",
+                "requires": ["PDLIB"]
             }
         ]
     },

regtests/bin/matrix.base

@@ -2047,6 +2047,7 @@
    if [ "$pdlib" = 'y' ]
    then
      echo "$rtst -s MPI -s PDLIB -w work_b  -g b        -f -p $mpi -n $np $ww3 ww3_tp2.21" >> matrix.body
+     echo "$rtst -s MPI -s PDLIB_METIS -w work_b_metis -g b -f -p $mpi -n $np $ww3 ww3_tp2.21" >> matrix.body
      echo "$rtst -s MPI -s PDLIB -w work_mb -m grdset_b -f -p $mpi -n $np $ww3 ww3_tp2.21" >> matrix.body
    fi
  fi

regtests/bin/matrix_cmake_ncep

@@ -65,6 +65,7 @@ EOF
     modcomp='hpc-intel/18.0.5.274'
     modmpi='hpc-impi/2018.0.4'
     metispath='/scratch2/COASTAL/coastal/save/Ali.Abdolali/hpc-stack/parmetis-4.0.3'
+    scotchpath='/scratch2/COASTAL/coastal/save/Ali.Abdolali/hpc-stack/scotch/install'
     modcmake='cmake/3.20.1'
   elif [ $isorion ]
   then
@@ -74,6 +75,7 @@ EOF
     modcomp='hpc-intel/2018.4'
     modmpi='hpc-impi/2018.4'
     metispath='/work/noaa/marine/ali.abdolali/Source/hpc-stack/parmetis-4.0.3'
+    scotchpath='/work/noaa/marine/ali.abdolali/Source/hpc-stack/scotch/install'
     modcmake='cmake/3.22.1'
   else
     batchq=
@@ -125,7 +127,7 @@ EOF
   echo "  cd $(dirname $main_dir)/regtests"                 >> matrix.head
   echo ' '                                                  >> matrix.head
 
-# Netcdf and Parmetis modules & variables
+# Netcdf, Parmetis and SCOTCH modules & variables
   echo "  module purge"                                     >> matrix.head
   echo "  module load $modcmake"                            >> matrix.head
   echo "  module use $hpcstackpath"                         >> matrix.head
@@ -143,6 +145,7 @@ EOF
   echo "  module load $modesmf"                             >> matrix.head
 
   echo "  export METIS_PATH=${metispath}"                   >> matrix.head
+  echo "  export SCOTCH_PATH=${scotchpath}"                   >> matrix.head
   echo "  export path_build_root=$(dirname $main_dir)/regtests/buildmatrix" >> matrix.head
   echo '  [[ -d ${path_build_root} ]] && rm -rf ${path_build_root}'         >> matrix.head
   echo ' '