updated interface to CombBLAS, working with both real and complex.

add CMakeLists.txt in FORTRAN/ directory.

CMakeLists.txt

@@ -8,10 +8,10 @@
 cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR)
 
 # Project version numbers
-project(SuperLU_DIST C CXX)
+project(SuperLU_DIST C CXX Fortran)
 set(VERSION_MAJOR "6")
 set(VERSION_MINOR "3")
-set(VERSION_BugFix "0")
+set(VERSION_BugFix "1")
 set(PROJECT_VERSION ${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_BugFix})
 
 list(APPEND CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake")
@@ -22,6 +22,7 @@ option(enable_double    "Enable double precision library" ON)
 option(enable_complex16 "Enable complex16 precision library" ON)
 option(enable_tests  "Build tests" ON)
 option(enable_examples  "Build examples" ON)
+option(enable_fortran  "Build fortran90 interface" OFF)
 option(TPL_ENABLE_BLASLIB  "Build the CBLAS library" ${enable_blaslib_DEFAULT})
 option(TPL_BLAS_LIBRARIES "List of absolute paths to blas libraries [].")
 option(TPL_ENABLE_LAPACKLIB "Enable LAPACK library" ON)
@@ -88,7 +89,8 @@ else()
   set(PROJECT_NAME_LIB_EXPORT libsuperlu_dist.a)
 endif()
 
-set(CMAKE_CXX_STANDARD 11)
+#set(CMAKE_CXX_STANDARD 11)
+set(CMAKE_CXX_STANDARD 14)
 
 if (XSDK_ENABLE_Fortran)
   enable_language (Fortran)
@@ -127,7 +129,7 @@ set(INSTALL_BIN_DIR "${default_install_bin_dir}" CACHE STRING "The folder where
 # Set up required compiler defines and options.
 ## get_directory_property( DirDefs COMPILE_DEFINITIONS )
 # set(CMAKE_C_FLAGS "-DDEBUGlevel=0 -DPRNTlevel=0 ${CMAKE_C_FLAGS}")
-set(CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS}")
+# set(CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS}")
 if(XSDK_INDEX_SIZE EQUAL 64)
     message("-- Using 64 bit integer for index size.")
 endif()	
@@ -347,6 +349,11 @@ if(enable_examples)
   add_subdirectory(EXAMPLE)
 endif()
 
+if(enable_fortran)
+  add_subdirectory(FORTRAN)
+  SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lstdc++")
+endif()
+
 # configure_file(${CMAKE_SOURCE_DIR}/make.inc.in ${CMAKE_BINARY_DIR}/make.inc)
 configure_file(${SuperLU_DIST_SOURCE_DIR}/make.inc.in ${SuperLU_DIST_SOURCE_DIR}/make.inc)
 configure_file(${SuperLU_DIST_SOURCE_DIR}/SRC/superlu_dist_config.h.in ${SuperLU_DIST_BINARY_DIR}/SRC/superlu_dist_config.h)
@@ -362,4 +369,4 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/superlu_dist.pc.in ${CMAKE_CURRENT_BI
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/superlu_dist.pc
 	DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig)
 
-
+message("CMAKE_CXX_LINK_FLAGS '${CMAKE_CXX_LINK_FLAGS}'")

EXAMPLE/pddrive.c

@@ -199,7 +199,7 @@ int main(int argc, char *argv[])
      */
     set_default_options_dist(&options);
 #if 0
-    options.RowPerm = NOROWPERM;
+    options.RowPerm = LargeDiag_HWPM;
     options.IterRefine = NOREFINE;
     options.ColPerm = NATURAL;
     options.Equil = NO; 

EXAMPLE/pzdrive.c

@@ -197,7 +197,7 @@ int main(int argc, char *argv[])
      */
     set_default_options_dist(&options);
 #if 0
-    options.RowPerm = NOROWPERM;
+    options.RowPerm = LargeDiag_HWPM;
     options.IterRefine = NOREFINE;
     options.ColPerm = NATURAL;
     options.Equil = NO; 

FORTRAN/Makefile

@@ -24,8 +24,8 @@ F_5x5 	= $(F_MOD) f_5x5.o sp_ienv.o
 all: f_pddrive f_pzdrive f_5x5
 
 f_pddrive: $(F_DEXM) $(C_DWRAP) $(DSUPERLULIB)
-	$(LOADER) $(LOADOPTS) $(F_DEXM) $(C_DWRAP) $(LIBS) -o $@
-#	$(FORTRAN) $(LOADOPTS) $(F_DEXM) $(C_DWRAP) $(LIBS) -o $@
+	$(FORTRAN) $(LOADOPTS) $(F_DEXM) $(C_DWRAP) $(LIBS) -o $@
+#	$(LOADER) $(LOADOPTS) $(F_DEXM) $(C_DWRAP) $(LIBS) -o $@
 
 f_5x5: $(F_5x5) $(C_DWRAP) $(DSUPERLULIB)
 	$(FORTRAN) $(LOADOPTS) $(F_5x5) $(C_DWRAP) $(LIBS) -o $@

FORTRAN/README

@@ -15,14 +15,20 @@ There are two examples in the directory.
 
 1. f_5x5.f90:
    A small 5x5 example appeared in the SuperLU Users Guide, Section 2.2.
-   To run the code on a Cray XT or XE, type 'aprun -n 2 f_5x5'
+   To run the code, type:
+      mpiexec -n 2 f_5x5
    (The example is set up to use 2 processors.)
 
-2. f_pddrive.f90 / f_pzdrive.f90:
-   An example Fortran driver routine that reads a matrix from a file
+2. f_pddrive.f90
+   A real example Fortran driver routine that reads a matrix from a file
    'g20.rua' in Harwell-Boeing format.
-   To run the code on a Cray XT or XE, type 'aprun -n 4 f_pddrive'
+   To run the code, type:
+      mpiexec -n 4 f_pddrive
    (The example is set up to use 4 processors.)
 
-   The complex version:
-   % aprun -n 4 f_pzdrive
+3. f_pzdrive.f90
+   A complex example Fortran driver routine that reads a matrix from a file
+   'cg20.cua' in Harwell-Boeing format.
+   To run the code, type:
+      mpiexec -n 4 f_pzdrive
+   (The example is set up to use 4 processors.)

SRC/dHWPM_CombBLAS.hpp

@@ -1,7 +1,28 @@
+/*! \file
+Copyright (c) 2003, The Regents of the University of California, through
+Lawrence Berkeley National Laboratory (subject to receipt of any required
+approvals from U.S. Dept. of Energy)
+
+All rights reserved.
+
+The source code is distributed under BSD license, see the file License.txt
+at the top-level directory.
+*/
+
+/*! @file
+ * \brief Get HWPM, heavy-weight perfect matching.
+ *
+ * <pre>
+ * -- Distributed SuperLU routine (version 6.0) --
+ * Lawrence Berkeley National Lab, Univ. of California Berkeley.
+ * April 2, 2020
+ * </pre>
+ */
+
 #ifndef dHWPM_CombBLAS_h
 #define dHWPM_CombBLAS_h
 
-#include "CombBLAS.h"
+#include "CombBLAS/CombBLAS.h"
 #include "ApproxWeightPerfectMatching.h"
 #include "superlu_ddefs.h"
 
@@ -11,9 +32,9 @@
  * <pre>
  * Purpose
  * =======
- *   Re-distribute A from distributed CSR storage to 2D block storage
- *   conforming CombBLAS API.
- *
+ *   Get perm_r from HWPM, heavy-weight perfect matching, as a
+ *   numerical pivoting permutation.
+ * 
  * Arguments
  * =========
  *
@@ -32,7 +53,7 @@
  *
  * Return value
  * ============
- * ScalePermstruct       = ScalePermstruct->perm_r stores the permutation
+ * ScalePermstruct->perm_r stores the permutation obtained from HWPM algorithm.
  *
  * </pre>
  */
@@ -72,6 +93,8 @@ dGetHWPM(SuperMatrix *A, gridinfo_t *grid, dScalePermstruct_t *ScalePermstruct)
      FIRST PASS OF A:
      COUNT THE NUMBER OF NONZEROS TO BE SENT TO EACH PROCESS,
      THEN ALLOCATE SPACE.
+     Re-distribute A from distributed CSR storage to 2D block storage
+     conforming CombBLAS API.
      ------------------------------------------------------------*/
     nzval_a = (double *) Astore->nzval;
     nnz_loc = 0;
@@ -118,7 +141,7 @@ dGetHWPM(SuperMatrix *A, gridinfo_t *grid, dScalePermstruct_t *ScalePermstruct)
     delete[] recvcnt;
 
 #if ( DEBUGlevel>=1 )
-    CHECK_MALLOC(iam, "Exit pdCSR_loc_to_2DBlock()");
+    CHECK_MALLOC(iam, "Exit dHWPM_CombBLAS()");
 #endif
 }
 

SRC/d_c2cpp_GetHWPM.cpp

@@ -10,12 +10,13 @@ at the top-level directory.
 */
 
 /*! @file 
- * \brief Get approximate weight perfect matching (AWPM).
+ * \brief Get heavy-weight perfect matching (HWPM).
  *
  * <pre>
  * -- Distributed SuperLU routine (version 5.4) --
  * Lawrence Berkeley National Lab, Univ. of California Berkeley.
  * April 1, 2018
+ * Modified: April 2, 2020
  * </pre>
  */
 #include <math.h>
@@ -28,7 +29,7 @@ at the top-level directory.
  * Purpose
  * =======
  *
- * Get approximate weight perfect matching (AWPM).
+ * Get heavy-weight perfect matching (HWPM).
  * 
  * Reference:
  * 

SRC/superlu_FortranCInterface.h

@@ -8,9 +8,9 @@
 #define FC_GLOBAL_(name,NAME) name##_
 
 /* Mangling for Fortran module symbols without underscores. */
-#define FC_MODULE(mod_name,name, mod_NAME,NAME) __##mod_name##_MOD_##name
+#define FC_MODULE(mod_name,name, mod_NAME,NAME) mod_name##_mp_##name##_
 
 /* Mangling for Fortran module symbols with underscores. */
-#define FC_MODULE_(mod_name,name, mod_NAME,NAME) __##mod_name##_MOD_##name
+#define FC_MODULE_(mod_name,name, mod_NAME,NAME) mod_name##_mp_##name##_
 
 #endif

SRC/superlu_defs.h

@@ -72,8 +72,8 @@ at the top-level directory.
  */
 #define SUPERLU_DIST_MAJOR_VERSION     6
 #define SUPERLU_DIST_MINOR_VERSION     3
-#define SUPERLU_DIST_PATCH_VERSION     0
-#define SUPERLU_DIST_RELEASE_DATE      "February 23, 2020"
+#define SUPERLU_DIST_PATCH_VERSION     1
+#define SUPERLU_DIST_RELEASE_DATE      "April 2, 2020"
 
 #include "superlu_dist_config.h"
 /* Define my integer size int_t */

SRC/superlu_enum_consts.h

@@ -27,7 +27,7 @@ at the top-level directory.
  ***********************************************************************/
 typedef enum {NO, YES}                                          yes_no_t;
 typedef enum {DOFACT, SamePattern, SamePattern_SameRowPerm, FACTORED} fact_t;
-typedef enum {NOROWPERM, LargeDiag_MC64, LargeDiag_AWPM, MY_PERMR} rowperm_t;
+typedef enum {NOROWPERM, LargeDiag_MC64, LargeDiag_HWPM, MY_PERMR} rowperm_t;
 typedef enum {NATURAL, MMD_ATA, MMD_AT_PLUS_A, COLAMD,
 	      METIS_AT_PLUS_A, PARMETIS, ZOLTAN, MY_PERMC}      colperm_t;
 typedef enum {NOTRANS, TRANS, CONJ}                             trans_t;

SRC/superlu_grid.c

@@ -45,8 +45,10 @@ void superlu_gridinit(MPI_Comm Bcomm, /* The base communicator upon which
 	ABORT("C main program must explicitly call MPI_Init()");
 
     MPI_Comm_size( Bcomm, &info );
-    if ( info < Np )
-	ABORT("Number of processes is smaller than NPROW * NPCOL");
+    if ( info < Np ) {
+	printf("Number of processes %d is smaller than NPROW * NPCOL %d", info, Np);
+	exit(-1);
+    }
 
     superlu_gridmap(Bcomm, nprow, npcol, usermap, nprow, grid);
 

SRC/zHWPM_CombBLAS.hpp

@@ -1,7 +1,28 @@
+/*! \file
+Copyright (c) 2003, The Regents of the University of California, through
+Lawrence Berkeley National Laboratory (subject to receipt of any required
+approvals from U.S. Dept. of Energy)
+
+All rights reserved.
+
+The source code is distributed under BSD license, see the file License.txt
+at the top-level directory.
+*/
+
+/*! @file
+ * \brief Get HWPM, heavy-weight perfect matching.
+ *
+ * <pre>
+ * -- Distributed SuperLU routine (version 6.0) --
+ * Lawrence Berkeley National Lab, Univ. of California Berkeley.
+ * April 2, 2020
+ * </pre>
+ */
+
 #ifndef zHWPM_CombBLAS_h
 #define zHWPM_CombBLAS_h
 
-#include "CombBLAS.h"
+#include "CombBLAS/CombBLAS.h"
 #include "ApproxWeightPerfectMatching.h"
 #include "superlu_zdefs.h"
 
@@ -11,8 +32,8 @@
  * <pre>
  * Purpose
  * =======
- *   Re-distribute A from distributed CSR storage to 2D block storage
- *   conforming CombBLAS API.
+ *   Get perm_r from HWPM, heavy-weight perfect matching, as a
+ *   numerical pivoting permutation.
  *
  * Arguments
  * =========
@@ -32,7 +53,7 @@
  *
  * Return value
  * ============
- * ScalePermstruct       = ScalePermstruct->perm_r stores the permutation
+ * ScalePermstruct->perm_r stores the permutation obtained from HWPM algorithm.
  *
  * </pre>
  */
@@ -43,7 +64,7 @@ zGetHWPM(SuperMatrix *A, gridinfo_t *grid, zScalePermstruct_t *ScalePermstruct)
     int_t  i, irow, fst_row, j, jcol, m, n, m_loc;
     int_t lirow, ljcol;
     int_t  nnz_loc;    /* number of local nonzeros */
-    doublecompex *nzval_a;
+    doublecomplex *nzval_a;
     int    iam, p, procs;
     int_t *perm=nullptr; // placeholder for load balancing permutation for CombBLAS
     procs = grid->nprow * grid->npcol;
@@ -72,8 +93,14 @@ zGetHWPM(SuperMatrix *A, gridinfo_t *grid, zScalePermstruct_t *ScalePermstruct)
      FIRST PASS OF A:
      COUNT THE NUMBER OF NONZEROS TO BE SENT TO EACH PROCESS,
      THEN ALLOCATE SPACE.
+     Re-distribute A from distributed CSR storage to 2D block storage
+     conforming CombBLAS API.
      ------------------------------------------------------------*/
     nzval_a = (doublecomplex *) Astore->nzval;
+    nnz_loc = Astore->nnz_loc;
+    double *nzval_abs = doubleMalloc_dist(nnz_loc);
+    for (i = 0; i < nnz_loc; ++i) nzval_abs[i] = slud_z_abs1(&nzval_a[i]);
+
     nnz_loc = 0;
     for (i = 0; i < m_loc; ++i) {
         for (j = Astore->rowptr[i]; j < Astore->rowptr[i+1]; ++j) {
@@ -88,9 +115,8 @@ zGetHWPM(SuperMatrix *A, gridinfo_t *grid, zScalePermstruct_t *ScalePermstruct)
                 jcol = Astore->colind[j];
             }
             p = Adcsc.Owner(m, n , irow, jcol, lirow, ljcol);
-            ++ nnz_loc;
-            data[p].push_back(std::make_tuple(lirow,ljcol,nzval_a[j]));
-
+            ++nnz_loc;
+            data[p].push_back(std::make_tuple(lirow,ljcol,nzval_abs[j]));
         }
     }
 
@@ -116,9 +142,10 @@ zGetHWPM(SuperMatrix *A, gridinfo_t *grid, zScalePermstruct_t *ScalePermstruct)
 
     delete[] rdispls;
     delete[] recvcnt;
+    SUPERLU_FREE(nzval_abs);
 
 #if ( DEBUGlevel>=1 )
-    CHECK_MALLOC(iam, "Exit pdCSR_loc_to_2DBlock()");
+    CHECK_MALLOC(iam, "Exit zHWPM_CombBLAS()");
 #endif
 }
 

SRC/z_c2cpp_GetHWPM.cpp

@@ -10,25 +10,26 @@ at the top-level directory.
 */
 
 /*! @file 
- * \brief Get approximate weight perfect matching (AWPM).
+ * \brief Get heavy-weight perfect matching (HWPM).
  *
  * <pre>
  * -- Distributed SuperLU routine (version 5.4) --
  * Lawrence Berkeley National Lab, Univ. of California Berkeley.
  * April 1, 2018
+ * Modified: April 2, 2020
  * </pre>
  */
 #include <math.h>
-#include "dHWPM_CombBLAS.hpp"
-#include "superlu_ddefs.h"
+#include "zHWPM_CombBLAS.hpp"
+#include "superlu_zdefs.h"
 
 /*! \brief
  *
  * <pre>
  * Purpose
  * =======
  *
- * Get approximate weight perfect matching (AWPM).
+ * Get heavy-weight perfect matching (HWPM).
  * 
  * Reference:
  * 
@@ -55,9 +56,8 @@ at the top-level directory.
  * </pre>
  */
 int
-d_c2cpp_GetHWPM(SuperMatrix *A, gridinfo_t *grid,
-		dScalePermstruct_t *ScalePermstruct)
+z_c2cpp_GetHWPM(SuperMatrix *A, gridinfo_t *grid, zScalePermstruct_t *ScalePermstruct)
 {
-    dGetHWPM(A, grid, ScalePermstruct);
+    zGetHWPM(A, grid, ScalePermstruct);
     return 0;
 }

make.inc.in

@@ -46,4 +46,4 @@ NOOPTS       = -O0
 FORTRAN	     = @CMAKE_Fortran_COMPILER@
 
 LOADER       = @CMAKE_CXX_COMPILER@
-LOADOPTS     = @CMAKE_EXE_LINKER_FLAGS@
+LOADOPTS     = @CMAKE_EXE_LINKER_FLAGS@ @CMAKE_CXX_LINK_FLAGS@