trisolve completed and well-tested; ready to merge into master

CMakeLists.txt

@@ -180,10 +180,10 @@ if (enable_parmetislib)   ## want to use parmetis
   # fix up PARMETIS library names
   string (REPLACE ";" " " PARMETIS_LIB_STR "${PARMETIS_LIB}")
   set(PARMETIS_LIB_EXPORT ${PARMETIS_LIB_STR})
-  set(VTUNE_LIB_EXPORT "dfdfdfddf")
+
 else()
   message("-- Will not link with ParMETIS.")
-endif() 
+endif()
 
 # if(NOT enable_parmetislib)
 #  find_package(PARMETIS)    ## does not have this Module yet.

EXAMPLE/Makefile

@@ -133,7 +133,8 @@ pzdrive4_ABglobal: $(ZEXMG4) $(DSUPERLULIB)
 #	$(CC) $(CFLAGS) $(CDEFS) $(BLASDEF) $(INCLUDEDIR) -c pdgstrf.c $(VERBOSE)
 .c.o:
 	$(CC) $(CFLAGS) $(CDEFS) $(BLASDEF) $(INCLUDEDIR) -c $< $(VERBOSE)
-
+.cpp.o:
+	$(CPP) $(CPPFLAGS) $(CDEFS) $(BLASDEF) $(INCLUDEDIR) -c $< $(VERBOSE)
 .f.o:
 	$(FORTRAN) $(FFLAGS) -c $< $(VERBOSE)
 

EXAMPLE/dcreate_matrix_perturbed.c

@@ -228,3 +228,192 @@ int dcreate_matrix_perturbed(SuperMatrix *A, int nrhs, double **rhs,
 #endif
     return 0;
 }
+
+
+
+int dcreate_matrix_perturbed_postfix(SuperMatrix *A, int nrhs, double **rhs,
+                   int *ldb, double **x, int *ldx,
+                   FILE *fp, char *postfix, gridinfo_t *grid)
+{
+    SuperMatrix GA;              /* global A */
+    double   *b_global, *xtrue_global;  /* replicated on all processes */
+    int_t    *rowind, *colptr;	 /* global */
+    double   *nzval;             /* global */
+    double   *nzval_loc;         /* local */
+    int_t    *colind, *rowptr;	 /* local */
+    int_t    m, n, nnz;
+    int_t    m_loc, fst_row, nnz_loc;
+    int_t    m_loc_fst; /* Record m_loc of the first p-1 processors,
+			   when mod(m, p) is not zero. */ 
+    int_t    row, col, i, j, relpos;
+    int      iam;
+    char     trans[1];
+    int_t      *marker;
+
+    iam = grid->iam;
+
+#if ( DEBUGlevel>=1 )
+    CHECK_MALLOC(iam, "Enter dcreate_matrix()");
+#endif
+
+    if ( !iam ) {
+    double t = SuperLU_timer_();       
+	if(!strcmp(postfix,"rua")){
+		/* Read the matrix stored on disk in Harwell-Boeing format. */
+		dreadhb_dist(iam, fp, &m, &n, &nnz, &nzval, &rowind, &colptr);
+	}else if(!strcmp(postfix,"mtx")){
+		/* Read the matrix stored on disk in Matrix Market format. */
+		dreadMM_dist(fp, &m, &n, &nnz, &nzval, &rowind, &colptr);
+	}else if(!strcmp(postfix,"rb")){
+		/* Read the matrix stored on disk in Rutherford-Boeing format. */
+		dreadrb_dist(iam, fp, &m, &n, &nnz, &nzval, &rowind, &colptr);		
+	}else if(!strcmp(postfix,"dat")){
+		/* Read the matrix stored on disk in triplet format. */
+		dreadtriple_dist(fp, &m, &n, &nnz, &nzval, &rowind, &colptr);
+	}else if(!strcmp(postfix,"bin")){
+		/* Read the matrix stored on disk in binary format. */
+		dread_binary(fp, &m, &n, &nnz, &nzval, &rowind, &colptr);		
+	}else {
+		ABORT("File format not known");
+	}
+
+	printf("Time to read and distribute matrix %.2f\n", 
+	        SuperLU_timer_() - t);  fflush(stdout);
+
+	/* Broadcast matrix A to the other PEs. */
+	MPI_Bcast( &m,     1,   mpi_int_t,  0, grid->comm );
+	MPI_Bcast( &n,     1,   mpi_int_t,  0, grid->comm );
+	MPI_Bcast( &nnz,   1,   mpi_int_t,  0, grid->comm );
+	MPI_Bcast( nzval,  nnz, MPI_DOUBLE, 0, grid->comm );
+	MPI_Bcast( rowind, nnz, mpi_int_t,  0, grid->comm );
+	MPI_Bcast( colptr, n+1, mpi_int_t,  0, grid->comm );
+    } else {
+	/* Receive matrix A from PE 0. */
+	MPI_Bcast( &m,   1,   mpi_int_t,  0, grid->comm );
+	MPI_Bcast( &n,   1,   mpi_int_t,  0, grid->comm );
+	MPI_Bcast( &nnz, 1,   mpi_int_t,  0, grid->comm );
+
+	/* Allocate storage for compressed column representation. */
+	dallocateA_dist(n, nnz, &nzval, &rowind, &colptr);
+
+	MPI_Bcast( nzval,   nnz, MPI_DOUBLE, 0, grid->comm );
+	MPI_Bcast( rowind,  nnz, mpi_int_t,  0, grid->comm );
+	MPI_Bcast( colptr,  n+1, mpi_int_t,  0, grid->comm );
+    }
+
+    /* Perturbed the 1st and last diagonal of the matrix to lower
+       values. Intention is to change perm_r[].   */
+    nzval[0] *= 0.01;
+    nzval[nnz-1] *= 0.0001; 
+
+    /* Compute the number of rows to be distributed to local process */
+    m_loc = m / (grid->nprow * grid->npcol); 
+    m_loc_fst = m_loc;
+    /* When m / procs is not an integer */
+    if ((m_loc * grid->nprow * grid->npcol) != m) {
+        /*m_loc = m_loc+1;
+          m_loc_fst = m_loc;*/
+      if (iam == (grid->nprow * grid->npcol - 1)) /* last proc. gets all*/
+	  m_loc = m - m_loc * (grid->nprow * grid->npcol - 1);
+    }
+
+    /* Create compressed column matrix for GA. */
+    dCreate_CompCol_Matrix_dist(&GA, m, n, nnz, nzval, rowind, colptr,
+				SLU_NC, SLU_D, SLU_GE);
+
+    /* Generate the exact solution and compute the right-hand side. */
+    if ( !(b_global = doubleMalloc_dist(m*nrhs)) )
+        ABORT("Malloc fails for b[]");
+    if ( !(xtrue_global = doubleMalloc_dist(n*nrhs)) )
+        ABORT("Malloc fails for xtrue[]");
+    *trans = 'N';
+
+    dGenXtrue_dist(n, nrhs, xtrue_global, n);
+    dFillRHS_dist(trans, nrhs, xtrue_global, n, &GA, b_global, m);
+
+    /*************************************************
+     * Change GA to a local A with NR_loc format     *
+     *************************************************/
+
+    rowptr = (int_t *) intMalloc_dist(m_loc+1);
+    marker = (int_t *) intCalloc_dist(n);
+
+    /* Get counts of each row of GA */
+    for (i = 0; i < n; ++i)
+      for (j = colptr[i]; j < colptr[i+1]; ++j) ++marker[rowind[j]];
+    /* Set up row pointers */
+    rowptr[0] = 0;
+    fst_row = iam * m_loc_fst;
+    nnz_loc = 0;
+    for (j = 0; j < m_loc; ++j) {
+      row = fst_row + j;
+      rowptr[j+1] = rowptr[j] + marker[row];
+      marker[j] = rowptr[j];
+    }
+    nnz_loc = rowptr[m_loc];
+
+    nzval_loc = (double *) doubleMalloc_dist(nnz_loc);
+    colind = (int_t *) intMalloc_dist(nnz_loc);
+
+    /* Transfer the matrix into the compressed row storage */
+    for (i = 0; i < n; ++i) {
+      for (j = colptr[i]; j < colptr[i+1]; ++j) {
+	row = rowind[j];
+	if ( (row>=fst_row) && (row<fst_row+m_loc) ) {
+	  row = row - fst_row;
+	  relpos = marker[row];
+	  colind[relpos] = i;
+	  nzval_loc[relpos] = nzval[j];
+	  ++marker[row];
+	}
+      }
+    }
+
+#if ( DEBUGlevel>=2 )
+    if ( !iam ) dPrint_CompCol_Matrix_dist(&GA);
+#endif   
+
+    /* Destroy GA */
+    Destroy_CompCol_Matrix_dist(&GA);
+
+    /******************************************************/
+    /* Change GA to a local A with NR_loc format */
+    /******************************************************/
+
+    /* Set up the local A in NR_loc format */
+    dCreate_CompRowLoc_Matrix_dist(A, m, n, nnz_loc, m_loc, fst_row,
+				   nzval_loc, colind, rowptr,
+				   SLU_NR_loc, SLU_D, SLU_GE);
+
+    /* Get the local B */
+    if ( !((*rhs) = doubleMalloc_dist(m_loc*nrhs)) )
+        ABORT("Malloc fails for rhs[]");
+    for (j =0; j < nrhs; ++j) {
+	for (i = 0; i < m_loc; ++i) {
+	    row = fst_row + i;
+	    (*rhs)[j*m_loc+i] = b_global[j*n+row];
+	}
+    }
+    *ldb = m_loc;
+
+    /* Set the true X */    
+    *ldx = m_loc;
+    if ( !((*x) = doubleMalloc_dist(*ldx * nrhs)) )
+        ABORT("Malloc fails for x_loc[]");
+
+    /* Get the local part of xtrue_global */
+    for (j = 0; j < nrhs; ++j) {
+      for (i = 0; i < m_loc; ++i)
+	(*x)[i + j*(*ldx)] = xtrue_global[i + fst_row + j*n];
+    }
+
+    SUPERLU_FREE(b_global);
+    SUPERLU_FREE(xtrue_global);
+    SUPERLU_FREE(marker);
+
+#if ( DEBUGlevel>=1 )
+    printf("sizeof(NRforamt_loc) %lu\n", sizeof(NRformat_loc));
+    CHECK_MALLOC(iam, "Exit dcreate_matrix()");
+#endif
+    return 0;
+}

EXAMPLE/pddrive.c

@@ -65,15 +65,26 @@ int main(int argc, char *argv[])
     FILE *fp, *fopen();
     int cpp_defs();
 	int ii;
+	int omp_mpi_level;
 
     nprow = 1;  /* Default process rows.      */
     npcol = 1;  /* Default process columns.   */
-    nrhs = 1;   /* Number of right-hand side. */
-
+    nrhs =1;   /* Number of right-hand side. */
+ 
     /* ------------------------------------------------------------
        INITIALIZE MPI ENVIRONMENT. 
        ------------------------------------------------------------*/
     MPI_Init( &argc, &argv );
+    //MPI_Init_thread( &argc, &argv, MPI_THREAD_MULTIPLE, &omp_mpi_level); 
+
+
+#if ( VAMPIR>=1 )
+    VT_traceoff(); 
+#endif
+
+#if ( VTUNE>=1 )
+	__itt_pause();
+#endif
 
     /* Parse command line argv[]. */
     for (cpp = argv+1; *cpp; ++cpp) {
@@ -105,6 +116,28 @@ int main(int argc, char *argv[])
        ------------------------------------------------------------*/
     superlu_gridinit(MPI_COMM_WORLD, nprow, npcol, &grid);
 
+	if(grid.iam==0){
+	MPI_Query_thread(&omp_mpi_level);
+    switch (omp_mpi_level) {
+      case MPI_THREAD_SINGLE:
+		printf("MPI_Query_thread with MPI_THREAD_SINGLE\n");
+		fflush(stdout);
+	break;
+      case MPI_THREAD_FUNNELED:
+		printf("MPI_Query_thread with MPI_THREAD_FUNNELED\n");
+		fflush(stdout);
+	break;
+      case MPI_THREAD_SERIALIZED:
+		printf("MPI_Query_thread with MPI_THREAD_SERIALIZED\n");
+		fflush(stdout);
+	break;
+      case MPI_THREAD_MULTIPLE:
+		printf("MPI_Query_thread with MPI_THREAD_MULTIPLE\n");
+		fflush(stdout);
+	break;
+    }
+	}
+
     /* Bail out if I do not belong in the grid. */
     iam = grid.iam;
     if ( iam >= nprow * npcol )	goto out;
@@ -164,14 +197,13 @@ int main(int argc, char *argv[])
     options.ReplaceTinyPivot = NO;
 #endif
 
-
-	//options.ParSymbFact       = YES;
-	//options.ColPerm           = PARMETIS;
+//	options.ParSymbFact       = YES;
+//	options.ColPerm           = PARMETIS;
+//	options.RowPerm = NOROWPERM;
 	options.IterRefine       = 0;
-	options.DiagInv       = YES;
-	options.RowPerm = NOROWPERM;
-	//options.ReplaceTinyPivot = NO;	
-	options.SymPattern = YES;
+//	options.DiagInv       = YES;
+	options.ReplaceTinyPivot = NO;	
+	options.SymPattern = YES;						  
 
     if (!iam) {
 	print_sp_ienv_dist(&options);

EXAMPLE/pddrive1.c

@@ -55,10 +55,10 @@ int main(int argc, char *argv[])
     gridinfo_t grid;
     double   *berr;
     double   *b, *xtrue, *b1;
-    int    i, j, m, n;
+    int    i, j, m, n, ii;
     int    nprow, npcol;
     int    iam, info, ldb, ldx, nrhs;
-    char     **cpp, c;
+    char     **cpp, c, *postfix;
     FILE *fp, *fopen();
     int cpp_defs();
 
@@ -118,11 +118,19 @@ int main(int argc, char *argv[])
     CHECK_MALLOC(iam, "Enter main()");
 #endif
 
+	for(ii = 0;ii<strlen(*cpp);ii++){
+		if((*cpp)[ii]=='.'){
+			postfix = &((*cpp)[ii+1]);
+		}
+	}	
+	// printf("%s\n", postfix);
+
     /* ------------------------------------------------------------
        GET THE MATRIX FROM FILE AND SETUP THE RIGHT HAND SIDE. 
        ------------------------------------------------------------*/
-    dcreate_matrix(&A, nrhs, &b, &ldb, &xtrue, &ldx, fp, &grid);
-    if ( !(b1 = doubleMalloc_dist(ldb * nrhs)) )
+    dcreate_matrix_postfix(&A, nrhs, &b, &ldb, &xtrue, &ldx, fp, postfix, &grid);
+
+	if ( !(b1 = doubleMalloc_dist(ldb * nrhs)) )
         ABORT("Malloc fails for b1[]");
     for (j = 0; j < nrhs; ++j)
         for (i = 0; i < ldb; ++i) b1[i+j*ldb] = b[i+j*ldb];

EXAMPLE/pddrive2.c

@@ -59,17 +59,20 @@ int main(int argc, char *argv[])
     double   *berr;
     double   *b, *b1, *xtrue, *xtrue1;
     int_t    *colind, *colind1, *rowptr, *rowptr1;
-    int_t    i, j, m, n, nnz_loc, m_loc;
+    int_t    i, j, ii, m, n, nnz_loc, m_loc;
     int      nprow, npcol;
     int      iam, info, ldb, ldx, nrhs;
-    char     **cpp, c;
+    char     **cpp, c, *postfix;
     FILE *fp, *fopen();
     int cpp_defs();
 
     /* prototypes */
     extern int dcreate_matrix_perturbed
         (SuperMatrix *, int, double **, int *, double **, int *,
-         FILE *, gridinfo_t *);
+         FILE *, gridinfo_t *);    
+	extern int dcreate_matrix_perturbed_postfix
+        (SuperMatrix *, int, double **, int *, double **, int *,
+         FILE *, char *, gridinfo_t *);
 
     nprow = 1;  /* Default process rows.      */
     npcol = 1;  /* Default process columns.   */
@@ -123,10 +126,17 @@ int main(int argc, char *argv[])
     CHECK_MALLOC(iam, "Enter main()");
 #endif
 
+	for(ii = 0;ii<strlen(*cpp);ii++){
+		if((*cpp)[ii]=='.'){
+			postfix = &((*cpp)[ii+1]);
+		}
+	}	
+	// printf("%s\n", postfix);
+
     /* ------------------------------------------------------------
-       GET THE MATRIX FROM FILE AND SETUP THE RIGHT-HAND SIDE. 
+       GET THE MATRIX FROM FILE AND SETUP THE RIGHT HAND SIDE. 
        ------------------------------------------------------------*/
-    dcreate_matrix(&A, nrhs, &b, &ldb, &xtrue, &ldx, fp, &grid);
+    dcreate_matrix_postfix(&A, nrhs, &b, &ldb, &xtrue, &ldx, fp, postfix, &grid);
 
     if ( !(berr = doubleMalloc_dist(nrhs)) )
 	ABORT("Malloc fails for berr[].");
@@ -198,7 +208,7 @@ int main(int argc, char *argv[])
     /* Get the matrix from file, perturbed some diagonal entries to force
        a different perm_r[]. Set up the right-hand side.   */
     if ( !(fp = fopen(*cpp, "r")) ) ABORT("File does not exist");
-    dcreate_matrix_perturbed(&A, nrhs, &b1, &ldb, &xtrue1, &ldx, fp, &grid);
+    dcreate_matrix_perturbed_postfix(&A, nrhs, &b1, &ldb, &xtrue1, &ldx, fp, postfix, &grid);
 
     PStatInit(&stat); /* Initialize the statistics variables. */