diff --git a/lulesh/lulesh-comm-dash-onesided.cc b/lulesh/lulesh-comm-dash-onesided.cc index 74c6aea..b4243fc 100644 --- a/lulesh/lulesh-comm-dash-onesided.cc +++ b/lulesh/lulesh-comm-dash-onesided.cc @@ -37,16 +37,45 @@ char tagname[][8] = "X1Y1Z1 ", }; +#ifdef WITH_DEBUG +static void +dump_buffer(Real_t *buf, size_t nelem) +{ + for (size_t i = 0; i < nelem; ++i) { + std::cout << std::fixed << std::setprecision(5) << std::setw(3) << buf[i] << " "; + } + std::cout << std::endl; +} + +// debug output for syncs performed below +void DBGSYNC_(const char *fname, Real_t *buf, int fields, int elem, int tag) +{ + cout << "DBGSYNC [" << dash::myid() << "] " << fields << "x" << elem << " (" + << fields*elem << ") " << tagname[tag] << " in " << fname << endl; + dump_buffer(buf, fields*elem); +} + +#define DBGSYNC(__buf, __fields, __elem, __tag) DBGSYNC_(__func__, __buf, __fields, __elem, __tag) // debug output for syncs performed below -void DBGSYNC(int fields, int elem, int tag) +void DBGPUT_(const char *fname, const dash::GlobIter > src, + Real_t *buf, int fields, int elem, int tag) { - /* - cout << "DBGSYNC [" << dash::myid() << "] " << elem - << " " << tagname[tag] << endl; - */ + cout << "DBGPUT [" << dash::myid() << "] " << fields << "x" << elem << " (" + << fields*elem << ") " << tagname[tag] << " into " << src.dart_gptr() + << " in " << fname << endl; + dump_buffer(buf, fields*elem); } +#define DBGPUT(__src, __buf, __fields, __elem, __tag) DBGPUT_(__func__, __src, __buf, __fields, __elem, __tag) + +#else + +#define DBGSYNC(...) +#define DBGPUT(...) + +#endif // WITH_DEBUG + void DASHCommPut(Domain& domain, DASHComm& comm, Index_t xferFields, Domain_member *fieldData, Index_t dx, Index_t dy, Index_t dz, @@ -54,17 +83,12 @@ void DASHCommPut(Domain& domain, DASHComm& comm, { Real_t *destAddr; bool rowNotMin, rowNotMax, colNotMin, colNotMax, planeNotMin, planeNotMax; - - Index_t maxPlaneComm = xferFields * domain.maxPlaneSize(); - Index_t maxEdgeComm = xferFields * domain.maxEdgeSize(); - - Index_t pmsg = 0; // plane comm msg - Index_t emsg = 0; // edge comm msg - Index_t cmsg = 0; // corner comm msg - // assume communication to 6 neighbors by default rowNotMin = rowNotMax = colNotMin = colNotMax = planeNotMin = planeNotMax = true; + Index_t futcntr = 0; + + if( domain.rowLoc() == 0 ) { rowNotMin = false; } if( domain.rowLoc() == (domain.tp()-1) ) { rowNotMax = false; } if( domain.colLoc() == 0 ) { colNotMin = false; } @@ -79,7 +103,7 @@ void DASHCommPut(Domain& domain, DASHComm& comm, int sendCount = dx * dy; if (planeNotMin) { - destAddr = &comm.commDataSend()[pmsg * maxPlaneComm]; + destAddr = &comm.commDataSend()[comm.offset(Z1)]; for (Index_t fi=0; filbegin(), 0, comBufSize*sizeof(Real_t) ); memset( m_commDataRecv->lbegin(), 0, comBufSize*sizeof(Real_t) ); + + m_commDataRecv->barrier(); } DASHComm::~DASHComm() @@ -59,6 +61,7 @@ void DASHComm::ExchangeNodalMass() dash::barrier(); DASHCommSBN(dom, comm, 1, &fieldData); + dash::barrier(); } void DASHComm::Recv_PosVel() @@ -98,6 +101,7 @@ void DASHComm::Sync_PosVel() dash::barrier(); DASHCommSyncPosVel(dom, comm); + dash::barrier(); } @@ -137,6 +141,7 @@ void DASHComm::Sync_Force() dash::barrier(); DASHCommSBN(dom, comm, 3, fieldData); + dash::barrier(); } void DASHComm::Recv_MonoQ() @@ -174,6 +179,7 @@ void DASHComm::Sync_MonoQ() dash::barrier(); DASHCommMonoQ(dom,comm); + dash::barrier(); } Int_t DASHComm::offset(Int_t desc) @@ -190,8 +196,8 @@ Int_t DASHComm::offset(Int_t desc) assert(0 <= cmsg && cmsg <= 8); auto offs = - pmsg * maxPlaneSize + - emsg * maxEdgeSize + + (pmsg * maxPlaneSize + + emsg * maxEdgeSize) * MAX_FIELDS_PER_COMM + cmsg * CACHE_COHERENCE_PAD_REAL; return offs; diff --git a/lulesh/lulesh-dash.cc b/lulesh/lulesh-dash.cc index 70b93ce..efef4fc 100644 --- a/lulesh/lulesh-dash.cc +++ b/lulesh/lulesh-dash.cc @@ -590,12 +590,14 @@ void Domain::SetupThreadSupportStructures() for (Index_t i=0; i < clSize; ++i) { Index_t clv = m_nodeElemCornerList[i] ; if ((clv < 0) || (clv > numElem()*8)) { - fprintf(stderr, - "AllocateNodeElemIndexes(): nodeElemCornerList entry out of range!\n"); + fprintf(stderr, + "AllocateNodeElemIndexes(): nodeElemCornerList entry out of range!\n"); #if USE_MPI - MPI_Abort(MPI_COMM_WORLD, -1); + MPI_Abort(MPI_COMM_WORLD, -1); +#elif USE_DASH + dart_abort(-1); #else - exit(-1); + exit(-1); #endif } } @@ -869,6 +871,8 @@ void Domain::CalcLagrangeElements(Real_t* vnew) { #if USE_MPI MPI_Abort(MPI_COMM_WORLD, VolumeError) ; +#elif USE_DASH + dart_abort(VolumeError); #else exit(VolumeError); #endif @@ -919,9 +923,11 @@ void Domain::CalcQForElems(Real_t vnew[]) if(idx >= 0) { #if USE_MPI - MPI_Abort(MPI_COMM_WORLD, QStopError) ; + MPI_Abort(MPI_COMM_WORLD, QStopError) ; +#elif USE_DASH + dart_abort(QStopError); #else - exit(QStopError); + exit(QStopError); #endif } } @@ -972,9 +978,11 @@ void Domain::ApplyMaterialPropertiesForElems(Real_t vnew[]) } if (vc <= 0.) { #if USE_MPI - MPI_Abort(MPI_COMM_WORLD, VolumeError) ; + MPI_Abort(MPI_COMM_WORLD, VolumeError) ; +#elif USE_DASH + dart_abort(VolumeError); #else - exit(VolumeError); + exit(VolumeError); #endif } }