Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

(WIP) Added code to use joined array #5363

Open
wants to merge 10 commits into
base: development
Choose a base branch
from
Open

(WIP) Added code to use joined array #5363

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
d525cb5
Added code to use joined array (currently joined array will be applied
guj Oct 3, 2024
f773584
fixed warning
guj Oct 3, 2024
d8d0680
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 3, 2024
b62cc8e
check openpmd-api version before using joined array
guj Oct 3, 2024
a9701cb
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Oct 3, 2024
61647dd
fix compiler warning
guj Oct 3, 2024
53a0d43
by pass compiler error
guj Oct 3, 2024
f687cf3
added nodiscard
guj Oct 3, 2024
7741def
fixed warning
guj Oct 3, 2024
3db0e75
more warning fix
guj Oct 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion Source/Diagnostics/WarpXOpenPMD.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ public:
using ParticleContainer = typename WarpXParticleContainer::ContainerLike<amrex::PinnedArenaAllocator>;
using ParticleIter = typename amrex::ParIterSoA<PIdx::nattribs, 0, amrex::PinnedArenaAllocator>;

WarpXParticleCounter (ParticleContainer* pc);
WarpXParticleCounter (ParticleContainer* pc, bool doLazyCount);
[[nodiscard]] unsigned long GetTotalNumParticles () const {return m_Total;}

std::vector<unsigned long long> m_ParticleOffsetAtRank;
Expand Down Expand Up @@ -336,6 +336,9 @@ private:
int m_MPIRank = 0;
int m_MPISize = 1;

bool m_ApplyJoinedArray = false; // joined array
[[nodiscard]] openPMD::Extent ProperExtent(unsigned long long n, [[maybe_unused]] bool init) const;

openPMD::IterationEncoding m_Encoding = openPMD::IterationEncoding::fileBased;
std::string m_OpenPMDFileType = "bp"; //! MPI-parallel openPMD backend: bp or h5
std::string m_OpenPMDoptions = "{}"; //! JSON option string for openPMD::Series constructor
Expand Down
88 changes: 70 additions & 18 deletions Source/Diagnostics/WarpXOpenPMD.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -391,6 +391,30 @@ WarpXOpenPMDPlot::WarpXOpenPMDPlot (
{
m_OpenPMDoptions = detail::getSeriesOptions(operator_type, operator_parameters,
engine_type, engine_parameters);

#if OPENPMDAPI_VERSION_GE(0, 16, 0)
// Automatically use Joined Array in particles for bp4/bp5 postfixed files
std::size_t found_bp4 = m_OpenPMDFileType.find("bp4");
std::size_t found_bp5 = m_OpenPMDFileType.find("bp5");
if ( (found_bp4 != std::string::npos) || (found_bp5 != std::string::npos) )
m_ApplyJoinedArray = true;
amrex::Print()<<" ... [Using Joined Array for particles in ADIOS??] "<<m_ApplyJoinedArray<<std::endl;
#endif
}

[[nodiscard]] openPMD::Extent WarpXOpenPMDPlot::ProperExtent (unsigned long long n, [[maybe_unused]] bool init) const
{
#if OPENPMDAPI_VERSION_GE(0, 16, 0)
if (!m_ApplyJoinedArray)
return {n};

if (init)
return {openPMD::Dataset::JOINED_DIMENSION};
else
return {};
#else
return {n};
#endif
}

WarpXOpenPMDPlot::~WarpXOpenPMDPlot ()
Expand Down Expand Up @@ -529,7 +553,12 @@ WarpXOpenPMDPlot::WriteOpenPMDParticles (const amrex::Vector<ParticleDiag>& part
const bool isLastBTDFlush
)
{
WARPX_PROFILE("WarpXOpenPMDPlot::WriteOpenPMDParticles()");
std::string name="WarpXOpenPMDPlot::WriteOpenPMDParticles()";
if ( m_ApplyJoinedArray ) {
name +="_joined ";
}
WARPX_PROFILE(name);


for (unsigned i = 0, n = particle_diags.size(); i < n; ++i) {

Expand Down Expand Up @@ -665,7 +694,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc,
AMREX_ALWAYS_ASSERT(real_comp_names.size() == pc->NumRealComps());
AMREX_ALWAYS_ASSERT(int_comp_names.size() == pc->NumIntComps());

WarpXParticleCounter counter(pc);
WarpXParticleCounter counter(pc, m_ApplyJoinedArray);
auto const num_dump_particles = counter.GetTotalNumParticles();

openPMD::Iteration currIteration = GetIteration(iteration, isBTD);
Expand Down Expand Up @@ -712,10 +741,15 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc,
NewParticleVectorSize, isBTD);
}

#ifdef NO_JOINED_DIM // need to see how to revive const vars in openpmd (needs the size which should get from gathering and we skip gathering with joined array)
if (is_last_flush_to_step) {
SetConstParticleRecordsEDPIC(currSpecies, positionComponents, NewParticleVectorSize, charge, mass);
}

#else
if ( iteration < -100 ) { // temporarily. avoid compiler error, should never be true
SetConstParticleRecordsEDPIC(currSpecies, positionComponents, NewParticleVectorSize, charge, mass);
}
#endif
// open files from all processors, in case some will not contribute below
m_Series->flush();

Expand Down Expand Up @@ -758,6 +792,7 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc,
// BP4 (ADIOS 2.8): last MPI rank's `Put` meta-data wins
// BP5 (ADIOS 2.8): everyone has to write an empty block
if (is_resizing_flush && !contributed_particles && isBTD && m_Series->backend() == "ADIOS2") {
#ifdef NO_JOINED_DIM // possibly not needed, will verify
for( auto & [record_name, record] : currSpecies ) {
for( auto & [comp_name, comp] : record ) {
if (comp.constant()) { continue; }
Expand Down Expand Up @@ -795,8 +830,8 @@ WarpXOpenPMDPlot::DumpToFile (ParticleContainer* pc,
}
}
}
#endif
}

m_Series->flush();
}

Expand All @@ -811,8 +846,8 @@ WarpXOpenPMDPlot::SetupRealProperties (ParticleContainer const * pc,
{
std::string options = "{}";
if (isBTD) { options = "{ \"resizable\": true }"; }
auto dtype_real = openPMD::Dataset(openPMD::determineDatatype<amrex::ParticleReal>(), {np}, options);
auto dtype_int = openPMD::Dataset(openPMD::determineDatatype<int>(), {np}, options);
auto dtype_real = openPMD::Dataset(openPMD::determineDatatype<amrex::ParticleReal>(), ProperExtent(np, true), options);
auto dtype_int = openPMD::Dataset(openPMD::determineDatatype<int>(), ProperExtent(np, true), options);
//
// the beam/input3d showed write_real_comp.size() = 16 while only 10 real comp names
// so using the min to be safe.
Expand Down Expand Up @@ -904,7 +939,7 @@ WarpXOpenPMDPlot::SaveRealProperty (ParticleIter& pti,
{
// todo: add support to not write the particle index
getComponentRecord("id").storeChunkRaw(
soa.GetIdCPUData().data(), {offset}, {numParticleOnTile64});
soa.GetIdCPUData().data(), ProperExtent(offset, false), {numParticleOnTile64});
}

// here we the save the SoA properties (real)
Expand Down Expand Up @@ -935,10 +970,10 @@ WarpXOpenPMDPlot::SaveRealProperty (ParticleIter& pti,
if (write_real_comp[1]) { y.get()[i] = yp; }
}
if (write_real_comp[0]) {
getComponentRecord(real_comp_names[0]).storeChunk(x, {offset}, {numParticleOnTile64});
getComponentRecord(real_comp_names[0]).storeChunk(x, ProperExtent(offset, false) , {numParticleOnTile64});
}
if (write_real_comp[1]) {
getComponentRecord(real_comp_names[1]).storeChunk(y, {offset}, {numParticleOnTile64});
getComponentRecord(real_comp_names[1]).storeChunk(y, ProperExtent(offset, false), {numParticleOnTile64});
}
#endif

Expand All @@ -958,7 +993,7 @@ WarpXOpenPMDPlot::SaveRealProperty (ParticleIter& pti,
#endif
if (write_real_comp[idx]) {
getComponentRecord(real_comp_names[idx]).storeChunkRaw(
soa.GetRealData(soa_r_idx).data(), {offset}, {numParticleOnTile64});
soa.GetRealData(soa_r_idx).data(), ProperExtent(offset, false), {numParticleOnTile64});
}
}
}
Expand All @@ -968,7 +1003,7 @@ WarpXOpenPMDPlot::SaveRealProperty (ParticleIter& pti,
for (auto idx=0; idx<int_counter; idx++) {
if (write_int_comp[idx]) {
getComponentRecord(int_comp_names[idx]).storeChunkRaw(
soa.GetIntData(idx).data(), {offset}, {numParticleOnTile64});
soa.GetIntData(idx).data(), ProperExtent(offset, false), {numParticleOnTile64});
}
}
}
Expand All @@ -984,8 +1019,8 @@ WarpXOpenPMDPlot::SetupPos (
{
std::string options = "{}";
if (isBTD) { options = "{ \"resizable\": true }"; }
auto realType = openPMD::Dataset(openPMD::determineDatatype<amrex::ParticleReal>(), {np}, options);
auto idType = openPMD::Dataset(openPMD::determineDatatype< uint64_t >(), {np}, options);
auto realType = openPMD::Dataset(openPMD::determineDatatype<amrex::ParticleReal>(), ProperExtent(np, true), options);
auto idType = openPMD::Dataset(openPMD::determineDatatype< uint64_t >(), ProperExtent(np, true), options);

for( auto const& comp : positionComponents ) {
currSpecies["position"][comp].resetDataset( realType );
Expand All @@ -1003,7 +1038,7 @@ WarpXOpenPMDPlot::SetConstParticleRecordsEDPIC (
amrex::ParticleReal const charge,
amrex::ParticleReal const mass)
{
auto realType = openPMD::Dataset(openPMD::determineDatatype<amrex::ParticleReal>(), {np});
auto realType = openPMD::Dataset(openPMD::determineDatatype<amrex::ParticleReal>(), ProperExtent(np, true));
const auto *const scalar = openPMD::RecordComponent::SCALAR;

// define record shape to be number of particles
Expand Down Expand Up @@ -1477,14 +1512,18 @@ WarpXOpenPMDPlot::WriteOpenPMDFieldsAll ( //const std::string& filename,
//
//
//
WarpXParticleCounter::WarpXParticleCounter (ParticleContainer* pc):
WarpXParticleCounter::WarpXParticleCounter (ParticleContainer* pc, bool doLazyCount):
m_MPIRank{amrex::ParallelDescriptor::MyProc()},
m_MPISize{amrex::ParallelDescriptor::NProcs()}
{
m_ParticleCounterByLevel.resize(pc->finestLevel()+1);
m_ParticleOffsetAtRank.resize(pc->finestLevel()+1);
m_ParticleSizeAtRank.resize(pc->finestLevel()+1);
WARPX_PROFILE("WarpXParticleCounter");

const unsigned long long zero=0;
m_ParticleCounterByLevel.resize(pc->finestLevel()+1, zero);
m_ParticleOffsetAtRank.resize(pc->finestLevel()+1, zero);
m_ParticleSizeAtRank.resize(pc->finestLevel()+1, zero);

if (!doLazyCount) {
for (auto currentLevel = 0; currentLevel <= pc->finestLevel(); currentLevel++)
{
long numParticles = 0; // numParticles in this processor
Expand All @@ -1510,6 +1549,19 @@ WarpXParticleCounter::WarpXParticleCounter (ParticleContainer* pc):

m_Total += sum;
}
} // !do lazy count
else
{
for (auto currentLevel = 0; currentLevel <= pc->finestLevel(); currentLevel++)
{
long numParticles = 0; // numParticles in this processor
for (ParticleIter pti(*pc, currentLevel); pti.isValid(); ++pti) {
auto numParticleOnTile = pti.numParticles();
numParticles += numParticleOnTile;
}
m_Total += numParticles;
}
} // else
}


Expand Down
Loading