Made numerous changes to get code working on GPUs.

PNNL-CompBio · Jan 19, 2024 · f605a5c · f605a5c
1 parent 4b38c31
commit f605a5c
Show file tree

Hide file tree

Showing 13 changed files with 98 additions and 39 deletions.
diff --git a/src/chemistry/bmx_cell_interaction.H b/src/chemistry/bmx_cell_interaction.H
@@ -31,7 +31,7 @@ class BMXCellInteraction
    * Return a vector containing force parameters
    * @return force parameters
    */
-  std::vector<Real> getForceParams();
+  amrex::Gpu::DeviceVector<Real> getForceParams();
 
   protected:
   /**
@@ -70,7 +70,7 @@ class BMXCellInteraction
 
   static Real p_fluc_mix;      // fluctuation mixing coefficient
 
-  static std::vector<Real> p_force_params;  // vector containing all force params
+  static amrex::Gpu::DeviceVector<Real> p_force_params;  // vector containing all force params
 
   private:
 

diff --git a/src/chemistry/bmx_cell_interaction.cpp b/src/chemistry/bmx_cell_interaction.cpp
@@ -25,7 +25,7 @@ amrex::Real BMXCellInteraction::p_bond_cutoff = 5.0;
 amrex::Real BMXCellInteraction::p_viscosity = 20.0;
 amrex::Real BMXCellInteraction::p_ran_scale = 0.0;
 amrex::Real BMXCellInteraction::p_fluc_mix = 0.0;
-std::vector<amrex::Real> BMXCellInteraction::p_force_params;
+amrex::Gpu::DeviceVector<amrex::Real> BMXCellInteraction::p_force_params;
 
 /**
  * Retrieve instance of BMXCellInteraction object
@@ -109,7 +109,7 @@ void BMXCellInteraction::setParams(const char* /*file*/)
  * Return a vector containing force parameters
  * @return force parameters
  */
-std::vector<Real> BMXCellInteraction::getForceParams()
+amrex::Gpu::DeviceVector<Real> BMXCellInteraction::getForceParams()
 {
   return p_force_params;
 }
diff --git a/src/chemistry/bmx_chem.H b/src/chemistry/bmx_chem.H
@@ -206,18 +206,18 @@ class BMXChemistry
    * chemistry
    * @param chempar vector of parameters used in chemistry
    */
-  void getChemParams(std::vector<Real> &chempar);
+  void getChemParams(amrex::Gpu::DeviceVector<Real> &chempar);
 
   /** Return a vector containing exchange parameters
    * @return vector of exchange parameters
    */
-  std::vector<Real> getExchangeParameters();
+  amrex::Gpu::DeviceVector<Real> getExchangeParameters();
 
   /**
    * Return a vector containing parameters for segment fusion
    * @return vector of fusion parameters
    */
-  std::vector<Real> getFusionParameters();
+  amrex::Gpu::DeviceVector<Real> getFusionParameters();
 
   // Fraction of overlap
   static Real p_overlap;

diff --git a/src/chemistry/bmx_chem.cpp b/src/chemistry/bmx_chem.cpp
@@ -231,7 +231,7 @@ int BMXChemistry::getIntData(int idx)
  * chemistry
  * @param chempar vector of parameters used in chemistry
  */
-void BMXChemistry::getChemParams(std::vector<Real> &chempar)
+void BMXChemistry::getChemParams(amrex::Gpu::DeviceVector<Real> &chempar)
 {
   chempar.clear();
   chempar.push_back(k1);
@@ -260,9 +260,9 @@ void BMXChemistry::getChemParams(std::vector<Real> &chempar)
 /** Returen a vector containing exchange parameters
  * @return vector of exchange parameters
  */
-std::vector<Real> BMXChemistry::getExchangeParameters()
+amrex::Gpu::DeviceVector<Real> BMXChemistry::getExchangeParameters()
 {
-  std::vector<Real> ret;
+  amrex::Gpu::DeviceVector<Real> ret;
   ret.push_back(mtA);
   ret.push_back(mtB);
   ret.push_back(mtC);
@@ -273,9 +273,9 @@ std::vector<Real> BMXChemistry::getExchangeParameters()
  * Return a vector containing parameters for segment fusion
  * @return vector of fusion parameters
  */
-std::vector<Real> BMXChemistry::getFusionParameters()
+amrex::Gpu::DeviceVector<Real> BMXChemistry::getFusionParameters()
 {
-  std::vector<Real> ret;
+  amrex::Gpu::DeviceVector<Real> ret;
   ret.push_back(fusion_prob);
   ret.push_back(max_fusion_separation);
   ret.push_back(scale_inc);

diff --git a/src/chemistry/bmx_chem_K.H b/src/chemistry/bmx_chem_K.H
@@ -30,6 +30,7 @@ checkSplit(Real *p_par, int *p_ipar, Real max_vol, Real max_len,
   int cell_type = p_ipar[intIdx::cell_type];
   *n_new = 0;
   if (cell_type == cellType::YEAST) {
+    std::printf("Check yeast split vol: %f max_vol: %f\n",p_par[realIdx::vol],max_vol);
     if (p_par[realIdx::vol] > max_vol) ret = true;
 #ifndef AMREX_USE_GPU
     if (ret) printf("SPLITTING cell id: %d cpu: %d\n",
@@ -1471,6 +1472,7 @@ void xferMeshToParticleAndUpdateChem(Real grid_vol, Real npart,
   if (cell_type == cellType::FUNGI) {
     if (cell_par[realIdx::radius] >= radius_max &&
         cell_par[realIdx::c_length] >= length_max) {
+      std::printf("Setting growth to zero\n");
       kg = 0.0;
       kv = 0.0;
     }

diff --git a/src/des/bmx_calc_txfr.cpp b/src/des/bmx_calc_txfr.cpp
@@ -76,7 +76,7 @@ bmx::bmx_calc_txfr_particle (Real time, Real dt)
 
 #ifdef NEW_CHEM
   BMXChemistry *bmxchem = BMXChemistry::instance();
-  std::vector<Real> chempar_vec;
+  amrex::Gpu::DeviceVector<Real> chempar_vec;
   bmxchem->getChemParams(chempar_vec);
   Real *chempar = &chempar_vec[0];
 #endif
@@ -310,6 +310,29 @@ bmx::bmx_calc_txfr_particle (Real time, Real dt)
             interp_gyarray,interp_gzarray]
             AMREX_GPU_DEVICE (int pid) noexcept
               {
+#if 0
+              std::printf("chempar[0]: %f\n",chempar[0]);
+              std::printf("chempar[1]: %f\n",chempar[1]);
+              std::printf("chempar[2]: %f\n",chempar[2]);
+              std::printf("chempar[3]: %f\n",chempar[3]);
+              std::printf("chempar[4]: %f\n",chempar[4]);
+              std::printf("chempar[5]: %f\n",chempar[5]);
+              std::printf("chempar[6]: %f\n",chempar[6]);
+              std::printf("chempar[7]: %f\n",chempar[7]);
+              std::printf("chempar[8]: %f\n",chempar[8]);
+              std::printf("chempar[9]: %f\n",chempar[9]);
+              std::printf("chempar[10]: %f\n",chempar[10]);
+              std::printf("chempar[11]: %f\n",chempar[11]);
+              std::printf("chempar[12]: %f\n",chempar[12]);
+              std::printf("chempar[13]: %f\n",chempar[13]);
+              std::printf("chempar[14]: %f\n",chempar[14]);
+              std::printf("chempar[15]: %f\n",chempar[15]);
+              std::printf("chempar[16]: %f\n",chempar[16]);
+              std::printf("chempar[17]: %f\n",chempar[17]);
+              std::printf("chempar[18]: %f\n",chempar[18]);
+              std::printf("chempar[19]: %f\n",chempar[19]);
+              std::printf("chempar[20]: %f\n",chempar[20]);
+#endif
               // Local array storing interpolated values
               GpuArray<Real, interp_ncomp> interp_loc;
 

diff --git a/src/des/bmx_pc.H b/src/des/bmx_pc.H
@@ -112,7 +112,7 @@ public:
     void InitParticlesAscii (const std::string& file);
 
     void WriteToAscii (const std::string& file,int nstep,amrex::Real time);
-    void countParticleTypes(int &nparticles, int &nsegments);
+    void CountParticleTypes(int &nparticles, int &nsegments);
     void WriteSegmentsToAscii (const std::string& file,int nstep,
         amrex::Real time, int nsegments);
     void WriteParticlesToAscii (const std::string& file,int nstep,

diff --git a/src/des/bmx_pc_fusion.cpp b/src/des/bmx_pc_fusion.cpp
@@ -32,11 +32,12 @@ bool BMXParticleContainer::EvaluateTipFusion (const Vector<MultiFab*> cost,
     if (n_at_lev == 0) continue;
 
     BMXChemistry *chemistry = BMXChemistry::instance();
-    std::vector<Real> fpar_vec = chemistry->getFusionParameters();
+    amrex::Gpu::DeviceVector<Real> fpar_vec = chemistry->getFusionParameters();
     Real *fpar = &fpar_vec[0];
     BMXCellInteraction *interaction = BMXCellInteraction::instance();
-    std::vector<Real> xpar_vec = interaction->getForceParams();
-    Real *xpar = &xpar_vec[0];
+    amrex::Gpu::DeviceVector<Real> xpar_vec = interaction->getForceParams();
+    // Real *xpar = &xpar_vec[0];
+    auto xpar = xpar_vec.data();
     /****************************************************************************
      * DEBUG flag toggles:                                                      *
      *   -> Print number of collisions                                          *
@@ -227,11 +228,12 @@ void BMXParticleContainer::EvaluateInteriorFusion (const Vector<MultiFab*> cost,
     if (n_at_lev == 0) continue;
 
     BMXChemistry *chemistry = BMXChemistry::instance();
-    std::vector<Real> fpar_vec = chemistry->getFusionParameters();
+    amrex::Gpu::DeviceVector<Real> fpar_vec = chemistry->getFusionParameters();
     Real *fpar = &fpar_vec[0];
     BMXCellInteraction *interaction = BMXCellInteraction::instance();
-    std::vector<Real> xpar_vec = interaction->getForceParams();
-    Real *xpar = &xpar_vec[0];
+    amrex::Gpu::DeviceVector<Real> xpar_vec = interaction->getForceParams();
+    // Real *xpar = &xpar_vec[0];
+    auto xpar = xpar_vec.data();
     /****************************************************************************
      * DEBUG flag toggles:                                                      *
      *   -> Print number of collisions                                          *
@@ -490,17 +492,14 @@ void BMXParticleContainer::CleanupFusion (const Vector<MultiFab*> cost,
   int l_num_ints  = BMXChemistry::p_num_ints;
   amrex::Print() << "Cleanup fusion bonds" <<std::endl;
 
-  /*
-  BMXCellInteraction *interaction = BMXCellInteraction::instance();
-  std::vector<Real> xpar_vec = interaction->getForceParams();
-  Real *xpar = &xpar_vec[0];
-  */
   BMXChemistry *bmxchem = BMXChemistry::instance();
-  std::vector<Real> fpar_vec = bmxchem->getFusionParameters();
-  Real *fpar = &fpar_vec[0];
+  amrex::Gpu::DeviceVector<Real> fpar_vec = bmxchem->getFusionParameters();
+  // Real *fpar = &fpar_vec[0];
+  auto fpar = fpar_vec.data();
   BMXCellInteraction *interaction = BMXCellInteraction::instance();
-  std::vector<Real> xpar_vec = interaction->getForceParams();
-  Real *xpar = &xpar_vec[0];
+  amrex::Gpu::DeviceVector<Real> xpar_vec = interaction->getForceParams();
+  // Real *xpar = &xpar_vec[0];
+  auto xpar = xpar_vec.data();
 
   Real max_len = SPECIES::max_len;
 

diff --git a/src/des/bmx_pc_init.cpp b/src/des/bmx_pc_init.cpp
@@ -98,6 +98,9 @@ void BMXParticleContainer::InitParticlesAscii (const std::string& file)
       host_particles[i].cpu() = ParallelDescriptor::MyProc();
       host_particles[i].idata(intIdx::id) = static_cast<int>(host_particles[i].id());
       host_particles[i].idata(intIdx::cpu) = static_cast<int>(host_particles[i].cpu());
+      host_particles[i].idata(intIdx::position) = siteLocation::TIP;
+      host_particles[i].idata(intIdx::n_bnds) = 0;
+      host_particles[i].idata(intIdx::fuse_flag) = 0;
 
       if (!ifs.good())
           amrex::Abort("Error initializing particles from Ascii file. \n");

diff --git a/src/des/bmx_pc_interaction.cpp b/src/des/bmx_pc_interaction.cpp
@@ -32,8 +32,9 @@ void BMXParticleContainer::EvolveParticles (Real dt,
                    << " ... with fluid dt " << dt << std::endl;
 
     BMXCellInteraction *interaction = BMXCellInteraction::instance();
-    std::vector<Real> fpar_vec = interaction->getForceParams();
-    Real *fpar = &fpar_vec[0];
+    amrex::Gpu::DeviceVector<Real> fpar_vec = interaction->getForceParams();
+    // Real *fpar = &fpar_vec[0];
+    auto fpar = fpar_vec.data();
     /****************************************************************************
      * DEBUG flag toggles:                                                      *
      *   -> Print number of collisions                                          *
@@ -172,6 +173,23 @@ void BMXParticleContainer::EvolveParticles (Real dt,
               AMREX_GPU_DEVICE (int i, amrex::RandomEngine const& engine) noexcept
               {
                   auto& particle = pstruct[i];
+#if 0
+                  std::printf("fpar[0]: %f\n",fpar[0]);
+                  std::printf("fpar[1]: %f\n",fpar[1]);
+                  std::printf("fpar[2]: %f\n",fpar[2]);
+                  std::printf("fpar[3]: %f\n",fpar[3]);
+                  std::printf("fpar[4]: %f\n",fpar[4]);
+                  std::printf("fpar[5]: %f\n",fpar[5]);
+                  std::printf("fpar[6]: %f\n",fpar[6]);
+                  std::printf("fpar[7]: %f\n",fpar[7]);
+                  std::printf("fpar[8]: %f\n",fpar[8]);
+                  std::printf("fpar[9]: %f\n",fpar[9]);
+                  std::printf("fpar[10]: %f\n",fpar[10]);
+                  std::printf("fpar[11]: %f\n",fpar[11]);
+                  std::printf("fpar[12]: %f\n",fpar[12]);
+                  std::printf("fpar[13]: %f\n",fpar[13]);
+                  std::printf("fpar[14]: %f\n",fpar[14]);
+#endif
 
                   RealVect pos1(particle.pos());
                   // clean up flags
@@ -515,8 +533,9 @@ void BMXParticleContainer::InitBonds (const Vector<MultiFab*> cost,
     Real rlim = 1.0e-7;
 
     BMXCellInteraction *interaction = BMXCellInteraction::instance();
-    std::vector<Real> fpar_vec = interaction->getForceParams();
-    Real *fpar = &fpar_vec[0];
+    amrex::Gpu::DeviceVector<Real> fpar_vec = interaction->getForceParams();
+    // Real *fpar = &fpar_vec[0];
+    auto fpar = fpar_vec.data();
 
     for (int lev = 0; lev <= finest_level; lev++)
     {
@@ -630,6 +649,10 @@ void BMXParticleContainer::InitBonds (const Vector<MultiFab*> cost,
 
           }
           } // end of neighbor loop
+          if (particle.idata(intIdx::cell_type) == cellType::FUNGI &&
+              particle.idata(intIdx::n_bnds) > 1) {
+            particle.idata(intIdx::position) = siteLocation::INTERIOR;
+          }
           }); // end of loop over particles
 
       amrex::Gpu::Device::synchronize();

diff --git a/src/des/bmx_pc_write.cpp b/src/des/bmx_pc_write.cpp
@@ -13,7 +13,8 @@ using PinnedTile = typename BMXParticleContainer::ContainerLike<amrex::PinnedAre
 void BMXParticleContainer::WriteToAscii (const std::string& vtk_filename,int nstep, Real time)
 {
   int nsegments, nparticles;
-  countParticleTypes(nparticles, nsegments);
+  CountParticleTypes(nparticles, nsegments);
+  amrex::Print()<<"Nparticles: "<<nparticles<<" Nsegments: "<<nsegments<<std::endl;
   if (nsegments > 0) WriteSegmentsToAscii(vtk_filename, nstep, time, nsegments);
   if (nparticles > 0) WriteParticlesToAscii(vtk_filename, nstep, time, nparticles);
 }
@@ -478,7 +479,7 @@ void BMXParticleContainer::WriteParticlesToAscii (const std::string& vtk_filenam
 
 }
 
-void BMXParticleContainer::countParticleTypes (int &nparticles, int &nsegments)
+void BMXParticleContainer::CountParticleTypes (int &nparticles, int &nsegments)
 {
   BL_PROFILE("BMXParticleContainer::WriteSegmentsToAscii");
 
@@ -520,6 +521,7 @@ void BMXParticleContainer::countParticleTypes (int &nparticles, int &nsegments)
         int type = particle.idata(intIdx::cell_type);
         if (type == cellType::FUNGI) {
           (*nseg)++;
+          std::printf("Increment nseg: %d\n",*nseg);
         } else {
           (*npart)++;
         }
@@ -531,6 +533,7 @@ void BMXParticleContainer::countParticleTypes (int &nparticles, int &nsegments)
   int buf[2];
   buf[0] = nparticles;
   buf[1] = nsegments;
+  std::printf("nparticles: %d nsegments: %d\n",nparticles,nsegments);
   ParallelDescriptor::ReduceIntSum(buf,2);
   nparticles = buf[0];
   nsegments = buf[1];

diff --git a/src/des/bmx_pc_xchng.cpp b/src/des/bmx_pc_xchng.cpp
@@ -32,8 +32,9 @@ void BMXParticleContainer::ParticleExchange (Real dt,
                    << " ... with fluid dt " << dt << std::endl;
 
     BMXChemistry *chemistry = BMXChemistry::instance();
-    std::vector<Real> xpar_vec = chemistry->getExchangeParameters();
-    Real *xpar = &xpar_vec[0];
+    amrex::Gpu::DeviceVector<Real> xpar_vec = chemistry->getExchangeParameters();
+    //Real *xpar = &xpar_vec[0];
+    auto xpar = xpar_vec.data();
 
     // Debug level controls the detail of debug output:
     //   -> debug_level = 0 : no debug output
@@ -123,9 +124,14 @@ void BMXParticleContainer::ParticleExchange (Real dt,
               if (particle.idata(intIdx::position) == siteLocation::TIP &&
                   particle.idata(intIdx::n_bnds) > 2) {
                 int *idata = &particle.idata(0);
-                printf("particle id: %d cpu: %d is Tip with %d bonds\n",
+                std::printf("particle id: %d cpu: %d is Tip with %d bonds\n",
                     idata[intIdx::id],idata[intIdx::cpu],idata[intIdx::n_bnds]);
               }
+#if 0
+              std::printf("xpar[0]: %f\n",xpar[0]);
+              std::printf("xpar[1]: %f\n",xpar[1]);
+              std::printf("xpar[2]: %f\n",xpar[2]);
+#endif
 
 
                   // initialize particle before evaluating exchange

diff --git a/subprojects/amrex b/subprojects/amrex
+4 −0		Src/Base/AMReX.cpp
+1 −1		Src/Base/AMReX_VisMF.cpp
+4 −0		Src/Base/AMReX_VisMFBuffer.H
+1 −14		Src/LinearSolvers/MLMG/AMReX_MLEBNodeFDLaplacian.cpp
+1 −1		Src/LinearSolvers/MLMG/AMReX_MLNodeLaplacian_sten.cpp
+3 −1		Src/LinearSolvers/MLMG/AMReX_MLNodeLinOp.H
+29 −7		Tools/CMake/AMReXBuildInfo.cmake
+21 −10		Tools/CMake/AMReXInstallHelpers.cmake
+6 −2		Tools/CMake/AMReXTypecheck.cmake
+0 −244		Tools/CMake/FindFilesystem.cmake
+0 −8		Tools/Plotfile/CMakeLists.txt
+1 −2		Tools/Plotfile/fgradient.cpp