Merge pull request #89 from georgiastuart/fix-cuda-16

Rework cuda file handling for >= 12
EXP-code · Oct 14, 2024 · a35a90c · a35a90c
2 parents d24ae4f + 876a923
commit a35a90c
Show file tree

Hide file tree

Showing 27 changed files with 54 additions and 57 deletions.
diff --git a/expui/CMakeLists.txt b/expui/CMakeLists.txt
@@ -16,7 +16,7 @@ set(common_INCLUDE $<INSTALL_INTERFACE:include>
 
 
 if(ENABLE_CUDA)
-  list(APPEND common_LINKLIB CUDA::cudart)
+  list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
   if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
     list(APPEND common_LINKLIB CUDA::nvtx3)
   else ()

diff --git a/exputil/CMakeLists.txt b/exputil/CMakeLists.txt
@@ -51,14 +51,15 @@ set(common_INCLUDE_DIRS $<INSTALL_INTERFACE:include>
   ${DEP_INC} ${EIGEN3_INCLUDE_DIR} ${HDF5_INCLUDE_DIRS}
   ${FFTW_INCLUDE_DIRS})
 
-set(common_LINKLIBS ${DEP_LIB} OpenMP::OpenMP_CXX MPI::MPI_CXX
+set(common_LINKLIB ${DEP_LIB} OpenMP::OpenMP_CXX MPI::MPI_CXX
   yaml-cpp ${VTK_LIBRARIES} ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES}
   ${FFTW_DOUBLE_LIB})
 
 if(ENABLE_CUDA)
-  list(APPEND common_LINKLIB CUDA::cudart)
+  list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
   if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
     list(APPEND common_LINKLIB CUDA::nvtx3)
+    set_source_files_properties(${BIORTH_SRC} ${PARTICLE_SRC} PROPERTIES LANGUAGE CUDA)
   else ()
     list(APPEND common_LINKLIB CUDA::nvToolsExt)
   endif ()
@@ -74,7 +75,7 @@ endif()
 add_library(exputil ${exputil_SOURCES})
 set_target_properties(exputil PROPERTIES OUTPUT_NAME exputil)
 target_include_directories(exputil PUBLIC ${common_INCLUDE_DIRS})
-target_link_libraries(exputil PUBLIC ${common_LINKLIBS})
+target_link_libraries(exputil PUBLIC ${common_LINKLIB})
 
 install(TARGETS exputil DESTINATION lib)
 

diff --git a/include/BiorthCube.H b/include/BiorthCube.H
@@ -16,7 +16,7 @@
 #include <massmodel.H>
 #include <yaml-cpp/yaml.h>
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
 #include <cudaUtil.cuH>
 #include <cudaMappingConstants.cuH>
 #endif
@@ -28,7 +28,7 @@
 #include <highfive/H5DataSpace.hpp>
 #include <highfive/H5Attribute.hpp>
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
 #include <cudaParticle.cuH>
 #include <cudaMappingConstants.cuH>
 #endif

diff --git a/include/BiorthCyl.H b/include/BiorthCyl.H
@@ -16,7 +16,7 @@
 #include <massmodel.H>
 #include <yaml-cpp/yaml.h>
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
 #include <cudaUtil.cuH>
 #include <cudaMappingConstants.cuH>
 #endif
@@ -28,7 +28,7 @@
 #include <highfive/H5DataSpace.hpp>
 #include <highfive/H5Attribute.hpp>
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
 #include <cudaParticle.cuH>
 #include <cudaMappingConstants.cuH>
 #endif
@@ -172,7 +172,7 @@ public:
   static std::map<std::string, std::string>
   cacheInfo(const std::string& cachefile, bool verbose=true);
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
   void initialize_cuda(std::vector<cudaArray_t>& cuArray,
 		       thrust::host_vector<cudaTextureObject_t>& tex);
 

diff --git a/include/EmpCylSL.H b/include/EmpCylSL.H
@@ -21,7 +21,7 @@
 #include <SLGridMP2.H>
 #include <coef.H>
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
 #include <cudaParticle.cuH>
 #include <cudaMappingConstants.cuH>
 #endif
@@ -924,7 +924,7 @@ public:
   //! Check orthogonality for basis (pyEXP style)
   std::vector<Eigen::MatrixXd> orthoCheck();
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
   cudaMappingConstants getCudaMappingConstants();
 
   void initialize_cuda(std::vector<cudaArray_t>& cuArray,

diff --git a/include/SLGridMP2.H b/include/SLGridMP2.H
@@ -18,7 +18,7 @@
 #include <libvars.H>
 using namespace __EXP__;
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
 #include <cudaUtil.cuH>
 #include <cudaMappingConstants.cuH>
 #endif
@@ -167,7 +167,7 @@ public:
   //! produce matrices
   std::vector<Eigen::MatrixXd> orthoCheck(int knots=40);
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
   void initialize_cuda(std::vector<cudaArray_t>& cuArray,
 		       thrust::host_vector<cudaTextureObject_t>& tex);
 
@@ -453,7 +453,7 @@ public:
 
   //@}
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
   void initialize_cuda(std::vector<cudaArray_t>& cuArray,
 		       thrust::host_vector<cudaTextureObject_t>& tex);
 

diff --git a/pyEXP/CMakeLists.txt b/pyEXP/CMakeLists.txt
@@ -17,7 +17,7 @@ set(common_INCLUDE $<INSTALL_INTERFACE:include>
   ${HDF5_INCLUDE_DIRS} ${EIGEN3_INCLUDE_DIR})
 
 if(ENABLE_CUDA)
-  list(APPEND common_LINKLIB CUDA::cudart)
+  list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
   if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
     list(APPEND common_LINKLIB CUDA::nvtx3)
   else ()

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
@@ -18,15 +18,6 @@ if (ENABLE_CUDA)
     cudaCylinder.cu cudaEmpCylSL.cu cudaComponent.cu NVTX.cc
     cudaIncpos.cu cudaIncvel.cu cudaMultistep.cu cudaOrient.cu
     cudaBiorthCyl.cu cudaCube.cu cudaSlabSL.cu)
-
-  set_source_files_properties(Component.cc ComponentContainer.cc
-    Cube.cc Cylinder.cc ExternalForce.cc NVTX.cc OrbTrace.cc Orient.cc
-    OutAscii.cc OutCHKPT.cc OutCHKPTQ.cc OutCalbr.cc OutFrac.cc
-    OutLog.cc OutPS.cc OutPSN.cc OutPSP.cc OutPSQ.cc OutPSR.cc
-    OutputContainer.cc PolarBasis.cc PotAccel.cc SlabSL.cc Sphere.cc
-    SphericalBasis.cc begin.cc incpos.cc incvel.cc step.cc
-    PROPERTIES LANGUAGE CUDA)
-
 endif()
 
 set(common_INCLUDE_DIRS 
@@ -46,9 +37,10 @@ if(PNG_FOUND)
 endif()
 
 if(ENABLE_CUDA)
-  list(APPEND common_LINKLIB CUDA::cudart)
+  list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart)
   if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12)
     list(APPEND common_LINKLIB CUDA::nvtx3)
+    set_source_files_properties(${exp_SOURCES} PROPERTIES LANGUAGE CUDA)
   else ()
     list(APPEND common_LINKLIB CUDA::nvToolsExt)
   endif ()
@@ -75,5 +67,9 @@ add_executable(exp expand.cc)
 target_include_directories(exp PUBLIC ${common_INCLUDE_DIRS})
 target_link_libraries(exp PUBLIC ${common_LINKLIB} EXPlib)
 
+if (ENABLE_CUDA)
+  set_target_properties(exp PROPERTIES LINKER_LANGUAGE CUDA)
+endif ()
+
 install(TARGETS EXPlib DESTINATION lib)
 install(TARGETS exp DESTINATION bin)
diff --git a/src/Component.H b/src/Component.H
@@ -19,7 +19,7 @@
 
 #include <config_exp.h>
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
 #include <cudaParticle.cuH>  
 #endif
 
@@ -569,15 +569,15 @@ public:
   //! Compute center of mass and center of velocity (CPU version)
   void fix_positions_cpu(unsigned mlevel=0);
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
   //! Compute center of mass and center of velocity (GPU version)
   void fix_positions_cuda(unsigned mlevel=0);
 #endif
 
   //! Compute center of mass and center of velocity
   void fix_positions(unsigned mlevel=0)
   {
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
     if (use_cuda) fix_positions_cuda(mlevel);
     else
 #endif
@@ -926,7 +926,7 @@ public:
   //! Compute level from minimum requested time step from last master step
   inline bool DTreset()  { return dtreset;  }
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
   //@{
   //! CUDA utilities for handling host <===> device exchange
 

diff --git a/src/Cube.H b/src/Cube.H
@@ -12,7 +12,7 @@
 #include <Coefficients.H>
 #include <PotAccel.H>
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
 #include <thrust/complex.h>
 #include <cudaParticle.cuH>
 #include <cudaMappingConstants.cuH>
@@ -41,7 +41,7 @@ private:
   //! Valid keys for YAML configurations
   static const std::set<std::string> valid_keys;
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
   virtual void determine_coefficients_cuda();
   virtual void determine_acceleration_cuda();
   virtual void multistep_update_cuda();
@@ -135,7 +135,7 @@ private:
 	std::cout << std::string(60, '=') << std::endl;
 	std::cout << "Time in CPU: "
 		  << duration0.count()-duration1.count() << std::endl;
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
 	if (c->cC->cudaDevice>=0) {
 	  std::cout << "Time in GPU: " << duration1.count() << std::endl;
 	}

diff --git a/src/CylEXP.H b/src/CylEXP.H
@@ -5,7 +5,7 @@
 #include <expand.H>
 #include <EmpCylSL.H>
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
 #include <cudaParticle.cuH>
 #include <cudaMappingConstants.cuH>
 #endif

diff --git a/src/Cylinder.H b/src/Cylinder.H
@@ -10,7 +10,7 @@
 
 #include <config_exp.h>
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
 #include <cudaParticle.cuH>
 #include <cudaMappingConstants.cuH>
 #endif
@@ -171,7 +171,7 @@ protected:
   int sampT, defSampT;
 
   //! CUDA method for coefficient accumulation
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
   virtual void determine_coefficients_cuda(bool compute_pca);
   virtual void determine_acceleration_cuda();
   virtual void multistep_update_cuda();

diff --git a/src/ExternalForce.H b/src/ExternalForce.H
@@ -53,7 +53,7 @@ public:
   //! Finish and clean-up (caching data necessary for restart)
   virtual void finish() {}
 
-  // #if HAVE_LIBCUDA==1 && defined (__NVCC__)
+  // #if HAVE_LIBCUDA==1
 #if HAVE_LIBCUDA==1
   //! Copy particles from device for non-cuda forces
   void getParticlesCuda(Component *c);

diff --git a/src/FlatDisk.H b/src/FlatDisk.H
@@ -9,7 +9,7 @@
 
 typedef std::shared_ptr<BiorthCyl> CylPtr;
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
 #include <cudaUtil.cuH>
 #endif
 
@@ -61,7 +61,7 @@ private:
 
   virtual double getRtable() { return ortho->getRtable(); }
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
   virtual void initialize_cuda()
   {
     sampT = floor(sqrt(component->CurTotal()));

diff --git a/src/NVTX.H b/src/NVTX.H
@@ -5,7 +5,7 @@
 
 #include <config_exp.h>
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
 
 #if __CUDACC_VER_MAJOR__ < 12
 #include <nvToolsExt.h>

diff --git a/src/PolarBasis.H b/src/PolarBasis.H
@@ -12,7 +12,7 @@
 
 #include <config_exp.h>
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
 #include <cudaParticle.cuH>
 #include <cudaMappingConstants.cuH>
 #endif
@@ -198,7 +198,7 @@ protected:
   virtual double getRtable() = 0;
 
   //! CUDA method for coefficient accumulation
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
   virtual void determine_coefficients_cuda(bool compute_pca);
   virtual void determine_acceleration_cuda();
   virtual void multistep_update_cuda();

diff --git a/src/PotAccel.H b/src/PotAccel.H
@@ -195,7 +195,7 @@ public:
   //! Execute to finish level shifts for particles
   virtual void multistep_update_finish() {}
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
   //! Cuda implementation of level shifts
   virtual void multistep_update_cuda() {}
 #endif

diff --git a/src/SlabSL.H b/src/SlabSL.H
@@ -11,7 +11,7 @@
 #include <biorth1d.H>
 #include <PotAccel.H>
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
 #include <thrust/complex.h>
 #include <cudaParticle.cuH>
 #include <cudaMappingConstants.cuH>
@@ -56,7 +56,7 @@ private:
 
   SlabSLCoefHeader coefheader;
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
   virtual void determine_coefficients_cuda();
   virtual void determine_acceleration_cuda();
   virtual void multistep_update_cuda();

diff --git a/src/Sphere.H b/src/Sphere.H
@@ -9,7 +9,7 @@
 
 typedef std::shared_ptr<SLGridSph> SLGridSphPtr;
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
 #include <cudaUtil.cuH>
 #endif
 
@@ -68,7 +68,7 @@ private:
   void make_model_bin();
   void make_model_plummer();
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
   virtual void initialize_cuda()
   {
     ortho->initialize_cuda(cuInterpArray, tex);

diff --git a/src/SphericalBasis.H b/src/SphericalBasis.H
@@ -12,7 +12,7 @@
 
 #include <config_exp.h>
 
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
 #include <cudaParticle.cuH>
 #include <cudaMappingConstants.cuH>
 #endif
@@ -117,7 +117,7 @@ protected:
   virtual void determine_coefficients_playback(void);
 
   //! CUDA method for coefficient accumulation
-#if HAVE_LIBCUDA==1 && defined (__NVCC__)
+#if HAVE_LIBCUDA==1
   virtual void determine_coefficients_cuda(bool compute_pca);
   virtual void determine_acceleration_cuda();
   virtual void multistep_update_cuda();