From 876a923a82b584f3eca9536a4bdd902ebfba96c9 Mon Sep 17 00:00:00 2001 From: Georgia Stuart Date: Sun, 13 Oct 2024 23:03:01 -0400 Subject: [PATCH] Rework cuda file handling Signed-off-by: Georgia Stuart --- expui/CMakeLists.txt | 2 +- exputil/CMakeLists.txt | 7 ++++--- include/BiorthCube.H | 4 ++-- include/BiorthCyl.H | 6 +++--- include/EmpCylSL.H | 4 ++-- include/SLGridMP2.H | 6 +++--- pyEXP/CMakeLists.txt | 2 +- src/CMakeLists.txt | 16 ++++++---------- src/Component.H | 8 ++++---- src/Cube.H | 6 +++--- src/CylEXP.H | 2 +- src/Cylinder.H | 4 ++-- src/ExternalForce.H | 2 +- src/FlatDisk.H | 4 ++-- src/NVTX.H | 2 +- src/PolarBasis.H | 4 ++-- src/PotAccel.H | 2 +- src/SlabSL.H | 4 ++-- src/Sphere.H | 4 ++-- src/SphericalBasis.H | 4 ++-- src/user/CMakeLists.txt | 6 +++--- utils/Analysis/CMakeLists.txt | 2 +- utils/ICs/CMakeLists.txt | 2 +- utils/MSSA/CMakeLists.txt | 2 +- utils/PhaseSpace/CMakeLists.txt | 2 +- utils/SL/CMakeLists.txt | 2 +- utils/Test/CMakeLists.txt | 2 +- 27 files changed, 54 insertions(+), 57 deletions(-) diff --git a/expui/CMakeLists.txt b/expui/CMakeLists.txt index 331717f5a..7e55a3667 100644 --- a/expui/CMakeLists.txt +++ b/expui/CMakeLists.txt @@ -16,7 +16,7 @@ set(common_INCLUDE $ if(ENABLE_CUDA) - list(APPEND common_LINKLIB CUDA::cudart) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) list(APPEND common_LINKLIB CUDA::nvtx3) else () diff --git a/exputil/CMakeLists.txt b/exputil/CMakeLists.txt index b00437a45..9ccb58eca 100644 --- a/exputil/CMakeLists.txt +++ b/exputil/CMakeLists.txt @@ -51,14 +51,15 @@ set(common_INCLUDE_DIRS $ ${DEP_INC} ${EIGEN3_INCLUDE_DIR} ${HDF5_INCLUDE_DIRS} ${FFTW_INCLUDE_DIRS}) -set(common_LINKLIBS ${DEP_LIB} OpenMP::OpenMP_CXX MPI::MPI_CXX +set(common_LINKLIB ${DEP_LIB} OpenMP::OpenMP_CXX MPI::MPI_CXX yaml-cpp ${VTK_LIBRARIES} ${HDF5_LIBRARIES} ${HDF5_HL_LIBRARIES} ${FFTW_DOUBLE_LIB}) if(ENABLE_CUDA) - list(APPEND common_LINKLIB CUDA::cudart) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) list(APPEND common_LINKLIB CUDA::nvtx3) + set_source_files_properties(${BIORTH_SRC} ${PARTICLE_SRC} PROPERTIES LANGUAGE CUDA) else () list(APPEND common_LINKLIB CUDA::nvToolsExt) endif () @@ -74,7 +75,7 @@ endif() add_library(exputil ${exputil_SOURCES}) set_target_properties(exputil PROPERTIES OUTPUT_NAME exputil) target_include_directories(exputil PUBLIC ${common_INCLUDE_DIRS}) -target_link_libraries(exputil PUBLIC ${common_LINKLIBS}) +target_link_libraries(exputil PUBLIC ${common_LINKLIB}) install(TARGETS exputil DESTINATION lib) diff --git a/include/BiorthCube.H b/include/BiorthCube.H index f7325fd44..55dd19108 100644 --- a/include/BiorthCube.H +++ b/include/BiorthCube.H @@ -16,7 +16,7 @@ #include #include -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 #include #include #endif @@ -28,7 +28,7 @@ #include #include -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 #include #include #endif diff --git a/include/BiorthCyl.H b/include/BiorthCyl.H index 35f0a1e60..b052c7979 100644 --- a/include/BiorthCyl.H +++ b/include/BiorthCyl.H @@ -16,7 +16,7 @@ #include #include -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 #include #include #endif @@ -28,7 +28,7 @@ #include #include -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 #include #include #endif @@ -172,7 +172,7 @@ public: static std::map cacheInfo(const std::string& cachefile, bool verbose=true); -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 void initialize_cuda(std::vector& cuArray, thrust::host_vector& tex); diff --git a/include/EmpCylSL.H b/include/EmpCylSL.H index c7e9bddd2..233039ef7 100644 --- a/include/EmpCylSL.H +++ b/include/EmpCylSL.H @@ -21,7 +21,7 @@ #include #include -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 #include #include #endif @@ -924,7 +924,7 @@ public: //! Check orthogonality for basis (pyEXP style) std::vector orthoCheck(); -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 cudaMappingConstants getCudaMappingConstants(); void initialize_cuda(std::vector& cuArray, diff --git a/include/SLGridMP2.H b/include/SLGridMP2.H index 1e50d604a..f83d98fe9 100644 --- a/include/SLGridMP2.H +++ b/include/SLGridMP2.H @@ -18,7 +18,7 @@ #include using namespace __EXP__; -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 #include #include #endif @@ -167,7 +167,7 @@ public: //! produce matrices std::vector orthoCheck(int knots=40); -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 void initialize_cuda(std::vector& cuArray, thrust::host_vector& tex); @@ -453,7 +453,7 @@ public: //@} -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 void initialize_cuda(std::vector& cuArray, thrust::host_vector& tex); diff --git a/pyEXP/CMakeLists.txt b/pyEXP/CMakeLists.txt index e93b59cf6..dc8cebf0e 100644 --- a/pyEXP/CMakeLists.txt +++ b/pyEXP/CMakeLists.txt @@ -17,7 +17,7 @@ set(common_INCLUDE $ ${HDF5_INCLUDE_DIRS} ${EIGEN3_INCLUDE_DIR}) if(ENABLE_CUDA) - list(APPEND common_LINKLIB CUDA::cudart) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) list(APPEND common_LINKLIB CUDA::nvtx3) else () diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0594c836f..45827e5f9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -18,15 +18,6 @@ if (ENABLE_CUDA) cudaCylinder.cu cudaEmpCylSL.cu cudaComponent.cu NVTX.cc cudaIncpos.cu cudaIncvel.cu cudaMultistep.cu cudaOrient.cu cudaBiorthCyl.cu cudaCube.cu cudaSlabSL.cu) - - set_source_files_properties(Component.cc ComponentContainer.cc - Cube.cc Cylinder.cc ExternalForce.cc NVTX.cc OrbTrace.cc Orient.cc - OutAscii.cc OutCHKPT.cc OutCHKPTQ.cc OutCalbr.cc OutFrac.cc - OutLog.cc OutPS.cc OutPSN.cc OutPSP.cc OutPSQ.cc OutPSR.cc - OutputContainer.cc PolarBasis.cc PotAccel.cc SlabSL.cc Sphere.cc - SphericalBasis.cc begin.cc incpos.cc incvel.cc step.cc - PROPERTIES LANGUAGE CUDA) - endif() set(common_INCLUDE_DIRS @@ -46,9 +37,10 @@ if(PNG_FOUND) endif() if(ENABLE_CUDA) - list(APPEND common_LINKLIB CUDA::cudart) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) list(APPEND common_LINKLIB CUDA::nvtx3) + set_source_files_properties(${exp_SOURCES} PROPERTIES LANGUAGE CUDA) else () list(APPEND common_LINKLIB CUDA::nvToolsExt) endif () @@ -75,5 +67,9 @@ add_executable(exp expand.cc) target_include_directories(exp PUBLIC ${common_INCLUDE_DIRS}) target_link_libraries(exp PUBLIC ${common_LINKLIB} EXPlib) +if (ENABLE_CUDA) + set_target_properties(exp PROPERTIES LINKER_LANGUAGE CUDA) +endif () + install(TARGETS EXPlib DESTINATION lib) install(TARGETS exp DESTINATION bin) diff --git a/src/Component.H b/src/Component.H index 3f00ed9e5..5dfc1ceb5 100644 --- a/src/Component.H +++ b/src/Component.H @@ -19,7 +19,7 @@ #include -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 #include #endif @@ -569,7 +569,7 @@ public: //! Compute center of mass and center of velocity (CPU version) void fix_positions_cpu(unsigned mlevel=0); -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 //! Compute center of mass and center of velocity (GPU version) void fix_positions_cuda(unsigned mlevel=0); #endif @@ -577,7 +577,7 @@ public: //! Compute center of mass and center of velocity void fix_positions(unsigned mlevel=0) { -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 if (use_cuda) fix_positions_cuda(mlevel); else #endif @@ -926,7 +926,7 @@ public: //! Compute level from minimum requested time step from last master step inline bool DTreset() { return dtreset; } -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 //@{ //! CUDA utilities for handling host <===> device exchange diff --git a/src/Cube.H b/src/Cube.H index c9c6c99af..8e00413db 100644 --- a/src/Cube.H +++ b/src/Cube.H @@ -12,7 +12,7 @@ #include #include -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 #include #include #include @@ -41,7 +41,7 @@ private: //! Valid keys for YAML configurations static const std::set valid_keys; -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 virtual void determine_coefficients_cuda(); virtual void determine_acceleration_cuda(); virtual void multistep_update_cuda(); @@ -135,7 +135,7 @@ private: std::cout << std::string(60, '=') << std::endl; std::cout << "Time in CPU: " << duration0.count()-duration1.count() << std::endl; -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 if (c->cC->cudaDevice>=0) { std::cout << "Time in GPU: " << duration1.count() << std::endl; } diff --git a/src/CylEXP.H b/src/CylEXP.H index 713a7690c..2df55949f 100644 --- a/src/CylEXP.H +++ b/src/CylEXP.H @@ -5,7 +5,7 @@ #include #include -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 #include #include #endif diff --git a/src/Cylinder.H b/src/Cylinder.H index e2b9f24a0..08e012c58 100644 --- a/src/Cylinder.H +++ b/src/Cylinder.H @@ -10,7 +10,7 @@ #include -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 #include #include #endif @@ -171,7 +171,7 @@ protected: int sampT, defSampT; //! CUDA method for coefficient accumulation -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 virtual void determine_coefficients_cuda(bool compute_pca); virtual void determine_acceleration_cuda(); virtual void multistep_update_cuda(); diff --git a/src/ExternalForce.H b/src/ExternalForce.H index 781775a07..f445852cb 100644 --- a/src/ExternalForce.H +++ b/src/ExternalForce.H @@ -53,7 +53,7 @@ public: //! Finish and clean-up (caching data necessary for restart) virtual void finish() {} - // #if HAVE_LIBCUDA==1 && defined (__NVCC__) + // #if HAVE_LIBCUDA==1 #if HAVE_LIBCUDA==1 //! Copy particles from device for non-cuda forces void getParticlesCuda(Component *c); diff --git a/src/FlatDisk.H b/src/FlatDisk.H index 7ec72b644..64be763a4 100644 --- a/src/FlatDisk.H +++ b/src/FlatDisk.H @@ -9,7 +9,7 @@ typedef std::shared_ptr CylPtr; -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 #include #endif @@ -61,7 +61,7 @@ private: virtual double getRtable() { return ortho->getRtable(); } -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 virtual void initialize_cuda() { sampT = floor(sqrt(component->CurTotal())); diff --git a/src/NVTX.H b/src/NVTX.H index b091792c2..44947c197 100644 --- a/src/NVTX.H +++ b/src/NVTX.H @@ -5,7 +5,7 @@ #include -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 #if __CUDACC_VER_MAJOR__ < 12 #include diff --git a/src/PolarBasis.H b/src/PolarBasis.H index 6d68ff11c..caab18f0d 100644 --- a/src/PolarBasis.H +++ b/src/PolarBasis.H @@ -12,7 +12,7 @@ #include -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 #include #include #endif @@ -198,7 +198,7 @@ protected: virtual double getRtable() = 0; //! CUDA method for coefficient accumulation -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 virtual void determine_coefficients_cuda(bool compute_pca); virtual void determine_acceleration_cuda(); virtual void multistep_update_cuda(); diff --git a/src/PotAccel.H b/src/PotAccel.H index f6b8ac99d..2c4dbe49f 100644 --- a/src/PotAccel.H +++ b/src/PotAccel.H @@ -195,7 +195,7 @@ public: //! Execute to finish level shifts for particles virtual void multistep_update_finish() {} -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 //! Cuda implementation of level shifts virtual void multistep_update_cuda() {} #endif diff --git a/src/SlabSL.H b/src/SlabSL.H index f9eb1cdd2..bad380c12 100644 --- a/src/SlabSL.H +++ b/src/SlabSL.H @@ -11,7 +11,7 @@ #include #include -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 #include #include #include @@ -56,7 +56,7 @@ private: SlabSLCoefHeader coefheader; -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 virtual void determine_coefficients_cuda(); virtual void determine_acceleration_cuda(); virtual void multistep_update_cuda(); diff --git a/src/Sphere.H b/src/Sphere.H index 0d2141d14..0ec8e9c7a 100644 --- a/src/Sphere.H +++ b/src/Sphere.H @@ -9,7 +9,7 @@ typedef std::shared_ptr SLGridSphPtr; -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 #include #endif @@ -68,7 +68,7 @@ private: void make_model_bin(); void make_model_plummer(); -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 virtual void initialize_cuda() { ortho->initialize_cuda(cuInterpArray, tex); diff --git a/src/SphericalBasis.H b/src/SphericalBasis.H index 01c38f780..cd949a96e 100644 --- a/src/SphericalBasis.H +++ b/src/SphericalBasis.H @@ -12,7 +12,7 @@ #include -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 #include #include #endif @@ -117,7 +117,7 @@ protected: virtual void determine_coefficients_playback(void); //! CUDA method for coefficient accumulation -#if HAVE_LIBCUDA==1 && defined (__NVCC__) +#if HAVE_LIBCUDA==1 virtual void determine_coefficients_cuda(bool compute_pca); virtual void determine_acceleration_cuda(); virtual void multistep_update_cuda(); diff --git a/src/user/CMakeLists.txt b/src/user/CMakeLists.txt index 695bc301d..e340cce7e 100644 --- a/src/user/CMakeLists.txt +++ b/src/user/CMakeLists.txt @@ -9,11 +9,11 @@ set (common_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/..) -set (common_LINKLIBS ${DEP_LIB} OpenMP::OpenMP_CXX MPI::MPI_CXX +set (common_LINKLIB ${DEP_LIB} OpenMP::OpenMP_CXX MPI::MPI_CXX exputil EXPlib yaml-cpp) if(ENABLE_CUDA) - list(APPEND common_LINKLIB CUDA::cudart) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) list(APPEND common_LINKLIB CUDA::nvtx3) else () @@ -40,7 +40,7 @@ foreach(mlib ${USER_MODULES}) add_library(${mlib} ${${mlib}_SRC}) set_target_properties(${mlib} PROPERTIES OUTPUT_NAME ${mlib}) target_include_directories(${mlib} PUBLIC ${common_INCLUDE_DIRS}) - target_link_libraries(${mlib} PUBLIC ${common_LINKLIBS}) + target_link_libraries(${mlib} PUBLIC ${common_LINKLIB}) install(TARGETS ${mlib} DESTINATION lib/user) endforeach() diff --git a/utils/Analysis/CMakeLists.txt b/utils/Analysis/CMakeLists.txt index 124f435aa..8bc7e3a08 100644 --- a/utils/Analysis/CMakeLists.txt +++ b/utils/Analysis/CMakeLists.txt @@ -13,7 +13,7 @@ if(PNG_FOUND) endif() if(ENABLE_CUDA) - list(APPEND common_LINKLIB CUDA::cudart) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) list(APPEND common_LINKLIB CUDA::nvtx3) else () diff --git a/utils/ICs/CMakeLists.txt b/utils/ICs/CMakeLists.txt index b53496e63..0c89b9942 100644 --- a/utils/ICs/CMakeLists.txt +++ b/utils/ICs/CMakeLists.txt @@ -9,7 +9,7 @@ set(common_LINKLIB OpenMP::OpenMP_CXX MPI::MPI_CXX yaml-cpp exputil ${HDF5_CXX_LIBRARIES}) if(ENABLE_CUDA) - list(APPEND common_LINKLIB CUDA::cudart) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) list(APPEND common_LINKLIB CUDA::nvtx3) else () diff --git a/utils/MSSA/CMakeLists.txt b/utils/MSSA/CMakeLists.txt index 3b21103b3..f2a2690cc 100644 --- a/utils/MSSA/CMakeLists.txt +++ b/utils/MSSA/CMakeLists.txt @@ -17,7 +17,7 @@ set(common_INCLUDE $ ${HDF5_INCLUDE_DIRS}) if(ENABLE_CUDA) - list(APPEND common_LINKLIB CUDA::cudart) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) list(APPEND common_LINKLIB CUDA::nvtx3) else () diff --git a/utils/PhaseSpace/CMakeLists.txt b/utils/PhaseSpace/CMakeLists.txt index 4893a413e..63ced2c1f 100644 --- a/utils/PhaseSpace/CMakeLists.txt +++ b/utils/PhaseSpace/CMakeLists.txt @@ -21,7 +21,7 @@ if(PNG_FOUND) endif() if(ENABLE_CUDA) - list(APPEND common_LINKLIB CUDA::cudart) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) list(APPEND common_LINKLIB CUDA::nvtx3) else () diff --git a/utils/SL/CMakeLists.txt b/utils/SL/CMakeLists.txt index 44b4ef080..59afb08fa 100644 --- a/utils/SL/CMakeLists.txt +++ b/utils/SL/CMakeLists.txt @@ -6,7 +6,7 @@ set(common_LINKLIB OpenMP::OpenMP_CXX MPI::MPI_CXX yaml-cpp exputil ${VTK_LIBRARIES}) if(ENABLE_CUDA) - list(APPEND common_LINKLIB CUDA::cudart) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) list(APPEND common_LINKLIB CUDA::nvtx3) else () diff --git a/utils/Test/CMakeLists.txt b/utils/Test/CMakeLists.txt index 0d53e8b7a..91a38c6ac 100644 --- a/utils/Test/CMakeLists.txt +++ b/utils/Test/CMakeLists.txt @@ -17,7 +17,7 @@ set(common_INCLUDE ${CMAKE_CURRENT_SOURCE_DIR}/..) if(ENABLE_CUDA) - list(APPEND common_LINKLIB CUDA::cudart) + list(APPEND common_LINKLIB CUDA::toolkit CUDA::cudart) if (CUDAToolkit_VERSION VERSION_GREATER_EQUAL 12) list(APPEND common_LINKLIB CUDA::nvtx3) else ()