From 1652c9a474a2a2d251bf30cee09cf919e84b5096 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Tue, 16 May 2017 10:16:55 +0200 Subject: [PATCH 01/19] Check for OpenMPI version and disable shared memory windows only if <2.1.1 --- CMakeExt/MPI.cmake | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/CMakeExt/MPI.cmake b/CMakeExt/MPI.cmake index fc2eaf00a..73a1b08b0 100644 --- a/CMakeExt/MPI.cmake +++ b/CMakeExt/MPI.cmake @@ -30,9 +30,18 @@ if (MPI_INCLUDE_PATH AND MPI_LIBRARY) elseif ("${MPI_INCLUDE_PATH}" MATCHES "openmpi") set(MPI_IMPL_IS_OPENMPI TRUE CACHE BOOL "OpenMPI detected") set(MPI_IMPL_ID "openmpi" CACHE STRING "MPI implementation identifier") - # temporarily disable shared memory windows due to alignment problems - set(ENABLE_SHARED_WINDOWS OFF) - message(WARNING "MPI shared windows disabled due to defective allocation in OpenMPI") + MESSAGE(STATUS "Checking OpenMPI version") + TRY_COMPILE(MPI_VERSION_OK ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/CMakeExt/Code/test_openmpi_version.c + CMAKE_FLAGS -DINCLUDE_DIRECTORIES=${MPI_INCLUDE_PATH} + OUTPUT_VARIABLE OUTPUT) + if (NOT MPI_VERSION_OK) + # disable shared memory windows due to alignment problems + set(ENABLE_SHARED_WINDOWS OFF) + message(WARNING + "Disabling shared memory window support due to defective allocation " + "in OpenMPI <2.1.1") + endif() endif() else (MPI_INCLUDE_PATH AND MPI_LIBRARY) set(MPI_FOUND FALSE CACHE BOOL "Did not find the MPI library") From e380dc5cac191c34fa07fcd8d4dd3b6563b84322 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Tue, 16 May 2017 10:17:20 +0200 Subject: [PATCH 02/19] Do not check for MPI twice --- dart-impl/mpi/CMakeLists.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/dart-impl/mpi/CMakeLists.txt b/dart-impl/mpi/CMakeLists.txt index bc9119560..67a7c2b1e 100644 --- a/dart-impl/mpi/CMakeLists.txt +++ b/dart-impl/mpi/CMakeLists.txt @@ -6,8 +6,6 @@ set(DASH_DART_IMPL_MPI_LIBRARY dart-mpi) set(DASH_DART_BASE_LIBRARY dart-base) -include(${CMAKE_SOURCE_DIR}/CMakeExt/MPI.cmake) - if (MPI_NOTFOUND) message(FATAL_ERROR, "No MPI implementation found for dart-mpi") endif() From 1c85eca4ad8e03f206385b18eb3ff45fe21f46e1 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Tue, 16 May 2017 10:26:44 +0200 Subject: [PATCH 03/19] Add check for OpenMPI 1.10.7 --- CMakeExt/Code/test_openmpi_version.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 CMakeExt/Code/test_openmpi_version.c diff --git a/CMakeExt/Code/test_openmpi_version.c b/CMakeExt/Code/test_openmpi_version.c new file mode 100644 index 000000000..7ef13cb51 --- /dev/null +++ b/CMakeExt/Code/test_openmpi_version.c @@ -0,0 +1,25 @@ +#include + + +/** + * OpenMPI <2.1.1 breaks alignment of + * allocated memory in shared memory + * windows. + */ + +#if (OMPI_MAJOR_VERSION == 1 \ + && OMPI_MINOR_VERSION < 11 \ + && OMPI_RELEASE_VERSION < 7) \ + || (OMPI_MAJOR_VERSION == 2 \ + && OMPI_MINOR_VERSION < 2 \ + && OMPI_RELEASE_VERSION < 1) + +#error "At least vesion 1.10.7 or 2.1.1 of OpenMPI required!" + +#endif + + +int main() +{ + return 0; +} From b8edff180c44677b68792f1580a1a77898aa39eb Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Tue, 16 May 2017 14:25:21 +0200 Subject: [PATCH 04/19] Move MPI check to dart-impl/ --- CMakeLists.txt | 1 - dart-impl/CMakeLists.txt | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3081deaaf..99a8b3131 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -124,7 +124,6 @@ endif() # Load build modules to locate libraries after environment setup # has been loaded: -include(${CMAKE_SOURCE_DIR}/CMakeExt/MPI.cmake) include(${CMAKE_SOURCE_DIR}/CMakeExt/PAPI.cmake) include(${CMAKE_SOURCE_DIR}/CMakeExt/Hwloc.cmake) include(${CMAKE_SOURCE_DIR}/CMakeExt/Likwid.cmake) diff --git a/dart-impl/CMakeLists.txt b/dart-impl/CMakeLists.txt index d366455ee..9050b3d5d 100644 --- a/dart-impl/CMakeLists.txt +++ b/dart-impl/CMakeLists.txt @@ -4,6 +4,7 @@ set(DART_IMPLEMENTATIONS_LIST ${DART_IMPLEMENTATIONS_LIST} set(ENABLE_THREADSUPPORT ${ENABLE_THREADSUPPORT} PARENT_SCOPE) + if (ENABLE_THREADSUPPORT) set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DDART_ENABLE_THREADSUPPORT") @@ -18,6 +19,7 @@ add_subdirectory(base) if (";${DART_IMPLEMENTATIONS_LIST};" MATCHES ";mpi;") set(DART_IMPLEMENTATION_MPI_ENABLED ON CACHE BOOL INTERNAL FORCE) + include(${CMAKE_SOURCE_DIR}/CMakeExt/MPI.cmake) add_subdirectory(mpi) endif() From 03bbe73ecd754a305dfba4fefd1a73fa5e446f05 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Tue, 16 May 2017 16:46:16 +0200 Subject: [PATCH 05/19] Detect MPI implementation based on macros and move OpenMPI check to CMake --- CMakeExt/Code/print_mpi_impl.c | 31 +++++++++++++++++++++ CMakeExt/Code/test_openmpi_version.c | 25 ----------------- CMakeExt/MPI.cmake | 41 ++++++++++++++++++++-------- 3 files changed, 61 insertions(+), 36 deletions(-) create mode 100644 CMakeExt/Code/print_mpi_impl.c delete mode 100644 CMakeExt/Code/test_openmpi_version.c diff --git a/CMakeExt/Code/print_mpi_impl.c b/CMakeExt/Code/print_mpi_impl.c new file mode 100644 index 000000000..20e846ba2 --- /dev/null +++ b/CMakeExt/Code/print_mpi_impl.c @@ -0,0 +1,31 @@ +#include + +int main() +{ + +#if defined(I_MPI_VERSION) + // Intel MPI defines both MPICH_VERSION + // and I_MPI_VERSION + printf("impi %s", I_MPI_VERSION); + return 0; +#elif defined(MVAPICH2_VERSION) + // MVAPICH defines both MPICH_VERSION + // and MVAPICH2_VERSION + printf("mvapich %s", MVAPICH2_VERSION); + return 0; +#elif defined(MPICH) + printf("mpich %s", MPICH_VERSION); + return 0; +#elif defined(OMPI_MAJOR_VERSION) + printf("openmpi %d.%d.%d", + OMPI_MAJOR_VERSION, + OMPI_MINOR_VERSION, + OMPI_RELEASE_VERSION); + return 0; +#else + // return error on unknown implementation + printf("unknown 0.0.0"); + return 1; +#endif +} + diff --git a/CMakeExt/Code/test_openmpi_version.c b/CMakeExt/Code/test_openmpi_version.c deleted file mode 100644 index 7ef13cb51..000000000 --- a/CMakeExt/Code/test_openmpi_version.c +++ /dev/null @@ -1,25 +0,0 @@ -#include - - -/** - * OpenMPI <2.1.1 breaks alignment of - * allocated memory in shared memory - * windows. - */ - -#if (OMPI_MAJOR_VERSION == 1 \ - && OMPI_MINOR_VERSION < 11 \ - && OMPI_RELEASE_VERSION < 7) \ - || (OMPI_MAJOR_VERSION == 2 \ - && OMPI_MINOR_VERSION < 2 \ - && OMPI_RELEASE_VERSION < 1) - -#error "At least vesion 1.10.7 or 2.1.1 of OpenMPI required!" - -#endif - - -int main() -{ - return 0; -} diff --git a/CMakeExt/MPI.cmake b/CMakeExt/MPI.cmake index 73a1b08b0..680c7f864 100644 --- a/CMakeExt/MPI.cmake +++ b/CMakeExt/MPI.cmake @@ -13,35 +13,54 @@ endif() find_package(MPI) if (MPI_INCLUDE_PATH AND MPI_LIBRARY) + try_run(RESULT_VAR COMPILE_RESULT_VAR + ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/CMakeExt/Code/print_mpi_impl.c + CMAKE_FLAGS -DINCLUDE_DIRECTORIES=${MPI_INCLUDE_PATH} + COMPILE_OUTPUT_VARIABLE COMPILE_OUTPUT + RUN_OUTPUT_VARIABLE MPI_STRING) + if (NOT COMPILE_RESULT_VAR) + message(FATAL_ERROR "Failed to determine MPI library: ${COMPILE_OUTPUT}") + endif() + if (NOT RESULT_VAR EQUAL 0) + message(FATAL_ERROR "Failed to determine MPI library: ${MPI_STRING}") + endif() + + string(REPLACE " " ";" MPI_LIST ${MPI_STRING}) + list(GET MPI_LIST 0 MPI_IMPL_NAME) + list(GET MPI_LIST 1 MPI_IMPL_VERSION) + + message(STATUS "Found MPI implementation: ${MPI_IMPL_NAME} ${MPI_IMPL_VERSION}") + set(MPI_FOUND TRUE CACHE BOOL "Found the MPI library") - if ("${MPI_INCLUDE_PATH}" MATCHES "mpich") + if (MPI_IMPL_NAME MATCHES "mpich") set(MPI_IMPL_IS_MPICH TRUE CACHE BOOL "MPICH detected") set(MPI_IMPL_ID "mpich" CACHE STRING "MPI implementation identifier") + # Cray MPI identifies as MPICH elseif ("${MPI_INCLUDE_PATH}" MATCHES "cray") set(MPI_IMPL_IS_CRAY TRUE CACHE BOOL "CrayMPI detected") set(MPI_IMPL_ID "craympi" CACHE STRING "MPI implementation identifier") - elseif ("${MPI_INCLUDE_PATH}" MATCHES "mvapich") + elseif (MPI_IMPL_NAME MATCHES "mvapich") set(MPI_IMPL_IS_MVAPICH TRUE CACHE BOOL "MVAPICH detected") set(MPI_IMPL_ID "mvapich" CACHE STRING "MPI implementation identifier") - elseif ("${MPI_INCLUDE_PATH}" MATCHES "impi" - OR "${MPI_INCLUDE_PATH}" MATCHES "intel") + elseif (MPI_IMPL_NAME MATCHES "impi") set(MPI_IMPL_IS_INTEL TRUE CACHE BOOL "IntelMPI detected") set(MPI_IMPL_ID "intelmpi" CACHE STRING "MPI implementation identifier") - elseif ("${MPI_INCLUDE_PATH}" MATCHES "openmpi") + elseif (MPI_IMPL_NAME MATCHES "openmpi") set(MPI_IMPL_IS_OPENMPI TRUE CACHE BOOL "OpenMPI detected") set(MPI_IMPL_ID "openmpi" CACHE STRING "MPI implementation identifier") - MESSAGE(STATUS "Checking OpenMPI version") - TRY_COMPILE(MPI_VERSION_OK ${CMAKE_BINARY_DIR} - ${CMAKE_SOURCE_DIR}/CMakeExt/Code/test_openmpi_version.c - CMAKE_FLAGS -DINCLUDE_DIRECTORIES=${MPI_INCLUDE_PATH} - OUTPUT_VARIABLE OUTPUT) - if (NOT MPI_VERSION_OK) + # Open MPI before 1.10.7 and 2.1.1 failed to properly align memory + if (${MPI_IMPL_VERSION} VERSION_LESS 1.10.7 OR + ${MPI_IMPL_VERSION} VERSION_EQUAL 2.0.0 OR + (${MPI_IMPL_VERSION} VERSION_GREATER 2.0.0 AND ${MPI_IMPL_VERSION} VERSION_LESS 2.1.1)) # disable shared memory windows due to alignment problems set(ENABLE_SHARED_WINDOWS OFF) message(WARNING "Disabling shared memory window support due to defective allocation " "in OpenMPI <2.1.1") endif() + else () + message(FATAL_ERROR "Unknown MPI implementation detected: ${MPI_STRING}") endif() else (MPI_INCLUDE_PATH AND MPI_LIBRARY) set(MPI_FOUND FALSE CACHE BOOL "Did not find the MPI library") From 4ed0184955f0e2d3fb3d2bbb7788ba77e393a48e Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Tue, 16 May 2017 16:48:27 +0200 Subject: [PATCH 06/19] Check for Cray MPI first as it claims to be MPICH --- CMakeExt/MPI.cmake | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CMakeExt/MPI.cmake b/CMakeExt/MPI.cmake index 680c7f864..bba36563e 100644 --- a/CMakeExt/MPI.cmake +++ b/CMakeExt/MPI.cmake @@ -33,13 +33,13 @@ if (MPI_INCLUDE_PATH AND MPI_LIBRARY) message(STATUS "Found MPI implementation: ${MPI_IMPL_NAME} ${MPI_IMPL_VERSION}") set(MPI_FOUND TRUE CACHE BOOL "Found the MPI library") - if (MPI_IMPL_NAME MATCHES "mpich") - set(MPI_IMPL_IS_MPICH TRUE CACHE BOOL "MPICH detected") - set(MPI_IMPL_ID "mpich" CACHE STRING "MPI implementation identifier") # Cray MPI identifies as MPICH - elseif ("${MPI_INCLUDE_PATH}" MATCHES "cray") + if ("${MPI_INCLUDE_PATH}" MATCHES "cray") set(MPI_IMPL_IS_CRAY TRUE CACHE BOOL "CrayMPI detected") set(MPI_IMPL_ID "craympi" CACHE STRING "MPI implementation identifier") + elseif (MPI_IMPL_NAME MATCHES "mpich") + set(MPI_IMPL_IS_MPICH TRUE CACHE BOOL "MPICH detected") + set(MPI_IMPL_ID "mpich" CACHE STRING "MPI implementation identifier") elseif (MPI_IMPL_NAME MATCHES "mvapich") set(MPI_IMPL_IS_MVAPICH TRUE CACHE BOOL "MVAPICH detected") set(MPI_IMPL_ID "mvapich" CACHE STRING "MPI implementation identifier") From ccabd5e369f9e199d582582d54f45893227c9133 Mon Sep 17 00:00:00 2001 From: Felix Moessbauer Date: Wed, 17 May 2017 11:31:33 +0200 Subject: [PATCH 07/19] moved MPI include to / and added MPI_IS_DART_COMPATIBLE to fail in dart if MPI is not compatible --- CMakeExt/HDF5.cmake | 2 +- CMakeExt/MPI.cmake | 6 +++++- CMakeLists.txt | 1 + dart-impl/CMakeLists.txt | 4 +++- dash/CMakeLists.txt | 4 ++-- dash/include/dash/io/hdf5/StorageDriver.h | 8 ++++---- 6 files changed, 16 insertions(+), 9 deletions(-) diff --git a/CMakeExt/HDF5.cmake b/CMakeExt/HDF5.cmake index 2a80c6851..7f4fbd129 100644 --- a/CMakeExt/HDF5.cmake +++ b/CMakeExt/HDF5.cmake @@ -63,7 +63,7 @@ if(HDF5_FOUND) check_symbol_exists(H5_HAVE_PARALLEL "H5pubconf.h" HAVE_H5_PARALLEL) cmake_pop_check_state() - if(NOT HAVE_H5_PARALLEL) + if(NOT HAVE_H5_PARALLEL OR "${MPI_IMPL_ID}" STREQUAL "") message(STATUS "HDF5 provides only serial version") set(HDF5_FOUND OFF CACHE BOOL "HDF5_FOUND" FORCE) unset(HDF5_LIBRARIES) diff --git a/CMakeExt/MPI.cmake b/CMakeExt/MPI.cmake index bba36563e..aac447bce 100644 --- a/CMakeExt/MPI.cmake +++ b/CMakeExt/MPI.cmake @@ -80,6 +80,10 @@ check_symbol_exists( cmake_pop_check_state() if (NOT HAVE_MPI_NO_OP) - message(FATAL_ERROR "Detected MPI library does not support MPI-3.") + set(MPI_IS_DART_COMPATIBLE FALSE CACHE BOOL + "MPI LIB has support for MPI-3") +else() + set(MPI_IS_DART_COMPATIBLE TRUE CACHE BOOL + "MPI LIB has support for MPI-3") endif() diff --git a/CMakeLists.txt b/CMakeLists.txt index 99a8b3131..3081deaaf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -124,6 +124,7 @@ endif() # Load build modules to locate libraries after environment setup # has been loaded: +include(${CMAKE_SOURCE_DIR}/CMakeExt/MPI.cmake) include(${CMAKE_SOURCE_DIR}/CMakeExt/PAPI.cmake) include(${CMAKE_SOURCE_DIR}/CMakeExt/Hwloc.cmake) include(${CMAKE_SOURCE_DIR}/CMakeExt/Likwid.cmake) diff --git a/dart-impl/CMakeLists.txt b/dart-impl/CMakeLists.txt index 9050b3d5d..fed5199cf 100644 --- a/dart-impl/CMakeLists.txt +++ b/dart-impl/CMakeLists.txt @@ -17,9 +17,11 @@ add_subdirectory(base) # DART implementation of every enabled DART variant: if (";${DART_IMPLEMENTATIONS_LIST};" MATCHES ";mpi;") + if(NOT MPI_IS_DART_COMPATIBLE) + message(FATAL_ERROR "Detected MPI library does not support MPI-3.") + endif() set(DART_IMPLEMENTATION_MPI_ENABLED ON CACHE BOOL INTERNAL FORCE) - include(${CMAKE_SOURCE_DIR}/CMakeExt/MPI.cmake) add_subdirectory(mpi) endif() diff --git a/dash/CMakeLists.txt b/dash/CMakeLists.txt index 477a30206..70206014b 100644 --- a/dash/CMakeLists.txt +++ b/dash/CMakeLists.txt @@ -287,14 +287,14 @@ foreach (dart_variant ${DART_IMPLEMENTATIONS_LIST}) ${ADDITIONAL_LINK_FLAGS} ${ADDITIONAL_LIBRARIES} ) - if (${dart_variant} STREQUAL "mpi") +# if (${dart_variant} STREQUAL "mpi") include_directories( ${MPI_INCLUDE_PATH}) target_link_libraries( ${DASH_LIBRARY} ${MPI_C_LIBRARIES} ${MPI_CXX_LIBRARIES}) - endif() +# endif() set_target_properties( ${DASH_LIBRARY} PROPERTIES COMPILE_FLAGS "${VARIANT_ADDITIONAL_COMPILE_FLAGS}" diff --git a/dash/include/dash/io/hdf5/StorageDriver.h b/dash/include/dash/io/hdf5/StorageDriver.h index 47c2ffbc7..2fc98a137 100644 --- a/dash/include/dash/io/hdf5/StorageDriver.h +++ b/dash/include/dash/io/hdf5/StorageDriver.h @@ -5,6 +5,10 @@ #ifdef DASH_ENABLE_HDF5 +#ifndef MPI_IMPL_ID +#pragma error "HDF5 module requires dart-mpi" +#endif + #include #include #include @@ -28,10 +32,6 @@ #include #include -#ifndef MPI_IMPL_ID -#pragma error "HDF5 module requires dart-mpi" -#endif - #include namespace dash { From 6f16c419a90827caa0605ff9fcd5f1f70c4df2da Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Wed, 17 May 2017 12:48:49 +0200 Subject: [PATCH 08/19] Open MPI 1.10.7 shared memory window allocation is still misaligned... --- CMakeExt/MPI.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeExt/MPI.cmake b/CMakeExt/MPI.cmake index aac447bce..96b89c98a 100644 --- a/CMakeExt/MPI.cmake +++ b/CMakeExt/MPI.cmake @@ -50,7 +50,7 @@ if (MPI_INCLUDE_PATH AND MPI_LIBRARY) set(MPI_IMPL_IS_OPENMPI TRUE CACHE BOOL "OpenMPI detected") set(MPI_IMPL_ID "openmpi" CACHE STRING "MPI implementation identifier") # Open MPI before 1.10.7 and 2.1.1 failed to properly align memory - if (${MPI_IMPL_VERSION} VERSION_LESS 1.10.7 OR + if (${MPI_IMPL_VERSION} VERSION_LESS 1.10.8 OR ${MPI_IMPL_VERSION} VERSION_EQUAL 2.0.0 OR (${MPI_IMPL_VERSION} VERSION_GREATER 2.0.0 AND ${MPI_IMPL_VERSION} VERSION_LESS 2.1.1)) # disable shared memory windows due to alignment problems From dc7ecbc511bc808ab49f83cc235db1df6697dadf Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Fri, 19 May 2017 17:30:47 +0200 Subject: [PATCH 09/19] Use multiple try_compile to detect MPI implementation --- CMakeExt/Code/print_mpi_impl.c | 31 ------ CMakeExt/Code/test_compatible_ompi.c | 17 +++ CMakeExt/Code/test_mpi_support.c | 12 +++ CMakeExt/MPI.cmake | 152 +++++++++++++++++---------- CMakeLists.txt | 2 +- 5 files changed, 124 insertions(+), 90 deletions(-) delete mode 100644 CMakeExt/Code/print_mpi_impl.c create mode 100644 CMakeExt/Code/test_compatible_ompi.c create mode 100644 CMakeExt/Code/test_mpi_support.c diff --git a/CMakeExt/Code/print_mpi_impl.c b/CMakeExt/Code/print_mpi_impl.c deleted file mode 100644 index 20e846ba2..000000000 --- a/CMakeExt/Code/print_mpi_impl.c +++ /dev/null @@ -1,31 +0,0 @@ -#include - -int main() -{ - -#if defined(I_MPI_VERSION) - // Intel MPI defines both MPICH_VERSION - // and I_MPI_VERSION - printf("impi %s", I_MPI_VERSION); - return 0; -#elif defined(MVAPICH2_VERSION) - // MVAPICH defines both MPICH_VERSION - // and MVAPICH2_VERSION - printf("mvapich %s", MVAPICH2_VERSION); - return 0; -#elif defined(MPICH) - printf("mpich %s", MPICH_VERSION); - return 0; -#elif defined(OMPI_MAJOR_VERSION) - printf("openmpi %d.%d.%d", - OMPI_MAJOR_VERSION, - OMPI_MINOR_VERSION, - OMPI_RELEASE_VERSION); - return 0; -#else - // return error on unknown implementation - printf("unknown 0.0.0"); - return 1; -#endif -} - diff --git a/CMakeExt/Code/test_compatible_ompi.c b/CMakeExt/Code/test_compatible_ompi.c new file mode 100644 index 000000000..526469ed7 --- /dev/null +++ b/CMakeExt/Code/test_compatible_ompi.c @@ -0,0 +1,17 @@ +#include + +int main() +{ + +#if defined(OMPI_MAJOR_VERSION) \ + && OMPI_MAJOR_VERSION > 2 \ + || ( OMPI_MAJOR_VERSION == 2 \ + && OMPI_MINOR_VERSION >= 1 \ + && OMPI_RELEASE_VERSION >= 1) + // Open MPI >=2.1.1 is fine + return 0; +#else +#error "Open MPI version with broken alignment of shared memory windows detected!" +#endif +} + diff --git a/CMakeExt/Code/test_mpi_support.c b/CMakeExt/Code/test_mpi_support.c new file mode 100644 index 000000000..af6bb2011 --- /dev/null +++ b/CMakeExt/Code/test_mpi_support.c @@ -0,0 +1,12 @@ +#include + +int main() +{ + +#if MPI_VERSION >= 3 + return 0; +#else +#error "Support for at least MPI 3.0 required!" +#endif +} + diff --git a/CMakeExt/MPI.cmake b/CMakeExt/MPI.cmake index 96b89c98a..c05895fff 100644 --- a/CMakeExt/MPI.cmake +++ b/CMakeExt/MPI.cmake @@ -10,78 +10,114 @@ if (NOT $ENV{MPI_CXX_COMPILER} STREQUAL "") CACHE STRING "MPI C++ compiler") endif() +if (CMAKE_CROSSCOMPILE) + message(STATUS "Cross compiling: ${CMAKE_CROSSCOMPILE}") +endif() + find_package(MPI) -if (MPI_INCLUDE_PATH AND MPI_LIBRARY) - try_run(RESULT_VAR COMPILE_RESULT_VAR - ${CMAKE_BINARY_DIR} - ${CMAKE_SOURCE_DIR}/CMakeExt/Code/print_mpi_impl.c - CMAKE_FLAGS -DINCLUDE_DIRECTORIES=${MPI_INCLUDE_PATH} - COMPILE_OUTPUT_VARIABLE COMPILE_OUTPUT - RUN_OUTPUT_VARIABLE MPI_STRING) - if (NOT COMPILE_RESULT_VAR) - message(FATAL_ERROR "Failed to determine MPI library: ${COMPILE_OUTPUT}") - endif() - if (NOT RESULT_VAR EQUAL 0) - message(FATAL_ERROR "Failed to determine MPI library: ${MPI_STRING}") + +# save current state +cmake_push_check_state() +set (CMAKE_REQUIRED_INCLUDES ${MPI_INCLUDE_PATH}) + +# check for Open MPI +check_symbol_exists( + OMPI_MAJOR_VERSION + mpi.h + HAVE_OPEN_MPI +) +if (HAVE_OPEN_MPI) + set (MPI_IMPL_IS_OPENMPI TRUE CACHE BOOL "OpenMPI detected") + set (MPI_IMPL_ID "openmpi" CACHE STRING "MPI implementation identifier") + try_compile(OMPI_OK ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/CMakeExt/Code/test_compatible_ompi.c + CMAKE_FLAGS -DINCLUDE_DIRECTORIES=${MPI_INCLUDE_PATH} + OUTPUT_VARIABLE OUTPUT) + if (NOT OMPI_OK) + message(WARNING + "Disabling shared memory window support due to defective allocation " + "in OpenMPI <2.1.1") + set (ENABLE_SHARED_WINDOWS OFF) endif() +endif() - string(REPLACE " " ";" MPI_LIST ${MPI_STRING}) - list(GET MPI_LIST 0 MPI_IMPL_NAME) - list(GET MPI_LIST 1 MPI_IMPL_VERSION) +# order matters: all of the following +# implementations also define MPICH +if (NOT DEFINED MPI_IMPL_ID) + # check for Intel MPI + check_symbol_exists( + I_MPI_VERSION + mpi.h + HAVE_I_MPI + ) + if (HAVE_I_MPI) + set (MPI_IMPL_IS_INTEL TRUE CACHE BOOL "IntelMPI detected") + set (MPI_IMPL_ID "intelmpi" CACHE STRING "MPI implementation identifier") + endif () +endif () - message(STATUS "Found MPI implementation: ${MPI_IMPL_NAME} ${MPI_IMPL_VERSION}") +if (NOT DEFINED MPI_IMPL_ID) + # check for MVAPICH + check_symbol_exists( + MVAPICH2_VERSION + mpi.h + HAVE_MVAPICH + ) + if (HAVE_MVAPICH) + set (MPI_IMPL_IS_MVAPICH TRUE CACHE BOOL "MVAPICH detected") + set (MPI_IMPL_ID "mvapich" CACHE STRING "MPI implementation identifier") + endif () +endif () - set(MPI_FOUND TRUE CACHE BOOL "Found the MPI library") - # Cray MPI identifies as MPICH - if ("${MPI_INCLUDE_PATH}" MATCHES "cray") +if (NOT DEFINED MPI_IMPL_ID) + # check for Cray MPI + # MPIX_PortName_Backlog is a Cray extension + # TODO: any better way to detect Cray MPI? + check_symbol_exists( + MPIX_PortName_Backlog + mpi.h + HAVE_CRAY_MPI + ) + if (HAVE_CRAY_MPI) set(MPI_IMPL_IS_CRAY TRUE CACHE BOOL "CrayMPI detected") set(MPI_IMPL_ID "craympi" CACHE STRING "MPI implementation identifier") - elseif (MPI_IMPL_NAME MATCHES "mpich") - set(MPI_IMPL_IS_MPICH TRUE CACHE BOOL "MPICH detected") - set(MPI_IMPL_ID "mpich" CACHE STRING "MPI implementation identifier") - elseif (MPI_IMPL_NAME MATCHES "mvapich") - set(MPI_IMPL_IS_MVAPICH TRUE CACHE BOOL "MVAPICH detected") - set(MPI_IMPL_ID "mvapich" CACHE STRING "MPI implementation identifier") - elseif (MPI_IMPL_NAME MATCHES "impi") - set(MPI_IMPL_IS_INTEL TRUE CACHE BOOL "IntelMPI detected") - set(MPI_IMPL_ID "intelmpi" CACHE STRING "MPI implementation identifier") - elseif (MPI_IMPL_NAME MATCHES "openmpi") - set(MPI_IMPL_IS_OPENMPI TRUE CACHE BOOL "OpenMPI detected") - set(MPI_IMPL_ID "openmpi" CACHE STRING "MPI implementation identifier") - # Open MPI before 1.10.7 and 2.1.1 failed to properly align memory - if (${MPI_IMPL_VERSION} VERSION_LESS 1.10.8 OR - ${MPI_IMPL_VERSION} VERSION_EQUAL 2.0.0 OR - (${MPI_IMPL_VERSION} VERSION_GREATER 2.0.0 AND ${MPI_IMPL_VERSION} VERSION_LESS 2.1.1)) - # disable shared memory windows due to alignment problems - set(ENABLE_SHARED_WINDOWS OFF) - message(WARNING - "Disabling shared memory window support due to defective allocation " - "in OpenMPI <2.1.1") - endif() - else () - message(FATAL_ERROR "Unknown MPI implementation detected: ${MPI_STRING}") - endif() -else (MPI_INCLUDE_PATH AND MPI_LIBRARY) - set(MPI_FOUND FALSE CACHE BOOL "Did not find the MPI library") -endif (MPI_INCLUDE_PATH AND MPI_LIBRARY) + endif () +endif () +if (NOT DEFINED MPI_IMPL_ID) + # check for MVAPICH + check_symbol_exists( + MPICH + mpi.h + HAVE_MPICH + ) + if (HAVE_MPICH) + set (MPI_IMPL_IS_MPICH TRUE CACHE BOOL "MPICH detected") + set (MPI_IMPL_ID "mpich" CACHE STRING "MPI implementation identifier") + endif () +endif () -# check for MPI-3 - -# save current state -cmake_push_check_state() -set(CMAKE_REQUIRED_INCLUDES ${MPI_INCLUDE_PATH}) -check_symbol_exists( - MPI_NO_OP - mpi.h - HAVE_MPI_NO_OP -) +# restore state cmake_pop_check_state() -if (NOT HAVE_MPI_NO_OP) +if (NOT DEFINED MPI_IMPL_ID) + set (MPI_IMPL_ID "UNKNOWN" CACHE STRING "MPI implementation identifier") +endif() + +message(STATUS "Detected MPI implementation: ${MPI_IMPL_ID}") + +# check for MPI-3 +try_compile(HAVE_MPI3 ${CMAKE_BINARY_DIR} + ${CMAKE_SOURCE_DIR}/CMakeExt/Code/test_mpi_support.c + OUTPUT_VARIABLE OUTPUT) + +if (NOT HAVE_MPI3) set(MPI_IS_DART_COMPATIBLE FALSE CACHE BOOL "MPI LIB has support for MPI-3") + unset (MPI_IMPL_ID CACHE) + message (WARNING + "Detected MPI implementation (${MPI_IMPL_ID}) does not support MPI-3") else() set(MPI_IS_DART_COMPATIBLE TRUE CACHE BOOL "MPI LIB has support for MPI-3") diff --git a/CMakeLists.txt b/CMakeLists.txt index 3081deaaf..6e6de6018 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -292,7 +292,7 @@ message(INFO "Enabled DART backends: (DART_IMPLEMENTATIONS) " ${DART_IMPLEMENTATIONS}) message(INFO "C compiler id: ${CMAKE_C_COMPILER_ID}") message(INFO "C++ compiler id: ${CMAKE_CXX_COMPILER_ID}") -if (MPI_FOUND) +if (DEFINED MPI_IMPL_ID) message(INFO "MPI implementation: " ${MPI_IMPL_ID}) endif() From 24033102836c4e3b1d5ad8ddfb4e68e396a52a94 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Fri, 19 May 2017 18:51:06 +0200 Subject: [PATCH 10/19] Use execute_process to compile MPI checks --- CMakeExt/MPI.cmake | 46 +++++++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/CMakeExt/MPI.cmake b/CMakeExt/MPI.cmake index c05895fff..8d403f642 100644 --- a/CMakeExt/MPI.cmake +++ b/CMakeExt/MPI.cmake @@ -10,12 +10,34 @@ if (NOT $ENV{MPI_CXX_COMPILER} STREQUAL "") CACHE STRING "MPI C++ compiler") endif() -if (CMAKE_CROSSCOMPILE) - message(STATUS "Cross compiling: ${CMAKE_CROSSCOMPILE}") -endif() - +# find MPI environment find_package(MPI) +# helper function that executes the MPI compiler +# on the given source file and sets +# resvar to true if the compile step succeeded +function (check_mpi_compile sourcefile resvar) + + # check for MPI-3 + get_filename_component(filename ${sourcefile} NAME) + execute_process( + COMMAND "${MPI_C_COMPILER}" -c "${sourcefile}" -o "${CMAKE_BINARY_DIR}/${filename}.o" + RESULT_VARIABLE RETURN_VAL + OUTPUT_VARIABLE OUTPUT + ERROR_VARIABLE OUTPUT + ) + + if (RETURN_VAL EQUAL 0) + set (${resvar} TRUE PARENT_SCOPE) + else () + set (${resvar} FALSE PARENT_SCOPE) + message (STATUS "Failed to execute MPI compiler: \n${OUTPUT}") + endif () + +endfunction () + + +# Determine MPI implementation # save current state cmake_push_check_state() @@ -30,10 +52,7 @@ check_symbol_exists( if (HAVE_OPEN_MPI) set (MPI_IMPL_IS_OPENMPI TRUE CACHE BOOL "OpenMPI detected") set (MPI_IMPL_ID "openmpi" CACHE STRING "MPI implementation identifier") - try_compile(OMPI_OK ${CMAKE_BINARY_DIR} - ${CMAKE_SOURCE_DIR}/CMakeExt/Code/test_compatible_ompi.c - CMAKE_FLAGS -DINCLUDE_DIRECTORIES=${MPI_INCLUDE_PATH} - OUTPUT_VARIABLE OUTPUT) + check_mpi_compile(${CMAKE_SOURCE_DIR}/CMakeExt/Code/test_compatible_ompi.c OMPI_OK) if (NOT OMPI_OK) message(WARNING "Disabling shared memory window support due to defective allocation " @@ -106,20 +125,21 @@ if (NOT DEFINED MPI_IMPL_ID) endif() message(STATUS "Detected MPI implementation: ${MPI_IMPL_ID}") +message(STATUS "Detected MPI C compiler: ${MPI_C_COMPILER}") -# check for MPI-3 -try_compile(HAVE_MPI3 ${CMAKE_BINARY_DIR} - ${CMAKE_SOURCE_DIR}/CMakeExt/Code/test_mpi_support.c - OUTPUT_VARIABLE OUTPUT) +check_mpi_compile(${CMAKE_SOURCE_DIR}/CMakeExt/Code/test_mpi_support.c HAVE_MPI3) if (NOT HAVE_MPI3) + message(${OUTPUT}) set(MPI_IS_DART_COMPATIBLE FALSE CACHE BOOL "MPI LIB has support for MPI-3") - unset (MPI_IMPL_ID CACHE) message (WARNING "Detected MPI implementation (${MPI_IMPL_ID}) does not support MPI-3") + message (STATUS "${OUTPUT}") + unset (MPI_IMPL_ID CACHE) else() set(MPI_IS_DART_COMPATIBLE TRUE CACHE BOOL "MPI LIB has support for MPI-3") endif() +set (CMAKE_C_COMPILER ${CMAKE_C_COMPILER_SAFE}) From 6e52b8c4816a46ec783fe7f1b52ff2375c4beb61 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Wed, 5 Jul 2017 17:13:56 +0200 Subject: [PATCH 11/19] Add support for detecting Cray MPI under MPT >=7.6.0 --- CMakeExt/MPI.cmake | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/CMakeExt/MPI.cmake b/CMakeExt/MPI.cmake index 8d403f642..c5492cc2d 100644 --- a/CMakeExt/MPI.cmake +++ b/CMakeExt/MPI.cmake @@ -91,13 +91,20 @@ endif () if (NOT DEFINED MPI_IMPL_ID) # check for Cray MPI - # MPIX_PortName_Backlog is a Cray extension - # TODO: any better way to detect Cray MPI? check_symbol_exists( - MPIX_PortName_Backlog + CRAY_MPICH_VERSION mpi.h HAVE_CRAY_MPI ) + if (NOT HAVE_CRAY_MPI) + # fall-back for versions prior to MPT 7.6.0 + # MPIX_PortName_Backlog is a Cray extension + check_symbol_exists( + MPIX_PortName_Backlog + mpi.h + HAVE_CRAY_MPI + ) + endif() if (HAVE_CRAY_MPI) set(MPI_IMPL_IS_CRAY TRUE CACHE BOOL "CrayMPI detected") set(MPI_IMPL_ID "craympi" CACHE STRING "MPI implementation identifier") From aadac4b33b41bef02e440199b27596c933584bf0 Mon Sep 17 00:00:00 2001 From: Bert Wesarg Date: Fri, 2 Mar 2018 09:43:41 +0100 Subject: [PATCH 12/19] Move content of `.gitignore` to `.gitmodules`. Git does not read any full fledged config files from the repository, in particular any `.gitconfig`. Thus this file is useless, without a `include.path` statement in any config file Git is reading (i.e., `.git/config`). Though it reads `.gitmodules` and as the content of `.gitconfig` is sub-modules related anyway, it has an effect, if it is in this file. Do so. --- .gitconfig | 2 -- .gitmodules | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) delete mode 100644 .gitconfig diff --git a/.gitconfig b/.gitconfig deleted file mode 100644 index 4b282ebb2..000000000 --- a/.gitconfig +++ /dev/null @@ -1,2 +0,0 @@ -[submodule "vendor/googletest"] - ignore = all diff --git a/.gitmodules b/.gitmodules index 7ea6cfccd..ce0d15335 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "vendor/googletest"] path = vendor/googletest url = https://github.com/google/googletest.git + ignore = all From 0d403246220014e410793c83fa19cf4447f93141 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Fri, 2 Mar 2018 10:43:05 +0100 Subject: [PATCH 13/19] Fix support for 64bit integer types on 32bit systems --- dart-if/include/dash/dart/if/dart_types.h | 47 +++++------------------ dart-impl/mpi/src/dart_mpi_types.c | 2 + dash/include/dash/Types.h | 20 +++++++--- 3 files changed, 26 insertions(+), 43 deletions(-) diff --git a/dart-if/include/dash/dart/if/dart_types.h b/dart-if/include/dash/dart/if/dart_types.h index c380d18ce..bbfea04a7 100644 --- a/dart-if/include/dash/dart/if/dart_types.h +++ b/dart-if/include/dash/dart/if/dart_types.h @@ -96,42 +96,10 @@ typedef enum * * \ingroup DartTypes */ -/** -typedef enum -{ - DART_TYPE_UNDEFINED = 0, - /// integral data types - DART_TYPE_BYTE, - DART_TYPE_SHORT, - DART_TYPE_INT, - DART_TYPE_UINT, - DART_TYPE_LONG, - DART_TYPE_ULONG, - DART_TYPE_LONGLONG, - /// floating point data types - DART_TYPE_FLOAT, - DART_TYPE_DOUBLE, - /// Reserved, do not use! - DART_TYPE_COUNT -} dart_datatype_t; -**/ - typedef intptr_t dart_datatype_t; -#if 0 -extern struct dart_datatype_struct __dart_type_undefined_t; -extern struct dart_datatype_struct __dart_type_byte_t; -extern struct dart_datatype_struct __dart_type_short_t; -extern struct dart_datatype_struct __dart_type_int_t; -extern struct dart_datatype_struct __dart_type_uint_t; -extern struct dart_datatype_struct __dart_type_long_t; -extern struct dart_datatype_struct __dart_type_ulong_t; -extern struct dart_datatype_struct __dart_type_longlong_t; -extern struct dart_datatype_struct __dart_type_float_t; -extern struct dart_datatype_struct __dart_type_double_t; -#endif - #define DART_TYPE_UNDEFINED (dart_datatype_t)(0) +/// integral data types #define DART_TYPE_BYTE (dart_datatype_t)(1) #define DART_TYPE_SHORT (dart_datatype_t)(2) #define DART_TYPE_INT (dart_datatype_t)(3) @@ -139,16 +107,19 @@ extern struct dart_datatype_struct __dart_type_double_t; #define DART_TYPE_LONG (dart_datatype_t)(5) #define DART_TYPE_ULONG (dart_datatype_t)(6) #define DART_TYPE_LONGLONG (dart_datatype_t)(7) -#define DART_TYPE_FLOAT (dart_datatype_t)(8) -#define DART_TYPE_DOUBLE (dart_datatype_t)(9) -#define DART_TYPE_LAST (dart_datatype_t)(10) +#define DART_TYPE_ULONGLONG (dart_datatype_t)(8) +/// floating point data types +#define DART_TYPE_FLOAT (dart_datatype_t)(9) +#define DART_TYPE_DOUBLE (dart_datatype_t)(10) +/// Reserved, do not use! +#define DART_TYPE_LAST (dart_datatype_t)(11) /** size for integral \c size_t */ #if (UINT32_MAX == SIZE_MAX) -# define DART_TYPE_SIZET DART_TYPE_UINT +# define DART_TYPE_SIZET DART_TYPE_ULONG #elif (UINT64_MAX == SIZE_MAX) -# define DART_TYPE_SIZET DART_TYPE_LONGLONG +# define DART_TYPE_SIZET DART_TYPE_ULONGLONG #else # error "Cannot determine DART type for size_t!" #endif diff --git a/dart-impl/mpi/src/dart_mpi_types.c b/dart-impl/mpi/src/dart_mpi_types.c index c7f22a1b8..0fd879e64 100644 --- a/dart-impl/mpi/src/dart_mpi_types.c +++ b/dart-impl/mpi/src/dart_mpi_types.c @@ -27,6 +27,7 @@ static const char* __dart_base_type_names[DART_TYPE_LAST+1] = { "LONG", "UNSIGNED LONG", "LONG LONG", + "UNSIGNED LONG LONG", "FLOAT", "DOUBLE", "INVALID" @@ -96,6 +97,7 @@ dart__mpi__datatype_init() init_basic_datatype(DART_TYPE_LONG, MPI_LONG); init_basic_datatype(DART_TYPE_ULONG, MPI_UNSIGNED_LONG); init_basic_datatype(DART_TYPE_LONGLONG, MPI_LONG_LONG); + init_basic_datatype(DART_TYPE_ULONGLONG, MPI_UNSIGNED_LONG_LONG); init_basic_datatype(DART_TYPE_FLOAT, MPI_FLOAT); init_basic_datatype(DART_TYPE_DOUBLE, MPI_DOUBLE); diff --git a/dash/include/dash/Types.h b/dash/include/dash/Types.h index 33d1d0a69..ce92ceabc 100644 --- a/dash/include/dash/Types.h +++ b/dash/include/dash/Types.h @@ -139,11 +139,6 @@ struct dart_datatype { static constexpr const dart_datatype_t value = DART_TYPE_UINT; }; -template<> -struct dart_datatype { - static constexpr const dart_datatype_t value = DART_TYPE_FLOAT; -}; - template<> struct dart_datatype { static constexpr const dart_datatype_t value = DART_TYPE_LONG; @@ -154,6 +149,21 @@ struct dart_datatype { static constexpr const dart_datatype_t value = DART_TYPE_ULONG; }; +template<> +struct dart_datatype { + static constexpr const dart_datatype_t value = DART_TYPE_LONGLONG; +}; + +template<> +struct dart_datatype { + static constexpr const dart_datatype_t value = DART_TYPE_ULONGLONG; +}; + +template<> +struct dart_datatype { + static constexpr const dart_datatype_t value = DART_TYPE_FLOAT; +}; + template<> struct dart_datatype { static constexpr const dart_datatype_t value = DART_TYPE_DOUBLE; From 609253d7b99ac9cbf31fa83822ffe9b58d330e1c Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Mon, 5 Mar 2018 11:30:40 +0100 Subject: [PATCH 14/19] Query MPI from dart-mpi and avoid repeated checking --- CMakeExt/MPI.cmake | 264 ++++++++++++++++++----------------- dart-impl/mpi/CMakeLists.txt | 2 + dash/CMakeLists.txt | 1 + 3 files changed, 137 insertions(+), 130 deletions(-) diff --git a/CMakeExt/MPI.cmake b/CMakeExt/MPI.cmake index c5492cc2d..c1e4a33e5 100644 --- a/CMakeExt/MPI.cmake +++ b/CMakeExt/MPI.cmake @@ -1,152 +1,156 @@ INCLUDE (CheckSymbolExists) INCLUDE (CMakePushCheckState) -if (NOT $ENV{MPI_C_COMPILER} STREQUAL "") - set(MPI_C_COMPILER $ENV{MPI_C_COMPILER} - CACHE STRING "MPI C compiler") -endif() -if (NOT $ENV{MPI_CXX_COMPILER} STREQUAL "") - set(MPI_CXX_COMPILER $ENV{MPI_CXX_COMPILER} - CACHE STRING "MPI C++ compiler") -endif() - -# find MPI environment -find_package(MPI) - -# helper function that executes the MPI compiler -# on the given source file and sets -# resvar to true if the compile step succeeded -function (check_mpi_compile sourcefile resvar) - - # check for MPI-3 - get_filename_component(filename ${sourcefile} NAME) - execute_process( - COMMAND "${MPI_C_COMPILER}" -c "${sourcefile}" -o "${CMAKE_BINARY_DIR}/${filename}.o" - RESULT_VARIABLE RETURN_VAL - OUTPUT_VARIABLE OUTPUT - ERROR_VARIABLE OUTPUT - ) - - if (RETURN_VAL EQUAL 0) - set (${resvar} TRUE PARENT_SCOPE) - else () - set (${resvar} FALSE PARENT_SCOPE) - message (STATUS "Failed to execute MPI compiler: \n${OUTPUT}") - endif () +if (NOT DEFINED MPI_IMPL_ID) -endfunction () - - -# Determine MPI implementation - -# save current state -cmake_push_check_state() -set (CMAKE_REQUIRED_INCLUDES ${MPI_INCLUDE_PATH}) - -# check for Open MPI -check_symbol_exists( - OMPI_MAJOR_VERSION - mpi.h - HAVE_OPEN_MPI -) -if (HAVE_OPEN_MPI) - set (MPI_IMPL_IS_OPENMPI TRUE CACHE BOOL "OpenMPI detected") - set (MPI_IMPL_ID "openmpi" CACHE STRING "MPI implementation identifier") - check_mpi_compile(${CMAKE_SOURCE_DIR}/CMakeExt/Code/test_compatible_ompi.c OMPI_OK) - if (NOT OMPI_OK) - message(WARNING - "Disabling shared memory window support due to defective allocation " - "in OpenMPI <2.1.1") - set (ENABLE_SHARED_WINDOWS OFF) + if (NOT $ENV{MPI_C_COMPILER} STREQUAL "") + set(MPI_C_COMPILER $ENV{MPI_C_COMPILER} + CACHE STRING "MPI C compiler") + endif() + if (NOT $ENV{MPI_CXX_COMPILER} STREQUAL "") + set(MPI_CXX_COMPILER $ENV{MPI_CXX_COMPILER} + CACHE STRING "MPI C++ compiler") endif() -endif() -# order matters: all of the following -# implementations also define MPICH -if (NOT DEFINED MPI_IMPL_ID) - # check for Intel MPI + # find MPI environment + find_package(MPI) + + # helper function that executes the MPI compiler + # on the given source file and sets + # resvar to true if the compile step succeeded + function (check_mpi_compile sourcefile resvar) + + # check for MPI-3 + get_filename_component(filename ${sourcefile} NAME) + execute_process( + COMMAND "${MPI_C_COMPILER}" -c "${sourcefile}" -o "${CMAKE_BINARY_DIR}/${filename}.o" + RESULT_VARIABLE RETURN_VAL + OUTPUT_VARIABLE OUTPUT + ERROR_VARIABLE OUTPUT + ) + + if (RETURN_VAL EQUAL 0) + set (${resvar} TRUE PARENT_SCOPE) + else () + set (${resvar} FALSE PARENT_SCOPE) + message (STATUS "Failed to execute MPI compiler: \n${OUTPUT}") + endif () + + endfunction () + + + # Determine MPI implementation + + # save current state + cmake_push_check_state() + set (CMAKE_REQUIRED_INCLUDES ${MPI_INCLUDE_PATH}) + + # check for Open MPI check_symbol_exists( - I_MPI_VERSION + OMPI_MAJOR_VERSION mpi.h - HAVE_I_MPI + HAVE_OPEN_MPI ) - if (HAVE_I_MPI) - set (MPI_IMPL_IS_INTEL TRUE CACHE BOOL "IntelMPI detected") - set (MPI_IMPL_ID "intelmpi" CACHE STRING "MPI implementation identifier") + if (HAVE_OPEN_MPI) + set (MPI_IMPL_IS_OPENMPI TRUE CACHE BOOL "OpenMPI detected") + set (MPI_IMPL_ID "openmpi" CACHE STRING "MPI implementation identifier") + check_mpi_compile(${CMAKE_SOURCE_DIR}/CMakeExt/Code/test_compatible_ompi.c OMPI_OK) + if (NOT OMPI_OK) + message(WARNING + "Disabling shared memory window support due to defective allocation " + "in OpenMPI <2.1.1") + set (ENABLE_SHARED_WINDOWS OFF) + endif() + endif() + + # order matters: all of the following + # implementations also define MPICH + if (NOT DEFINED MPI_IMPL_ID) + # check for Intel MPI + check_symbol_exists( + I_MPI_VERSION + mpi.h + HAVE_I_MPI + ) + if (HAVE_I_MPI) + set (MPI_IMPL_IS_INTEL TRUE CACHE BOOL "IntelMPI detected") + set (MPI_IMPL_ID "intelmpi" CACHE STRING "MPI implementation identifier") + endif () endif () -endif () -if (NOT DEFINED MPI_IMPL_ID) - # check for MVAPICH - check_symbol_exists( - MVAPICH2_VERSION - mpi.h - HAVE_MVAPICH - ) - if (HAVE_MVAPICH) - set (MPI_IMPL_IS_MVAPICH TRUE CACHE BOOL "MVAPICH detected") - set (MPI_IMPL_ID "mvapich" CACHE STRING "MPI implementation identifier") + if (NOT DEFINED MPI_IMPL_ID) + # check for MVAPICH + check_symbol_exists( + MVAPICH2_VERSION + mpi.h + HAVE_MVAPICH + ) + if (HAVE_MVAPICH) + set (MPI_IMPL_IS_MVAPICH TRUE CACHE BOOL "MVAPICH detected") + set (MPI_IMPL_ID "mvapich" CACHE STRING "MPI implementation identifier") + endif () endif () -endif () -if (NOT DEFINED MPI_IMPL_ID) - # check for Cray MPI - check_symbol_exists( - CRAY_MPICH_VERSION - mpi.h - HAVE_CRAY_MPI - ) - if (NOT HAVE_CRAY_MPI) - # fall-back for versions prior to MPT 7.6.0 - # MPIX_PortName_Backlog is a Cray extension + if (NOT DEFINED MPI_IMPL_ID) + # check for Cray MPI check_symbol_exists( - MPIX_PortName_Backlog + CRAY_MPICH_VERSION mpi.h HAVE_CRAY_MPI ) - endif() - if (HAVE_CRAY_MPI) - set(MPI_IMPL_IS_CRAY TRUE CACHE BOOL "CrayMPI detected") - set(MPI_IMPL_ID "craympi" CACHE STRING "MPI implementation identifier") + if (NOT HAVE_CRAY_MPI) + # fall-back for versions prior to MPT 7.6.0 + # MPIX_PortName_Backlog is a Cray extension + check_symbol_exists( + MPIX_PortName_Backlog + mpi.h + HAVE_CRAY_MPI + ) + endif() + if (HAVE_CRAY_MPI) + set(MPI_IMPL_IS_CRAY TRUE CACHE BOOL "CrayMPI detected") + set(MPI_IMPL_ID "craympi" CACHE STRING "MPI implementation identifier") + endif () endif () -endif () -if (NOT DEFINED MPI_IMPL_ID) - # check for MVAPICH - check_symbol_exists( - MPICH - mpi.h - HAVE_MPICH - ) - if (HAVE_MPICH) - set (MPI_IMPL_IS_MPICH TRUE CACHE BOOL "MPICH detected") - set (MPI_IMPL_ID "mpich" CACHE STRING "MPI implementation identifier") + if (NOT DEFINED MPI_IMPL_ID) + # check for MVAPICH + check_symbol_exists( + MPICH + mpi.h + HAVE_MPICH + ) + if (HAVE_MPICH) + set (MPI_IMPL_IS_MPICH TRUE CACHE BOOL "MPICH detected") + set (MPI_IMPL_ID "mpich" CACHE STRING "MPI implementation identifier") + endif () endif () -endif () -# restore state -cmake_pop_check_state() + # restore state + cmake_pop_check_state() -if (NOT DEFINED MPI_IMPL_ID) - set (MPI_IMPL_ID "UNKNOWN" CACHE STRING "MPI implementation identifier") -endif() - -message(STATUS "Detected MPI implementation: ${MPI_IMPL_ID}") -message(STATUS "Detected MPI C compiler: ${MPI_C_COMPILER}") - -check_mpi_compile(${CMAKE_SOURCE_DIR}/CMakeExt/Code/test_mpi_support.c HAVE_MPI3) - -if (NOT HAVE_MPI3) - message(${OUTPUT}) - set(MPI_IS_DART_COMPATIBLE FALSE CACHE BOOL - "MPI LIB has support for MPI-3") - message (WARNING - "Detected MPI implementation (${MPI_IMPL_ID}) does not support MPI-3") - message (STATUS "${OUTPUT}") - unset (MPI_IMPL_ID CACHE) -else() - set(MPI_IS_DART_COMPATIBLE TRUE CACHE BOOL - "MPI LIB has support for MPI-3") -endif() - -set (CMAKE_C_COMPILER ${CMAKE_C_COMPILER_SAFE}) + if (NOT DEFINED MPI_IMPL_ID) + set (MPI_IMPL_ID "UNKNOWN" CACHE STRING "MPI implementation identifier") + endif() + + message(STATUS "Detected MPI implementation: ${MPI_IMPL_ID}") + message(STATUS "Detected MPI C compiler: ${MPI_C_COMPILER}") + + check_mpi_compile(${CMAKE_SOURCE_DIR}/CMakeExt/Code/test_mpi_support.c HAVE_MPI3) + + if (NOT HAVE_MPI3) + message(${OUTPUT}) + set(MPI_IS_DART_COMPATIBLE FALSE CACHE BOOL + "MPI LIB has support for MPI-3") + message (WARNING + "Detected MPI implementation (${MPI_IMPL_ID}) does not support MPI-3") + message (STATUS "${OUTPUT}") + unset (MPI_IMPL_ID CACHE) + else() + set(MPI_IS_DART_COMPATIBLE TRUE CACHE BOOL + "MPI LIB has support for MPI-3") + endif() + + set (CMAKE_C_COMPILER ${CMAKE_C_COMPILER_SAFE}) + +endif(NOT DEFINED MPI_IMPL_ID) diff --git a/dart-impl/mpi/CMakeLists.txt b/dart-impl/mpi/CMakeLists.txt index fdf845f67..bcb8e3977 100644 --- a/dart-impl/mpi/CMakeLists.txt +++ b/dart-impl/mpi/CMakeLists.txt @@ -6,6 +6,8 @@ set(DASH_DART_IMPL_MPI_LIBRARY dart-mpi) set(DASH_DART_BASE_LIBRARY dart-base) +include(${CMAKE_SOURCE_DIR}/CMakeExt/MPI.cmake) + if (MPI_NOTFOUND) message(FATAL_ERROR, "No MPI implementation found for dart-mpi") endif() diff --git a/dash/CMakeLists.txt b/dash/CMakeLists.txt index d0dd57d8d..3f52ac635 100644 --- a/dash/CMakeLists.txt +++ b/dash/CMakeLists.txt @@ -299,6 +299,7 @@ foreach (dart_variant ${DART_IMPLEMENTATIONS_LIST}) ${ADDITIONAL_LINK_FLAGS} ${ADDITIONAL_LIBRARIES} ) + set_target_properties( ${DASH_LIBRARY} PROPERTIES POSITION_INDEPENDENT_CODE TRUE From 231f63c97eb4dd70f01d363d0c374bc76553821e Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Mon, 5 Mar 2018 13:33:28 +0100 Subject: [PATCH 15/19] Fix dash::Array move assignment --- dash/include/dash/Array.h | 53 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 51 insertions(+), 2 deletions(-) diff --git a/dash/include/dash/Array.h b/dash/include/dash/Array.h index 1c9813ca9..49ae85071 100644 --- a/dash/include/dash/Array.h +++ b/dash/include/dash/Array.h @@ -902,7 +902,29 @@ class Array * The pattern has to be movable or copyable * The underlying memory does not have to be movable (it might). */ - Array(self_t && other) = default; + Array(self_t && other) + : local(this), + async(this), + m_team(other.m_team), + m_myid(other.m_myid), + m_pattern(std::move(other.m_pattern)), + m_globmem(std::move(other.m_globmem)), + m_begin(other.m_begin), + m_end(other.m_end), + m_size(other.m_size), + m_lsize(other.m_lsize), + m_lcapacity(other.m_lcapacity), + m_lbegin(other.m_lbegin), + m_lend(other.m_lend) { + + other.m_globmem = nullptr; + other.m_lbegin = nullptr; + other.m_lend = nullptr; + // Register deallocator of this array instance at the team + // instance that has been used to initialized it: + m_team->register_deallocator( + this, std::bind(&Array::deallocate, this)); + } /** * Assignment operator is deleted to prevent unintentional copies of @@ -931,7 +953,34 @@ class Array * The pattern has to be movable or copyable * The underlying memory does not have to be movable (it might). */ - self_t & operator=(self_t && other) = default; + self_t & operator=(self_t && other) { + + if (this->m_globmem != nullptr) + deallocate(); + + this->m_begin = other.m_begin; + this->m_end = other.m_end; + this->m_globmem = std::move(other.m_globmem); + this->m_lbegin = other.m_lbegin; + this->m_lcapacity = other.m_lcapacity; + this->m_lend = other.m_lend; + this->m_lsize = other.m_lsize; + this->m_myid = other.m_myid; + this->m_pattern = std::move(other.m_pattern); + this->m_size = other.m_size; + this->m_team = other.m_team; + + other.m_globmem = nullptr; + other.m_lbegin = nullptr; + other.m_lend = nullptr; + + // Re-register deallocator of this array instance at the team + // instance that has been used to initialized it: + if (this->m_globmem != nullptr) { + m_team->register_deallocator( + this, std::bind(&Array::deallocate, this)); + } + } /** * Destructor, deallocates array elements. From e928d312259914bdc384b2ccb75b48ee3f78f329 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Mon, 5 Mar 2018 13:34:03 +0100 Subject: [PATCH 16/19] Fix dash::Matrix move assignment / ctor --- .../include/dash/matrix/internal/Matrix-inl.h | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/dash/include/dash/matrix/internal/Matrix-inl.h b/dash/include/dash/matrix/internal/Matrix-inl.h index a197c0aa9..f394a07cc 100644 --- a/dash/include/dash/matrix/internal/Matrix-inl.h +++ b/dash/include/dash/matrix/internal/Matrix-inl.h @@ -92,11 +92,15 @@ ::Matrix( _lend(other._lend), _ref(other._ref) { - // do not free other globmem - other._glob_mem = nullptr; - other._lbegin = nullptr; - other._lend = nullptr; - DASH_LOG_TRACE("Matrix()", "Move-Constructed"); + // do not free other globmem + other._glob_mem = nullptr; + other._lbegin = nullptr; + other._lend = nullptr; + + // Register team deallocator: + _team->register_deallocator( + this, std::bind(&Matrix::deallocate, this)); + DASH_LOG_TRACE("Matrix()", "Move-Constructed"); } template @@ -113,7 +117,9 @@ Matrix ::operator= ( Matrix && other) { - deallocate(); + if (_glob_mem != nullptr) + deallocate(); + _team = other._team; _size = other._size; _lcapacity = other._lcapacity; @@ -124,6 +130,12 @@ ::operator= ( _lend = other._lend; _ref = other._ref; + // Re-register team deallocator: + if (_glob_mem != nullptr) { + _team->register_deallocator( + this, std::bind(&Matrix::deallocate, this)); + } + // do not free other globmem other._glob_mem = nullptr; other._lbegin = nullptr; From 1653e064c0384f43a3d4b3be64abf5b834882648 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Mon, 5 Mar 2018 14:48:26 +0100 Subject: [PATCH 17/19] Fix missing return value in dash::Array::operator= --- dash/include/dash/Array.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dash/include/dash/Array.h b/dash/include/dash/Array.h index 49ae85071..4fb3bb5ac 100644 --- a/dash/include/dash/Array.h +++ b/dash/include/dash/Array.h @@ -955,6 +955,8 @@ class Array */ self_t & operator=(self_t && other) { + if (this == &other) return *this; + if (this->m_globmem != nullptr) deallocate(); @@ -980,6 +982,8 @@ class Array m_team->register_deallocator( this, std::bind(&Array::deallocate, this)); } + + return *this; } /** From e01baf7f44a22167d907c6bde5f3e48e914fc7eb Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Mon, 5 Mar 2018 15:14:54 +0100 Subject: [PATCH 18/19] Fix setting the C++ standard in the DASH compiler wrapper --- CMakeExt/CompilerFlags.cmake | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/CMakeExt/CompilerFlags.cmake b/CMakeExt/CompilerFlags.cmake index 54b51a36f..3ae2c0165 100644 --- a/CMakeExt/CompilerFlags.cmake +++ b/CMakeExt/CompilerFlags.cmake @@ -211,19 +211,20 @@ elseif ("${CMAKE_C_COMPILER_ID}" MATCHES "Cray") CACHE STRING "C compiler std flag") endif() -if(${CMAKE_VERSION} VERSION_GREATER 3.0.0 ) +if(${CMAKE_VERSION} VERSION_LESS 3.0.0 ) # clear STD flags as set using CXX_STANDARD - set(CXX_STD_FLAG "") - set(CC_STD_FLAG "") + set(CMAKE_CXX_FLAGS + "${CMAKE_CXX_FLAGS} ${CXX_STD_FLAG}") + set(CMAKE_C_FLAGS + "${CMAKE_C_FLAGS} ${CC_STD_FLAG}") endif() set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CC_ENV_SETUP_FLAGS}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CXX_ENV_SETUP_FLAGS}") - set(CMAKE_C_FLAGS - "${CMAKE_C_FLAGS} ${CC_STD_FLAG} ${CC_OMP_FLAG}") + "${CMAKE_C_FLAGS} ${CC_OMP_FLAG}") set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${CC_REPORT_FLAG} ${CC_WARN_FLAG}") set(CMAKE_C_FLAGS_DEBUG @@ -236,7 +237,7 @@ set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -Ofast -DDASH_RELEASE") set(CMAKE_CXX_FLAGS - "${CMAKE_CXX_FLAGS} ${CXX_STD_FLAG} ${CXX_OMP_FLAG}") + "${CMAKE_CXX_FLAGS} ${CXX_OMP_FLAG}") set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CC_REPORT_FLAG} ${CXX_WARN_FLAG}") set(CMAKE_CXX_FLAGS_DEBUG From 4ddd5e4a9037bee36a045395e7defcfdd676f365 Mon Sep 17 00:00:00 2001 From: Joseph Schuchart Date: Mon, 5 Mar 2018 16:57:43 +0100 Subject: [PATCH 19/19] Array: Only deregister deallocator if it has previously been registered --- dash/include/dash/Array.h | 31 +++++++++++++------------------ dash/test/container/ArrayTest.cc | 5 +++++ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/dash/include/dash/Array.h b/dash/include/dash/Array.h index 4fb3bb5ac..5211ecbbd 100644 --- a/dash/include/dash/Array.h +++ b/dash/include/dash/Array.h @@ -718,8 +718,6 @@ class Array private: /// Team containing all units interacting with the array dash::Team * m_team = nullptr; - /// DART id of the unit that created the array - team_unit_t m_myid; /// Element distribution pattern PatternType m_pattern; /// Global memory allocation and -access @@ -738,18 +736,10 @@ class Array ElementType * m_lbegin = nullptr; /// Native pointer past last local element in the array ElementType * m_lend = nullptr; - -public: -/* - Check requirements on element type - is_trivially_copyable is not implemented presently, and is_trivial - is too strict (e.g. fails on std::pair). - - static_assert(std::is_trivially_copyable::value, - "Element type must be trivially copyable"); - static_assert(std::is_trivial::value, - "Element type must be trivially copyable"); -*/ + /// DART id of the unit that created the array + team_unit_t m_myid; + /// Whether or not the array was actually allocated + bool m_registered = false; public: /** @@ -924,6 +914,7 @@ class Array // instance that has been used to initialized it: m_team->register_deallocator( this, std::bind(&Array::deallocate, this)); + m_registered = true; } /** @@ -957,8 +948,7 @@ class Array if (this == &other) return *this; - if (this->m_globmem != nullptr) - deallocate(); + deallocate(); this->m_begin = other.m_begin; this->m_end = other.m_end; @@ -981,6 +971,7 @@ class Array if (this->m_globmem != nullptr) { m_team->register_deallocator( this, std::bind(&Array::deallocate, this)); + m_registered = true; } return *this; @@ -1383,8 +1374,10 @@ class Array } // Remove this function from team deallocator list to avoid // double-free: - m_team->unregister_deallocator( - this, std::bind(&Array::deallocate, this)); + if (m_registered) { + m_team->unregister_deallocator( + this, std::bind(&Array::deallocate, this)); + } // Actual destruction of the array instance: DASH_LOG_TRACE_VAR("Array.deallocate()", m_globmem.get()); if (m_globmem != nullptr) { @@ -1434,6 +1427,7 @@ class Array // instance that has been used to initialized it: m_team->register_deallocator( this, std::bind(&Array::deallocate, this)); + m_registered = true; // Assure all units are synchronized after allocation, otherwise // other units might start working on the array before allocation // completed at all units: @@ -1484,6 +1478,7 @@ class Array // instance that has been used to initialized it: m_team->register_deallocator( this, std::bind(&Array::deallocate, this)); + m_registered = true; // Assure all units are synchronized after allocation, otherwise // other units might start working on the array before allocation // completed at all units: diff --git a/dash/test/container/ArrayTest.cc b/dash/test/container/ArrayTest.cc index 825d52103..c96207b1e 100644 --- a/dash/test/container/ArrayTest.cc +++ b/dash/test/container/ArrayTest.cc @@ -17,6 +17,11 @@ dash::Array array_global; TEST_F(ArrayTest, Declaration) +{ + dash::Array arr; +} + +TEST_F(ArrayTest, Initialization) { dash::Array array_local(19 * dash::size(), dash::BLOCKED); }