Merge branch 'develop' into feature/guthrey1/venado

.clang-format

@@ -1,27 +1,84 @@
-BasedOnStyle : google
+BasedOnStyle : LLVM
+# Indent formatting
 IndentWidth : 2
-BreakBeforeBraces : Linux
+Language: Cpp
+UseTab: Never
 KeepEmptyLinesAtTheStartOfBlocks : true
 MaxEmptyLinesToKeep : 2
 AccessModifierOffset : -2
-UseTab: Never
+# This must be off so that include order in RAJA is preserved
+SortIncludes: false
+
+# Alignment of consecutive declarations, assignments etc
+AlignConsecutiveAssignments : true
+AlignConsecutiveDeclarations : false
+AlignConsecutiveMacros : true
+AlignTrailingComments : true
+
+# Control curly brace placement
+BreakBeforeBraces : Custom
+BraceWrapping:
+  AfterCaseLabel:  true
+  AfterClass:      true
+  AfterControlStatement: true
+  AfterEnum:       true
+  AfterFunction:   true
+  AfterNamespace:  true
+  AfterObjCDeclaration: false
+  AfterStruct:     true
+  AfterUnion:      true
+  AfterExternBlock: false
+  BeforeCatch:     true
+  BeforeElse:      true
+  BeforeLambdaBody: false
+  IndentBraces:    false
+  SplitEmptyFunction: false
+  SplitEmptyRecord: false
+  SplitEmptyNamespace: false
+
+# Pointer alignment
+DerivePointerAlignment: false
+PointerAlignment: Left
+
+# Single line config
 AllowShortIfStatementsOnASingleLine : true
-ConstructorInitializerAllOnOneLineOrOnePerLine : true
 AllowShortFunctionsOnASingleLine : true
 AllowShortLoopsOnASingleLine : false
-BinPackParameters : false
+AllowShortLambdasOnASingleLine : None
+PackConstructorInitializers : CurrentLine
+AllowAllArgumentsOnNextLine : true
 AllowAllParametersOfDeclarationOnNextLine : false
-AlignTrailingComments : true
+BinPackArguments : true
+BinPackParameters : false
+ConstructorInitializerAllOnOneLineOrOnePerLine : true
 ColumnLimit : 80
-PenaltyBreakBeforeFirstCallParameter : 100
-PenaltyReturnTypeOnItsOwnLine : 65000
-PenaltyBreakString : 10
 
-# These improve formatting results but require clang 3.6/7 or higher
-BreakBeforeBinaryOperators : None
-AlignAfterOpenBracket: true
-BinPackArguments : false
+AlignAfterOpenBracket: Align
 AlignOperands : true
 AlwaysBreakTemplateDeclarations : true
-Cpp11BracedListStyle : true
+AlwaysBreakAfterDefinitionReturnType : None
+PenaltyReturnTypeOnItsOwnLine : 10000
+BreakBeforeBinaryOperators : None
+
+# Indents
+IndentCaseLabels: true
+
+# Lambda body
+LambdaBodyIndentation : Signature
+
+SeparateDefinitionBlocks : Always
 
+# Space before/after settings
+SpaceAfterTemplateKeyword: false
+SpaceBeforeCpp11BracedList: true
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyBlock: false
+SpacesBeforeTrailingComments: 2
+SpacesInAngles: false
+SpacesInCStyleCastParentheses: false
+SpacesInContainerLiterals: false
+SpacesInConditionalStatement: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false

.gitlab/jobs/lassen.yml

@@ -60,12 +60,8 @@ gcc_8_3_1_cuda_11_7_desul_atomics:
     MODULE_LIST: "cuda/11.7.0"
   extends: .job_on_lassen
 
-# Warning: Allowed to fail temporarily
-# Deactivated due to issues with OpenMP Target and various tests and compilers.
-clang_16_0_6_ibm_omptarget:
+clang_16_0_6_omptarget:
   variables:
-    SPEC: " ~shared +openmp +omptarget +tests %clang@=16.0.6.ibm.gcc.8.3.1"
-    ON_LASSEN: "OFF"
+    SPEC: " ~shared +openmp +omptarget +tests %clang@=16.0.6.cuda.11.8.0.gcc.11.2.1"
   extends: .job_on_lassen
-  allow_failure: true
 

CMakeLists.txt

@@ -41,7 +41,7 @@ project(RAJA LANGUAGES CXX C
   VERSION ${RAJA_LOADED})
 
 set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}/cmake/thirdparty" ${CMAKE_MODULE_PATH})
-
+set(BLT_REQUIRED_CLANGFORMAT_VERSION  "14" CACHE STRING "")
 include(cmake/SetupRajaOptions.cmake)
 
 cmake_minimum_required(VERSION 3.23)
@@ -136,6 +136,9 @@ include(cmake/SetupCompilers.cmake)
 # Macros for building executables and libraries
 include (cmake/RAJAMacros.cmake)
 
+# Configure `style` target for enforcing code style
+raja_add_code_checks()
+
 set (raja_sources
   src/AlignedRangeIndexSetBuilders.cpp
   src/DepGraphNode.cpp
@@ -171,6 +174,7 @@ endif()
 
 if(RAJA_ENABLE_HIP)
   message(STATUS "HIP version: ${hip_VERSION}")
+  set(RAJA_HIP_WAVESIZE "64" CACHE STRING "Set the wave size for GPU architecture. E.g. MI200/MI300 this is 64.")
   if("${hip_VERSION}" VERSION_LESS "3.5")
     message(FATAL_ERROR "Trying to use HIP/ROCm version ${hip_VERSION}. RAJA requires HIP/ROCm version 3.5 or newer. ")
   endif()

cmake/RAJAMacros.cmake

@@ -204,3 +204,62 @@ macro(raja_add_benchmark)
     NUM_OMP_THREADS ${arg_NUM_OMP_THREADS}
     COMMAND ${TEST_DRIVER} ${arg_NAME})
 endmacro(raja_add_benchmark)
+
+##------------------------------------------------------------------------------
+## raja_add_code_checks()
+##
+## Adds code checks for all source files recursively in the RAJA repository.
+##
+## This creates the following parent build targets:
+##  check - Runs a non file changing style check and CppCheck
+##  style - In-place code formatting
+##
+## Creates various child build targets that follow this pattern:
+##  raja_<check|style>
+##  raja_<cppcheck|clangformat>_<check|style>
+##------------------------------------------------------------------------------
+macro(raja_add_code_checks)
+
+  set(options)
+  set(singleValueArgs)
+  set(multiValueArgs)
+
+  # Parse the arguments to the macro
+  cmake_parse_arguments(arg
+       "${options}" "${singleValueArgs}" "${multiValueArgs}" ${ARGN})
+
+  # Only do code checks if building raja by itself and not included in
+  # another project
+  if ("${PROJECT_SOURCE_DIR}" STREQUAL "${CMAKE_SOURCE_DIR}")
+      # Create file globbing expressions that only include directories that contain source
+      # TODO(bowen) Add examples, exercises, test, and benchmark to the list below
+      set(_base_dirs "RAJA" "include" "src")
+      set(_ext_expressions "*.cpp" "*.hpp" "*.inl"
+                           "*.cxx" "*.hxx" "*.cc" "*.c" "*.h" "*.hh")
+
+      set(_glob_expressions)
+      foreach(_exp ${_ext_expressions})
+          foreach(_base_dir ${_base_dirs})
+              list(APPEND _glob_expressions "${PROJECT_SOURCE_DIR}/${_base_dir}/${_exp}")
+          endforeach()
+      endforeach()
+
+      # Glob for list of files to run code checks on
+      set(_sources)
+      file(GLOB_RECURSE _sources ${_glob_expressions})
+
+      blt_add_code_checks(PREFIX          RAJA
+                          SOURCES         ${_sources}
+                          CLANGFORMAT_CFG_FILE ${PROJECT_SOURCE_DIR}/.clang-format
+                          CPPCHECK_FLAGS  --enable=all --inconclusive)
+
+      # Set FOLDER property for code check targets
+      foreach(_suffix clangformat_check clangformat_style clang_tidy_check clang_tidy_style)
+          set(_tgt ${arg_PREFIX}_${_suffix})
+          if(TARGET ${_tgt})
+              set_target_properties(${_tgt} PROPERTIES FOLDER "RAJA/code_checks")
+          endif()
+      endforeach()
+  endif()
+
+endmacro(raja_add_code_checks)

docs/sphinx/dev_guide/contributing.rst

@@ -193,6 +193,62 @@ likely the develop branch.
                **username/feature/<name-of-feature>** for a new feature, 
                **username/bugfix/<issue-fixed>** for a bugfix, etc.
 
+-----------------------------------
+Code Formatting
+-----------------------------------
+RAJA enforces style within the ``src`` and ``include`` directories using clang-format,
+major version 14.  Formatting will be enforced on pull requests through CI.
+Each contributor must guarantee that their code is in compliance with the
+clang-format settings specified within ``.clang-format``.  To make this easy,
+RAJA has provided two options for applying clang-format:
+
+* The CMake build target ``style``
+
+  *  If a ``clang-format`` executable with major version 14 is available in the ``PATH``
+     when running CMake, RAJA's build system will find that executable and set 
+     the CMake variable ``CLANGFORMAT_EXECUTABLE`` to that executable's path.  
+  *  Alternatively, the ``CLANGFORMAT_EXECUTABLE`` CMake variable can be set by the user 
+     to the path of a clang-format 14 executable.  For example, on Linux, this would look 
+     like
+
+    .. code-block:: bash
+
+      mkdir build
+      cd build
+      cmake ../ <other CMake options> -DCLANGFORMAT_EXECUTABLE=<path to clang-format 14>
+
+
+  *  If an invalid version of ``clang-format`` is supplied, the following error will be 
+     emitted at build config time:
+
+    .. code-block:: bash
+
+      blt_add_clangformat_target: clang-format '14' is required, found <incorrect version>.
+        Disabling 'style' build target.  
+
+
+  *  If no ``CLANGFORMAT_EXECUTABLE`` is supplied, ``cmake`` will print the warning 
+     ``Failed to locate CMakeFormat executable``.
+
+* Git hooks
+
+  * Follow these steps to setup githooks, from the root directory of RAJA
+
+    .. code-block:: bash
+
+     # Only necessary if clang-format 14 is not in the $PATH variable.  
+     # This line can also be placed in .zshrc
+     $ export RAJA_CLANG_FORMAT=<path to clang-format install>
+     $ scripts/setup-hooks.sh
+
+  * The ``scripts/setup-hooks.sh`` script will install a ``pre-commit`` git hook 
+    script that applies clang-format to any changes staged with git.  If a ``clang-format``
+    executable with major version 14 is available in the ``PATH``, this executable will be used.
+    If not, the user must set the environment variable ``RAJA_CLANG_FORMAT`` to a valid 
+    clang-format executable.  If the script cannot find a valid clang-format installation
+    from either the ``PATH`` or from the environment variable ``RAJA_CLANG_FORMAT``, the 
+    script will print a warning and exit, allowing the commit to continue.
+
 .. _prfromfork-label:
 
 ===========================================================

examples/dynamic-forall.cpp

@@ -77,28 +77,25 @@ int main(int argc, char *argv[])
 
 //----------------------------------------------------------------------------//
 
-  std::cout << "\n Running C-style vector addition...\n";
-
-  // _cstyle_vector_add_start
-  for (int i = 0; i < N; ++i) {
-    c[i] = a[i] + b[i];
-  }
-  // _cstyle_vector_add_end
-
-  checkResult(c, N);
-//printResult(c, N);
-
-
-//----------------------------------------------------------------------------//
-// Example of dynamic policy selection for forall
-//----------------------------------------------------------------------------//
+  std::cout << "\n Running dynamic forall vector addition and reductions...\n";
 
+  int sum = 0;
+  using VAL_INT_SUM = RAJA::expt::ValOp<int, RAJA::operators::plus>;
+
+  RAJA::RangeSegment range(0, N); 
+
   //policy is chosen from the list
-  RAJA::expt::dynamic_forall<policy_list>(pol, RAJA::RangeSegment(0, N), [=] RAJA_HOST_DEVICE (int i)   {
+  RAJA::dynamic_forall<policy_list>(pol, range,
+    RAJA::expt::Reduce<RAJA::operators::plus>(&sum),
+      RAJA::expt::KernelName("RAJA dynamic forall"),
+      [=] RAJA_HOST_DEVICE (int i, VAL_INT_SUM &_sum) {
+
       c[i] = a[i] + b[i];
+      _sum += 1;
   });
   // _rajaseq_vector_add_end
 
+  std::cout << "Sum = " << sum << ", expected sum: " << N << std::endl;
   checkResult(c, N);
 //printResult(c, N);
 
@@ -126,9 +123,9 @@ void checkResult(int* res, int len)
     if ( res[i] != 0 ) { correct = false; }
   }
   if ( correct ) {
-    std::cout << "\n\t result -- PASS\n";
+    std::cout << "\n\t Vector sum result -- PASS\n";
   } else {
-    std::cout << "\n\t result -- FAIL\n";
+    std::cout << "\n\t Vector sum result -- FAIL\n";
   }
 }
 

examples/memoryManager.hpp

@@ -76,7 +76,7 @@ void deallocate(T *&ptr)
     hipErrchk(hipMalloc((void **)&ptr, sizeof(T) * size));
 #elif defined(RAJA_ENABLE_SYCL)
       auto qu = sycl_res->get<camp::resources::Sycl>().get_queue();
-      ptr = cl::sycl::malloc_device<T>(size, *qu);
+      ptr = ::sycl::malloc_device<T>(size, *qu);
 #endif
     return ptr;
   }

examples/resource-dynamic-forall.cpp

@@ -121,7 +121,7 @@ int main(int argc, char *argv[])
   RAJA::resources::Resource res = RAJA::Get_Host_Resource(host_res, select_cpu_or_gpu);
 #endif
 
-  RAJA::expt::dynamic_forall<policy_list>
+  RAJA::dynamic_forall<policy_list>
   (res, pol, RAJA::RangeSegment(0, N), [=] RAJA_HOST_DEVICE (int i)   {
 
     c[i] = a[i] + b[i];

exercises/memoryManager.hpp

@@ -76,7 +76,7 @@ void deallocate(T *&ptr)
     hipErrchk(hipMalloc((void **)&ptr, sizeof(T) * size));
 #elif defined(RAJA_ENABLE_SYCL)
       auto qu = sycl_res->get<camp::resources::Sycl>().get_queue();
-      ptr = cl::sycl::malloc_device<T>(size, *qu);
+      ptr = ::sycl::malloc_device<T>(size, *qu);
 #endif
     return ptr;
   }

include/RAJA/RAJA.hpp

@@ -88,7 +88,7 @@
 #endif
 
 #if defined(RAJA_ENABLE_DESUL_ATOMICS)
-    #include "RAJA/policy/desul.hpp"
+#include "RAJA/policy/desul.hpp"
 #endif
 
 #include "RAJA/index/IndexSet.hpp"
@@ -197,11 +197,14 @@
 
 #include "RAJA/pattern/sort.hpp"
 
-namespace RAJA {
-namespace expt{}
+namespace RAJA
+{
+namespace expt
+{}
+
 //  // provide a RAJA::expt namespace for experimental work, but bring alias
 //  // it into RAJA so it doesn't affect user code
 //  using namespace expt;
-}
+}  // namespace RAJA
 
 #endif  // closing endif for header file include guard

include/RAJA/config.hpp.in

@@ -182,6 +182,8 @@ static_assert(RAJA_HAS_SOME_CXX14,
 #cmakedefine RAJA_ENABLE_NV_TOOLS_EXT
 #cmakedefine RAJA_ENABLE_ROCTX
 
+#cmakedefine RAJA_HIP_WAVESIZE @RAJA_HIP_WAVESIZE@
+
 /*!
  ******************************************************************************
  *