Jump to Parthenon develop, build stuff

* Parthenon develop (+pep1 +solvers +non-cell AMR)
* Include git commit version/hash in parameters
* Print version header, control prints a bit
* Build script touchups and docs (not done yet)

BREAKS resizing for the moment, until labeled resize vars are
in Parthenon proper

.gitmodules

@@ -1,7 +1,7 @@
 [submodule "external/parthenon"]
 	path = external/parthenon
-	url = https://github.com/parthenon-hpc-lab/parthenon.git
-	branch = bprather/backport-bicgstab
+	url = https://github.com/AFD-Illinois/parthenon.git
+	branch = kharma
 [submodule "external/variant"]
 	path = external/variant
 	url = https://github.com/mpark/variant.git

CMakeLists.txt

@@ -7,11 +7,10 @@ cmake_minimum_required(VERSION 3.10)
 project(kharma LANGUAGES C CXX)
 
 # We follow Parthenon in requiring C++17 going forward
-#set(CMAKE_CXX_STANDARD 17)
-#set(CMAKE_CXX_STANDARD_REQUIRED ON)
-#set(CMAKE_CXX17_STANDARD_COMPILE_OPTION "-std=c++17")
-#set(PARTHENON_ENABLE_CPP17 ON CACHE BOOL "KHARMA Override")
-set(CMAKE_CXX_STANDARD 14)
+set(CMAKE_CXX_STANDARD 17)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+set(CMAKE_CXX17_STANDARD_COMPILE_OPTION "-std=c++17")
+set(PARTHENON_ENABLE_CPP17 ON CACHE BOOL "KHARMA Override")
 
 # Set the path to include cmake/ dir
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
@@ -27,8 +26,6 @@ set(PARTHENON_DISABLE_SPARSE ON CACHE BOOL "KHARMA Override")
 # Parthenon internal build options
 set(BUILD_TESTING OFF CACHE BOOL "KHARMA Override")
 set(ENABLE_COMPILER_WARNINGS OFF CACHE BOOL "KHARMA Override")
-# TODO upstream Parthenon needs support before we do this
-#set(COORDINATE_TYPE GRCoordinates)
 # Always use static HDF5
 set(HDF5_USE_STATIC_LIBRARIES ON CACHE BOOL "KHARMA Override")
 
@@ -39,9 +36,9 @@ set(Kokkos_ENABLE_CUDA_CONSTEXPR ON CACHE BOOL "KHARMA Override")
 set(Kokkos_ENABLE_HWLOC OFF CACHE BOOL "KHARMA Override") # Possible speed improvement?
 set(Kokkos_ENABLE_AGGRESSIVE_VECTORIZATION ON CACHE BOOL "KHARMA Override")
 
-# If we build KokkosKernels at all, disable the extras
-set(KokkosKernels_ENABLE_TPL_CUSPARSE OFF CACHE BOOL "KHARMA Override")
-set(KokkosKernels_ENABLE_TPL_CUBLAS OFF CACHE BOOL "KHARMA Override")
+include(GetGitRevisionDescription)
+get_git_head_revision(GIT_REFSPEC GIT_SHA1)
+git_describe_working_tree(GIT_VERSION --tags)
 
 # Offer a KHARMA option to disable the MPI requirement
 # The only difference from setting PARTHENON_DISABLE_MPI is that

cmake/GetGitRevisionDescription.cmake

@@ -0,0 +1,284 @@
+# - Returns a version string from Git
+#
+# These functions force a re-configure on each git commit so that you can
+# trust the values of the variables in your build system.
+#
+#  get_git_head_revision(<refspecvar> <hashvar> [ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR])
+#
+# Returns the refspec and sha hash of the current head revision
+#
+#  git_describe(<var> [<additional arguments to git describe> ...])
+#
+# Returns the results of git describe on the source tree, and adjusting
+# the output so that it tests false if an error occurs.
+#
+#  git_describe_working_tree(<var> [<additional arguments to git describe> ...])
+#
+# Returns the results of git describe on the working tree (--dirty option),
+# and adjusting the output so that it tests false if an error occurs.
+#
+#  git_get_exact_tag(<var> [<additional arguments to git describe> ...])
+#
+# Returns the results of git describe --exact-match on the source tree,
+# and adjusting the output so that it tests false if there was no exact
+# matching tag.
+#
+#  git_local_changes(<var>)
+#
+# Returns either "CLEAN" or "DIRTY" with respect to uncommitted changes.
+# Uses the return code of "git diff-index --quiet HEAD --".
+# Does not regard untracked files.
+#
+# Requires CMake 2.6 or newer (uses the 'function' command)
+#
+# Original Author:
+# 2009-2020 Ryan Pavlik <[email protected]> <[email protected]>
+# http://academic.cleardefinition.com
+#
+# Copyright 2009-2013, Iowa State University.
+# Copyright 2013-2020, Ryan Pavlik
+# Copyright 2013-2020, Contributors
+# SPDX-License-Identifier: BSL-1.0
+# Distributed under the Boost Software License, Version 1.0.
+# (See accompanying file LICENSE_1_0.txt or copy at
+# http://www.boost.org/LICENSE_1_0.txt)
+
+if(__get_git_revision_description)
+    return()
+endif()
+set(__get_git_revision_description YES)
+
+# We must run the following at "include" time, not at function call time,
+# to find the path to this module rather than the path to a calling list file
+get_filename_component(_gitdescmoddir ${CMAKE_CURRENT_LIST_FILE} PATH)
+
+# Function _git_find_closest_git_dir finds the next closest .git directory
+# that is part of any directory in the path defined by _start_dir.
+# The result is returned in the parent scope variable whose name is passed
+# as variable _git_dir_var. If no .git directory can be found, the
+# function returns an empty string via _git_dir_var.
+#
+# Example: Given a path C:/bla/foo/bar and assuming C:/bla/.git exists and
+# neither foo nor bar contain a file/directory .git. This wil return
+# C:/bla/.git
+#
+function(_git_find_closest_git_dir _start_dir _git_dir_var)
+    set(cur_dir "${_start_dir}")
+    set(git_dir "${_start_dir}/.git")
+    while(NOT EXISTS "${git_dir}")
+        # .git dir not found, search parent directories
+        set(git_previous_parent "${cur_dir}")
+        get_filename_component(cur_dir "${cur_dir}" DIRECTORY)
+        if(cur_dir STREQUAL git_previous_parent)
+            # We have reached the root directory, we are not in git
+            set(${_git_dir_var}
+                ""
+                PARENT_SCOPE)
+            return()
+        endif()
+        set(git_dir "${cur_dir}/.git")
+    endwhile()
+    set(${_git_dir_var}
+        "${git_dir}"
+        PARENT_SCOPE)
+endfunction()
+
+function(get_git_head_revision _refspecvar _hashvar)
+    _git_find_closest_git_dir("${CMAKE_CURRENT_SOURCE_DIR}" GIT_DIR)
+
+    if("${ARGN}" STREQUAL "ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR")
+        set(ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR TRUE)
+    else()
+        set(ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR FALSE)
+    endif()
+    if(NOT "${GIT_DIR}" STREQUAL "")
+        file(RELATIVE_PATH _relative_to_source_dir "${CMAKE_SOURCE_DIR}"
+             "${GIT_DIR}")
+        if("${_relative_to_source_dir}" MATCHES "[.][.]" AND NOT ALLOW_LOOKING_ABOVE_CMAKE_SOURCE_DIR)
+            # We've gone above the CMake root dir.
+            set(GIT_DIR "")
+        endif()
+    endif()
+    if("${GIT_DIR}" STREQUAL "")
+        set(${_refspecvar}
+            "GITDIR-NOTFOUND"
+            PARENT_SCOPE)
+        set(${_hashvar}
+            "GITDIR-NOTFOUND"
+            PARENT_SCOPE)
+        return()
+    endif()
+
+    # Check if the current source dir is a git submodule or a worktree.
+    # In both cases .git is a file instead of a directory.
+    #
+    if(NOT IS_DIRECTORY ${GIT_DIR})
+        # The following git command will return a non empty string that
+        # points to the super project working tree if the current
+        # source dir is inside a git submodule.
+        # Otherwise the command will return an empty string.
+        #
+        execute_process(
+            COMMAND "${GIT_EXECUTABLE}" rev-parse
+                    --show-superproject-working-tree
+            WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+            OUTPUT_VARIABLE out
+            ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+        if(NOT "${out}" STREQUAL "")
+            # If out is empty, GIT_DIR/CMAKE_CURRENT_SOURCE_DIR is in a submodule
+            file(READ ${GIT_DIR} submodule)
+            string(REGEX REPLACE "gitdir: (.*)$" "\\1" GIT_DIR_RELATIVE
+                                 ${submodule})
+            string(STRIP ${GIT_DIR_RELATIVE} GIT_DIR_RELATIVE)
+            get_filename_component(SUBMODULE_DIR ${GIT_DIR} PATH)
+            get_filename_component(GIT_DIR ${SUBMODULE_DIR}/${GIT_DIR_RELATIVE}
+                                   ABSOLUTE)
+            set(HEAD_SOURCE_FILE "${GIT_DIR}/HEAD")
+        else()
+            # GIT_DIR/CMAKE_CURRENT_SOURCE_DIR is in a worktree
+            file(READ ${GIT_DIR} worktree_ref)
+            # The .git directory contains a path to the worktree information directory
+            # inside the parent git repo of the worktree.
+            #
+            string(REGEX REPLACE "gitdir: (.*)$" "\\1" git_worktree_dir
+                                 ${worktree_ref})
+            string(STRIP ${git_worktree_dir} git_worktree_dir)
+            _git_find_closest_git_dir("${git_worktree_dir}" GIT_DIR)
+            set(HEAD_SOURCE_FILE "${git_worktree_dir}/HEAD")
+        endif()
+    else()
+        set(HEAD_SOURCE_FILE "${GIT_DIR}/HEAD")
+    endif()
+    set(GIT_DATA "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/git-data")
+    if(NOT EXISTS "${GIT_DATA}")
+        file(MAKE_DIRECTORY "${GIT_DATA}")
+    endif()
+
+    if(NOT EXISTS "${HEAD_SOURCE_FILE}")
+        return()
+    endif()
+    set(HEAD_FILE "${GIT_DATA}/HEAD")
+    configure_file("${HEAD_SOURCE_FILE}" "${HEAD_FILE}" COPYONLY)
+
+    configure_file("${_gitdescmoddir}/GetGitRevisionDescription.cmake.in"
+                   "${GIT_DATA}/grabRef.cmake" @ONLY)
+    include("${GIT_DATA}/grabRef.cmake")
+
+    set(${_refspecvar}
+        "${HEAD_REF}"
+        PARENT_SCOPE)
+    set(${_hashvar}
+        "${HEAD_HASH}"
+        PARENT_SCOPE)
+endfunction()
+
+function(git_describe _var)
+    if(NOT GIT_FOUND)
+        find_package(Git QUIET)
+    endif()
+    get_git_head_revision(refspec hash)
+    if(NOT GIT_FOUND)
+        set(${_var}
+            "GIT-NOTFOUND"
+            PARENT_SCOPE)
+        return()
+    endif()
+    if(NOT hash)
+        set(${_var}
+            "HEAD-HASH-NOTFOUND"
+            PARENT_SCOPE)
+        return()
+    endif()
+
+    # TODO sanitize
+    #if((${ARGN}" MATCHES "&&") OR
+    #	(ARGN MATCHES "||") OR
+    #	(ARGN MATCHES "\\;"))
+    #	message("Please report the following error to the project!")
+    #	message(FATAL_ERROR "Looks like someone's doing something nefarious with git_describe! Passed arguments ${ARGN}")
+    #endif()
+
+    #message(STATUS "Arguments to execute_process: ${ARGN}")
+
+    execute_process(
+        COMMAND "${GIT_EXECUTABLE}" describe --tags --always ${hash} ${ARGN}
+        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+        RESULT_VARIABLE res
+        OUTPUT_VARIABLE out
+        ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+    if(NOT res EQUAL 0)
+        set(out "${out}-${res}-NOTFOUND")
+    endif()
+
+    set(${_var}
+        "${out}"
+        PARENT_SCOPE)
+endfunction()
+
+function(git_describe_working_tree _var)
+    if(NOT GIT_FOUND)
+        find_package(Git QUIET)
+    endif()
+    if(NOT GIT_FOUND)
+        set(${_var}
+            "GIT-NOTFOUND"
+            PARENT_SCOPE)
+        return()
+    endif()
+
+    execute_process(
+        COMMAND "${GIT_EXECUTABLE}" describe --dirty ${ARGN}
+        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+        RESULT_VARIABLE res
+        OUTPUT_VARIABLE out
+        ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+    if(NOT res EQUAL 0)
+        set(out "${out}-${res}-NOTFOUND")
+    endif()
+
+    set(${_var}
+        "${out}"
+        PARENT_SCOPE)
+endfunction()
+
+function(git_get_exact_tag _var)
+    git_describe(out --exact-match ${ARGN})
+    set(${_var}
+        "${out}"
+        PARENT_SCOPE)
+endfunction()
+
+function(git_local_changes _var)
+    if(NOT GIT_FOUND)
+        find_package(Git QUIET)
+    endif()
+    get_git_head_revision(refspec hash)
+    if(NOT GIT_FOUND)
+        set(${_var}
+            "GIT-NOTFOUND"
+            PARENT_SCOPE)
+        return()
+    endif()
+    if(NOT hash)
+        set(${_var}
+            "HEAD-HASH-NOTFOUND"
+            PARENT_SCOPE)
+        return()
+    endif()
+
+    execute_process(
+        COMMAND "${GIT_EXECUTABLE}" diff-index --quiet HEAD --
+        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+        RESULT_VARIABLE res
+        OUTPUT_VARIABLE out
+        ERROR_QUIET OUTPUT_STRIP_TRAILING_WHITESPACE)
+    if(res EQUAL 0)
+        set(${_var}
+            "CLEAN"
+            PARENT_SCOPE)
+    else()
+        set(${_var}
+            "DIRTY"
+            PARENT_SCOPE)
+    endif()
+endfunction()

cmake/GetGitRevisionDescription.cmake.in

@@ -0,0 +1,46 @@
+#
+# Internal file for GetGitRevisionDescription.cmake
+#
+# Requires CMake 2.6 or newer (uses the 'function' command)
+#
+# Original Author:
+# 2009-2010 Ryan Pavlik <[email protected]> <[email protected]>
+# http://academic.cleardefinition.com
+# Iowa State University HCI Graduate Program/VRAC
+#
+# Copyright 2009-2012, Iowa State University
+# Copyright 2011-2015, Contributors
+# Distributed under the Boost Software License, Version 1.0.
+# (See accompanying file LICENSE_1_0.txt or copy at
+# http://www.boost.org/LICENSE_1_0.txt)
+# SPDX-License-Identifier: BSL-1.0
+
+set(HEAD_HASH)
+
+file(READ "@HEAD_FILE@" HEAD_CONTENTS LIMIT 1024)
+
+string(STRIP "${HEAD_CONTENTS}" HEAD_CONTENTS)
+if(HEAD_CONTENTS MATCHES "ref")
+	# named branch
+	string(REPLACE "ref: " "" HEAD_REF "${HEAD_CONTENTS}")
+	if(EXISTS "@GIT_DIR@/${HEAD_REF}")
+		configure_file("@GIT_DIR@/${HEAD_REF}" "@GIT_DATA@/head-ref" COPYONLY)
+	else()
+		if(EXISTS "@GIT_DIR@/packed-refs")
+			configure_file("@GIT_DIR@/packed-refs" "@GIT_DATA@/packed-refs" COPYONLY)
+			file(READ "@GIT_DATA@/packed-refs" PACKED_REFS)
+			if(${PACKED_REFS} MATCHES "([0-9a-z]*) ${HEAD_REF}")
+				set(HEAD_HASH "${CMAKE_MATCH_1}")
+			endif()
+		endif()
+	endif()
+else()
+	# detached HEAD
+	configure_file("@GIT_DIR@/HEAD" "@GIT_DATA@/head-ref" COPYONLY)
+endif()
+
+if(NOT HEAD_HASH)
+	file(READ "@GIT_DATA@/head-ref" HEAD_HASH LIMIT 1024)
+	string(STRIP "${HEAD_HASH}" HEAD_HASH)
+endif()
+

kharma/CMakeLists.txt

@@ -55,6 +55,12 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/reductions)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/emhd)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/wind)
 
+# generate version.cpp file from current git commit in a way that
+# 1. Re-generates if hash changes
+# 2. Does not re-build everything when hash changes
+configure_file("${CMAKE_CURRENT_SOURCE_DIR}/version.cpp.in" "${CMAKE_CURRENT_BINARY_DIR}/version.cpp" @ONLY)
+list(APPEND EXE_NAME_SRC "${CMAKE_CURRENT_BINARY_DIR}/version.cpp" version.hpp)
+
 add_executable(${EXE_NAME} ${EXE_NAME_SRC})
 
 target_link_libraries(${EXE_NAME} PUBLIC kokkos)