diff --git a/.github/workflows/clang_tidy.yml b/.github/workflows/clang_tidy.yml index 5abb42352f4..af8632ed698 100644 --- a/.github/workflows/clang_tidy.yml +++ b/.github/workflows/clang_tidy.yml @@ -13,6 +13,7 @@ jobs: dim: [1, 2, RZ, 3] name: clang-tidy-${{ matrix.dim }}D runs-on: ubuntu-22.04 + timeout-minutes: 120 if: github.event.pull_request.draft == false steps: - uses: actions/checkout@v4 @@ -43,7 +44,7 @@ jobs: -DWarpX_DIMS="${{ matrix.dim }}" \ -DWarpX_MPI=ON \ -DWarpX_COMPUTE=OMP \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DWarpX_QED=ON \ -DWarpX_QED_TABLE_GEN=ON \ -DWarpX_OPENPMD=ON \ diff --git a/.github/workflows/cuda.yml b/.github/workflows/cuda.yml index de1c8237021..8f5e59ccaa4 100644 --- a/.github/workflows/cuda.yml +++ b/.github/workflows/cuda.yml @@ -70,7 +70,7 @@ jobs: -DWarpX_OPENPMD=ON \ -DWarpX_openpmd_internal=OFF \ -DWarpX_PRECISION=SINGLE \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON cmake --build build_sp -j 4 @@ -115,8 +115,8 @@ jobs: which nvcc || echo "nvcc not in PATH!" git clone https://github.com/AMReX-Codes/amrex.git ../amrex - cd ../amrex && git checkout --detach 7ca419ebb90da60fefc01d8c1816846fff8638a5 && cd - - make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_PSATD=TRUE USE_CCACHE=TRUE -j 4 + cd ../amrex && git checkout --detach 20e6f2eadf0c297517588ba38973ec7c7084fa31 && cd - + make COMP=gcc QED=FALSE USE_MPI=TRUE USE_GPU=TRUE USE_OMP=FALSE USE_FFT=TRUE USE_CCACHE=TRUE -j 4 ccache -s du -hs ~/.cache/ccache @@ -168,7 +168,7 @@ jobs: -DWarpX_PYTHON=OFF \ -DAMReX_CUDA_ARCH=8.0 \ -DWarpX_OPENPMD=ON \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DAMReX_CUDA_ERROR_CROSS_EXECUTION_SPACE_CALL=ON \ -DAMReX_CUDA_ERROR_CAPTURE_THIS=ON cmake --build build -j 4 diff --git a/.github/workflows/hip.yml b/.github/workflows/hip.yml index 0e311f061ef..0f6710b1405 100644 --- a/.github/workflows/hip.yml +++ b/.github/workflows/hip.yml @@ -55,7 +55,7 @@ jobs: -DWarpX_MPI=ON \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=SINGLE \ - -DWarpX_PSATD=ON + -DWarpX_FFT=ON cmake --build build_sp -j 4 export WARPX_MPI=OFF @@ -115,7 +115,7 @@ jobs: -DWarpX_MPI=ON \ -DWarpX_OPENPMD=ON \ -DWarpX_PRECISION=DOUBLE \ - -DWarpX_PSATD=ON + -DWarpX_FFT=ON cmake --build build_2d -j 4 export WARPX_MPI=OFF diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 21d5ae04faf..00ac8f06b5d 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -44,7 +44,7 @@ jobs: - name: CCache Cache uses: actions/cache@v4 with: - path: /Users/runner/Library/Caches/ccache + path: ~/Library/Caches/ccache key: ccache-${{ github.workflow }}-${{ github.job }}-git-${{ github.sha }} restore-keys: | ccache-${{ github.workflow }}-${{ github.job }}-git- @@ -53,7 +53,7 @@ jobs: export CCACHE_COMPRESS=1 export CCACHE_COMPRESSLEVEL=10 export CCACHE_MAXSIZE=100M - export CCACHE_DEPEND=1 + export CCACHE_SLOPPINESS=time_macros ccache -z source py-venv/bin/activate @@ -76,6 +76,7 @@ jobs: cmake --build build_sp -j 3 cmake --build build_sp --target pip_install + du -hs ~/Library/Caches/ccache ccache -s - name: run pywarpx diff --git a/.github/workflows/ubuntu.yml b/.github/workflows/ubuntu.yml index 2997e9cdd16..9f4953b7a98 100644 --- a/.github/workflows/ubuntu.yml +++ b/.github/workflows/ubuntu.yml @@ -77,7 +77,7 @@ jobs: -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_DIMS="1;2" \ -DWarpX_EB=OFF \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DWarpX_QED_TABLE_GEN=ON \ -DWarpX_QED_TOOLS=ON @@ -127,7 +127,7 @@ jobs: -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_DIMS="RZ;3" \ -DWarpX_EB=OFF \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DWarpX_PRECISION=SINGLE \ -DWarpX_PARTICLE_PRECISION=SINGLE \ -DWarpX_QED_TABLE_GEN=ON @@ -211,7 +211,7 @@ jobs: cmake -S . -B build \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_APP=OFF \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DWarpX_PYTHON=ON \ -DWarpX_QED_TABLE_GEN=ON cmake --build build -j 4 --target pip_install @@ -260,7 +260,7 @@ jobs: -GNinja \ -DCMAKE_VERBOSE_MAKEFILE=ON \ -DWarpX_DIMS="RZ;1;2;3" \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DWarpX_QED=ON \ -DWarpX_QED_TABLE_GEN=ON \ -DWarpX_OPENPMD=ON \ diff --git a/.gitignore b/.gitignore index 4fa65351331..094fee87efc 100644 --- a/.gitignore +++ b/.gitignore @@ -4,7 +4,7 @@ Python/pywarpx/libwarpx*.so d/ f/ o/ -build/ +build*/ tmp_build_dir/ test_dir test_dir/ diff --git a/CMakeLists.txt b/CMakeLists.txt index 8efd9e137bc..66332e9ff50 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,7 @@ # Preamble #################################################################### # cmake_minimum_required(VERSION 3.20.0) -project(WarpX VERSION 24.05) +project(WarpX VERSION 24.07) include(${WarpX_SOURCE_DIR}/cmake/WarpXFunctions.cmake) @@ -16,6 +16,12 @@ endif() # CMake policies ############################################################## # +# Setting a cmake_policy to OLD is deprecated by definition and will raise a +# verbose warning +if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR) + set(CMAKE_WARN_DEPRECATED OFF CACHE BOOL "" FORCE) +endif() + # AMReX 21.06+ supports CUDA_ARCHITECTURES with CMake 3.20+ # CMake 3.18+: CMAKE_CUDA_ARCHITECTURES # https://cmake.org/cmake/help/latest/policy/CMP0104.html @@ -72,7 +78,8 @@ option(WarpX_EB "Embedded boundary support" OFF) option(WarpX_LIB "Build WarpX as a library" OFF) option(WarpX_MPI "Multi-node support (message-passing)" ON) option(WarpX_OPENPMD "openPMD I/O (HDF5, ADIOS)" ON) -option(WarpX_PSATD "spectral solver support" OFF) +option(WarpX_FFT "FFT-based solvers" OFF) +option(WarpX_HEFFTE "Multi-node FFT-based solvers" OFF) option(WarpX_PYTHON "Python bindings" OFF) option(WarpX_SENSEI "SENSEI in situ diagnostics" OFF) option(WarpX_QED "QED support (requires PICSAR)" ON) @@ -130,15 +137,28 @@ mark_as_advanced(WarpX_MPI_THREAD_MULTIPLE) option(WarpX_amrex_internal "Download & build AMReX" ON) +if(WarpX_HEFFTE AND NOT WarpX_MPI) + message(FATAL_ERROR "WarpX_HEFFTE (${WarpX_HEFFTE}) can only be used if WarpX_MPI is ON.") +endif() + # change the default build type to Release (or RelWithDebInfo) instead of Debug set_default_build_type("Release") # Option to enable interprocedural optimization # (also know as "link-time optimization" or "whole program optimization") -option(WarpX_IPO "Compile WarpX with interprocedural optimization (will take more time)" OFF) +set(_WarpX_IPO_DEFAULT OFF) +set(_WarpX_PYTHON_IPO_DEFAULT ON) +if(DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION) + set(_WarpX_IPO_DEFAULT ${CMAKE_INTERPROCEDURAL_OPTIMIZATION}) + set(_WarpX_PYTHON_IPO_DEFAULT ${CMAKE_INTERPROCEDURAL_OPTIMIZATION}) +endif() +option(WarpX_IPO + "Compile WarpX with interprocedural optimization (will take more time)" + ${_WarpX_IPO_DEFAULT} +) option(WarpX_PYTHON_IPO "Compile Python bindings with interprocedural optimization (IPO) / link-time optimization (LTO)" - ON + ${_WarpX_PYTHON_IPO_DEFAULT} ) set(pyWarpX_VERSION_INFO "" CACHE STRING @@ -149,6 +169,14 @@ if(WarpX_APP OR WarpX_PYTHON) set(WarpX_LIB ON CACHE STRING "Build WarpX as a library" FORCE) endif() +# deprecated options: transition phase + +if(DEFINED WarpX_PSATD) + message(WARNING "CMake option WarpX_PSATD is deprecated. Use WarpX_FFT instead.\n" + "Overwriting WarpX_FFT with '${WarpX_PSATD}' because WarpX_PSATD was set.") + set(WarpX_FFT ${WarpX_PSATD} CACHE STRING "FFT-based solvers" FORCE) +endif() + # note: we could skip this if we solely build WarpX_APP, but if we build a # shared WarpX library or a third party, like ImpactX, uses ablastr in a # shared library (e.g., for Python bindings), then we need relocatable code. @@ -156,9 +184,13 @@ option(ABLASTR_POSITION_INDEPENDENT_CODE "Build ABLASTR with position independent code" ON) mark_as_advanced(ABLASTR_POSITION_INDEPENDENT_CODE) -option(ABLASTR_FFT "compile AnyFFT wrappers" ${WarpX_PSATD}) -if(WarpX_PSATD) - set(ABLASTR_FFT ON CACHE STRING "compile AnyFFT wrappers" FORCE) +option(ABLASTR_FFT "compile AnyFFT wrappers" ${WarpX_FFT}) +if(WarpX_FFT) + set(ABLASTR_FFT ON CACHE STRING "FFT-based solvers" FORCE) +endif() +option(ABLASTR_HEFFTE "compile AnyFFT wrappers" ${WarpX_HEFFTE}) +if(WarpX_HEFFTE) + set(ABLASTR_HEFFTE ON CACHE STRING "Multi-Node FFT-based solvers" FORCE) endif() # this defined the variable BUILD_TESTING which is ON by default @@ -192,7 +224,7 @@ include(${WarpX_SOURCE_DIR}/cmake/dependencies/openPMD.cmake) # PSATD include(${WarpX_SOURCE_DIR}/cmake/dependencies/FFT.cmake) -if(WarpX_PSATD) +if(WarpX_FFT) # BLASPP and LAPACKPP if(RZ IN_LIST WarpX_DIMS) find_package(blaspp CONFIG REQUIRED) @@ -201,6 +233,23 @@ if(WarpX_PSATD) endif() endif() +# multi-node FFT +if(WarpX_HEFFTE) + if(WarpX_COMPUTE STREQUAL CUDA) + set(_heFFTe_COMPS CUDA) + elseif(WarpX_COMPUTE STREQUAL HIP) + set(_heFFTe_COMPS ROCM) + elseif(WarpX_COMPUTE STREQUAL SYCL) + set(_heFFTe_COMPS ONEAPI) + else() # NOACC, OMP + set(_heFFTe_COMPS FFTW) # or MKL + endif() + # note: we could also enforce GPUAWARE for CUDA and HIP, which can still be + # disabled at runtime + + find_package(Heffte REQUIRED COMPONENTS ${_heFFTe_COMPS}) +endif() + # Python if(WarpX_PYTHON) find_package(Python COMPONENTS Interpreter Development.Module REQUIRED) @@ -415,7 +464,7 @@ endif() # Interprocedural optimization (IPO) / Link-Time Optimization (LTO) if(WarpX_IPO) - enable_IPO("${_ALL_TARGETS}") + warpx_enable_IPO("${_ALL_TARGETS}") endif() # link dependencies @@ -444,10 +493,20 @@ foreach(D IN LISTS WarpX_DIMS) endif() endif() + if(ABLASTR_HEFFTE) + target_link_libraries(ablastr_${SD} PUBLIC Heffte::Heffte) + endif() + if(WarpX_PYTHON) target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::module pybind11::windows_extras) if(WarpX_PYTHON_IPO) - target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::lto) + if(DEFINED CMAKE_INTERPROCEDURAL_OPTIMIZATION) + warpx_enable_IPO(pyWarpX_${SD}) + else() + # conditionally defined target in pybind11 + # https://github.com/pybind/pybind11/blob/v2.12.0/tools/pybind11Common.cmake#L397-L403 + target_link_libraries(pyWarpX_${SD} PRIVATE pybind11::lto) + endif() endif() endif() @@ -520,14 +579,21 @@ foreach(D IN LISTS WarpX_DIMS) endif() endif() - if(WarpX_PSATD) - target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_PSATD) + if(WarpX_FFT) + target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_FFT) endif() if(ABLASTR_FFT) # We need to enable FFT support in ABLASTR for PSATD solver target_compile_definitions(ablastr_${SD} PUBLIC ABLASTR_USE_FFT) endif() + if(WarpX_HEFFTE) + target_compile_definitions(ablastr_${SD} PUBLIC WARPX_USE_HEFFTE) + endif() + if(ABLASTR_HEFFTE) + target_compile_definitions(ablastr_${SD} PUBLIC ABLASTR_USE_HEFFTE) + endif() + if(WarpX_PYTHON AND pyWarpX_VERSION_INFO) # for module __version__ target_compile_definitions(pyWarpX_${SD} PRIVATE diff --git a/Docs/Doxyfile b/Docs/Doxyfile index 550ed9a7a01..5fbb7651b18 100644 --- a/Docs/Doxyfile +++ b/Docs/Doxyfile @@ -2301,7 +2301,7 @@ PREDEFINED = AMREX_Linux=1 \ WARPX_DIM_XZ=1 \ WARPX_USE_GPU=1 \ WARPX_USE_OPENPMD=1 \ - WARPX_USE_PSATD=1 \ + WARPX_USE_FFT=1 \ WARPX_QED=1 \ WARPX_QED_TABLE_GEN=1 diff --git a/Docs/requirements.txt b/Docs/requirements.txt index b16bf161144..1ce1ee6c1a0 100644 --- a/Docs/requirements.txt +++ b/Docs/requirements.txt @@ -13,7 +13,7 @@ openpmd-viewer # for checksumAPI # PICMI API docs # note: keep in sync with version in ../requirements.txt -picmistandard==0.28.0 +picmistandard==0.29.0 # for development against an unreleased PICMI version, use: # picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python diff --git a/Docs/source/acknowledge_us.rst b/Docs/source/acknowledge_us.rst index 4c7a2d8137c..b01ebf00a39 100644 --- a/Docs/source/acknowledge_us.rst +++ b/Docs/source/acknowledge_us.rst @@ -23,14 +23,14 @@ Please add the following sentence to your publications, it helps contributors ke **Plain text:** - This research used the open-source particle-in-cell code WarpX https://github.com/ECP-WarpX/WarpX, primarily funded by the US DOE Exascale Computing Project. Primary WarpX contributors are with LBNL, LLNL, CEA-LIDYL, SLAC, DESY, CERN, and TAE Technologies. We acknowledge all WarpX contributors. + This research used the open-source particle-in-cell code WarpX https://github.com/ECP-WarpX/WarpX. Primary WarpX contributors are with LBNL, LLNL, CEA-LIDYL, SLAC, DESY, CERN, and TAE Technologies. We acknowledge all WarpX contributors. **LaTeX:** .. code-block:: latex \usepackage{hyperref} - This research used the open-source particle-in-cell code WarpX \url{https://github.com/ECP-WarpX/WarpX}, primarily funded by the US DOE Exascale Computing Project. + This research used the open-source particle-in-cell code WarpX \url{https://github.com/ECP-WarpX/WarpX}. Primary WarpX contributors are with LBNL, LLNL, CEA-LIDYL, SLAC, DESY, CERN, and TAE Technologies. We acknowledge all WarpX contributors. @@ -55,8 +55,8 @@ If your project uses a specific algorithm or component, please consider citing t - Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. - Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. - `preprint __` + Proc. of Platform for Advanced Scientific Computing (PASC'24), *PASC24 Best Paper Award*, 2024. + `DOI:10.1145/3659914.3659937 `__ - Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. diff --git a/Docs/source/acknowledgements.rst b/Docs/source/acknowledgements.rst index 14877839294..6d2529705d0 100644 --- a/Docs/source/acknowledgements.rst +++ b/Docs/source/acknowledgements.rst @@ -3,13 +3,18 @@ Funding and Acknowledgements ============================ -WarpX is supported by the Exascale Computing Project (17-SC-20-SC), a collaborative effort of two U.S. Department of Energy organizations (Office of Science and the National Nuclear Security Administration) responsible for the planning and preparation of a capable exascale ecosystem, including software, applications, hardware, advanced system engineering, and early testbed platforms, in support of the nation's exascale computing imperative. +WarpX is hosted by the High Performance Computing Foundation (HPSF). +If your organization wants to help steer the evolution of the HPC software ecosystem, visit `hpsf.io `__ and consider joining! WarpX is supported by the CAMPA collaboration, a project of the U.S. Department of Energy, Office of Science, Office of Advanced Scientific Computing Research and Office of High Energy Physics, Scientific Discovery through Advanced Computing (SciDAC) program. -ABLASTR seed development is supported by the Laboratory Directed Research and Development Program of Lawrence Berkeley National Laboratory under U.S. Department of Energy Contract No. DE-AC02-05CH11231. +WarpX is supported by the KISMET project, a project of the U.S. Department of Energy, Office of Science, Office of Advanced Scientific Computing Research and Office of Fusion Energy Science, Scientific Discovery through Advanced Computing (SciDAC) program. CEA-LIDYL actively contributes to the co-development of WarpX. As part of this initiative, WarpX also receives funding from the French National Research Agency (ANR - Plasm-On-Chip), the Horizon H2020 program and CEA. +WarpX was supported by the Exascale Computing Project (17-SC-20-SC), a collaborative effort of two U.S. Department of Energy organizations (Office of Science and the National Nuclear Security Administration) responsible for the planning and preparation of a capable exascale ecosystem, including software, applications, hardware, advanced system engineering, and early testbed platforms, in support of the nation's exascale computing imperative. + +ABLASTR seed development was supported by the Laboratory Directed Research and Development Program of Lawrence Berkeley National Laboratory under U.S. Department of Energy Contract No. DE-AC02-05CH11231. + We acknowledge all the contributors and users of the WarpX community who participate to the code quality with valuable code improvement and important feedback. diff --git a/Docs/source/conf.py b/Docs/source/conf.py index 471f6f2b6a3..7997b1e338c 100644 --- a/Docs/source/conf.py +++ b/Docs/source/conf.py @@ -103,9 +103,9 @@ def __init__(self, *args, **kwargs): # built documents. # # The short X.Y version. -version = u'24.05' +version = u'24.07' # The full version, including alpha/beta/rc tags. -release = u'24.05' +release = u'24.07' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. diff --git a/Docs/source/developers/gnumake.rst b/Docs/source/developers/gnumake.rst index ca51dacfbd5..6324ceb4ac7 100644 --- a/Docs/source/developers/gnumake.rst +++ b/Docs/source/developers/gnumake.rst @@ -56,7 +56,7 @@ options are: * ``DIM=3`` or ``2``: Geometry of the simulation (note that running an executable compiled for 3D with a 2D input file will crash). * ``DEBUG=FALSE`` or ``TRUE``: Compiling in ``DEBUG`` mode can help tremendously during code development. - * ``USE_PSATD=FALSE`` or ``TRUE``: Compile the Pseudo-Spectral Analytical Time Domain Maxwell solver. Requires an FFT library. + * ``USE_FFT=FALSE`` or ``TRUE``: Compile the Pseudo-Spectral Analytical Time Domain Maxwell solver. Requires an FFT library. * ``USE_RZ=FALSE`` or ``TRUE``: Compile for 2D axisymmetric geometry. * ``COMP=gcc`` or ``intel``: Compiler. * ``USE_MPI=TRUE`` or ``FALSE``: Whether to compile with MPI support. diff --git a/Docs/source/developers/gnumake/rzgeometry.rst b/Docs/source/developers/gnumake/rzgeometry.rst index ac90fb754f2..a5c0993a805 100644 --- a/Docs/source/developers/gnumake/rzgeometry.rst +++ b/Docs/source/developers/gnumake/rzgeometry.rst @@ -17,7 +17,7 @@ RZ geometry with spectral solver Additional steps are needed to build the spectral solver. Some of the steps are the same as is done for the Cartesian spectral solver, setting up the FFTW -package and setting ``USE_PSATD=TRUE``. +package and setting ``USE_FFT=TRUE``. - Install (or load) an MPI-enabled version of FFTW. For instance, for Debian, this can be done with @@ -54,7 +54,7 @@ package and setting ``USE_PSATD=TRUE``. export BLAS_LIB=-lblas - - Set ``USE_PSATD=TRUE`` when compiling: + - Set ``USE_FFT=TRUE`` when compiling: :: - make -j 4 USE_RZ=TRUE USE_PSATD=TRUE + make -j 4 USE_RZ=TRUE USE_FFT=TRUE diff --git a/Docs/source/developers/gnumake/spectral.rst b/Docs/source/developers/gnumake/spectral.rst index a5a0e6448f5..bb0baf01c09 100644 --- a/Docs/source/developers/gnumake/spectral.rst +++ b/Docs/source/developers/gnumake/spectral.rst @@ -16,10 +16,10 @@ In order to run the code with a spectral solver, you need to: export FFTW_HOME=/usr/ - - Set ``USE_PSATD=TRUE`` when compiling: + - Set ``USE_FFT=TRUE`` when compiling: :: - make -j 4 USE_PSATD=TRUE + make -j 4 USE_FFT=TRUE See :doc:`rzgeometry` for using the spectral solver with USE_RZ. Additional steps are needed. PSATD is compatible with single precision, but please note that, on CPU, FFTW needs to be compiled with option ``--enable-float``. diff --git a/Docs/source/highlights.rst b/Docs/source/highlights.rst index b243f62cd97..108f685a551 100644 --- a/Docs/source/highlights.rst +++ b/Docs/source/highlights.rst @@ -14,7 +14,22 @@ Plasma-Based Acceleration Scientific works in laser-plasma and beam-plasma acceleration. -#. Peng, H. and Huang, T. W. and Jiang, K. and Li, R. and Wu, C. N. and Yu, M. Y. and Riconda, C. and Weber, S. and Zhou, C. T. and Ruan, S. C. +#. Shrock JE, Rockafellow E, Miao B, Le M, Hollinger RC, Wang S, Gonsalves AJ, Picksley A, Rocca JJ, and Milchberg HM + **Guided Mode Evolution and Ionization Injection in Meter-Scale Multi-GeV Laser Wakefield Accelerators**. + Phys. Rev. Lett. **133**, 045002, 2024 + `DOI:10.1103/PhysRevLett.133.045002 `__ + +#. Ross AJ, Chappell J, van de Wetering JJ, Cowley J, Archer E, Bourgeois N, Corner L, Emerson DR, Feder L, Gu XJ, Jakobsson O, Jones H, Picksley A, Reid L, Wang W, Walczak R, Hooker SM. + **Resonant excitation of plasma waves in a plasma channel**. + Phys. Rev. Research **6**, L022001, 2024 + `DOI:10.1103/PhysRevResearch.6.L022001 `__ + +#. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. + **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. + Proc. of Platform for Advanced Scientific Computing (PASC'24), *PASC24 Best Paper Award*, 2024. + `DOI:10.1145/3659914.3659937 `__ + +#. Peng H, Huang TW, Jiang K, Li R, Wu CN, Yu MY, Riconda C, Weber S, Zhou CT, Ruan SC. **Coherent Subcycle Optical Shock from a Superluminal Plasma Wake**. Phys. Rev. Lett. **131**, 145003, 2023 `DOI:10.1103/PhysRevLett.131.145003 `__ @@ -24,11 +39,6 @@ Scientific works in laser-plasma and beam-plasma acceleration. Phys. Rev. Research **5**, 033112, 2023 `DOI:10.1103/PhysRevResearch.5.033112 `__ -#. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. - **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. - Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. - `preprint `__ - #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L and Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. 14th International Particle Accelerator Conference (IPAC'23), WEPA101, 2023. @@ -103,14 +113,20 @@ Scientific works in particle and beam modeling. #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Myers A, Qiang J, Vay J-L and Huebl A. **Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines**. - Proc. of Platform for Advanced Scientific Computing (PASC'24), *submitted*, 2024. - `preprint `__ + Proc. of Platform for Advanced Scientific Computing (PASC'24), *PASC24 Best Paper Award*, 2024. + `DOI:10.1145/3659914.3659937 `__ + +#. Nguyen B, Formenti A, Lehe R, Vay J-L, Gessner S, and Fedeli L. + **Comparison of WarpX and GUINEA-PIG for electron positron collisions**. + 15th International Particle Accelerator Conference (IPAC'24), WEPC84, 2024. + `preprint `__, + `DOI:10.18429/JACoW-IPAC2024-WEPC84 `__ #. Sandberg R T, Lehe R, Mitchell C E, Garten M, Qiang J, Vay J-L, Huebl A. **Hybrid Beamline Element ML-Training for Surrogates in the ImpactX Beam-Dynamics Code**. - 14th International Particle Accelerator Conference (IPAC'23), WEPA101, *in print*, 2023. + 14th International Particle Accelerator Conference (IPAC'23), WEPA101, 2023. `preprint `__, - `DOI:10.18429/JACoW-IPAC-23-WEPA101 `__ + `DOI:10.18429/JACoW-IPAC2023-WEPA101 `__ #. Tan W H, Piot P, Myers A, Zhang W, Rheaume T, Jambunathan R, Huebl A, Lehe R, Vay J-L. **Simulation studies of drive-beam instability in a dielectric wakefield accelerator**. diff --git a/Docs/source/install/cmake.rst b/Docs/source/install/cmake.rst index 006cd843b31..fde927c10e1 100644 --- a/Docs/source/install/cmake.rst +++ b/Docs/source/install/cmake.rst @@ -96,7 +96,8 @@ CMake Option Default & Values Descr ``WarpX_OPENPMD`` **ON**/OFF openPMD I/O (HDF5, ADIOS) ``WarpX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double) ``WarpX_PARTICLE_PRECISION`` SINGLE/**DOUBLE** Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set -``WarpX_PSATD`` ON/**OFF** Spectral solver +``WarpX_FFT`` ON/**OFF** FFT-based solvers +``WarpX_HEFFTE`` ON/**OFF** Multi-Node FFT-based solvers ``WarpX_PYTHON`` ON/**OFF** Python bindings ``WarpX_QED`` **ON**/OFF QED support (requires PICSAR) ``WarpX_QED_TABLE_GEN`` ON/**OFF** QED table generation support (requires PICSAR and Boost) @@ -270,7 +271,8 @@ Environment Variable Default & Values Descr ``WARPX_OPENPMD`` **ON**/OFF openPMD I/O (HDF5, ADIOS) ``WARPX_PRECISION`` SINGLE/**DOUBLE** Floating point precision (single/double) ``WARPX_PARTICLE_PRECISION`` SINGLE/**DOUBLE** Particle floating point precision (single/double), defaults to WarpX_PRECISION value if not set -``WARPX_PSATD`` ON/**OFF** Spectral solver +``WARPX_FFT`` ON/**OFF** FFT-based solvers +``WARPX_HEFFTE`` ON/**OFF** Multi-Node FFT-based solvers ``WARPX_QED`` **ON**/OFF PICSAR QED (requires PICSAR) ``WARPX_QED_TABLE_GEN`` ON/**OFF** QED table generation (requires PICSAR and Boost) ``BUILD_PARALLEL`` ``2`` Number of threads to use for parallel builds @@ -310,11 +312,11 @@ Some Developers like to code directly against a local copy of AMReX, changing bo WARPX_AMREX_SRC=$PWD/../amrex python3 -m pip install --force-reinstall --no-deps -v . Additional environment control as common for CMake (:ref:`see above `) can be set as well, e.g. ``CC``, `CXX``, and ``CMAKE_PREFIX_PATH`` hints. -So another sophisticated example might be: use Clang as the compiler, build with local source copies of PICSAR and AMReX, support the PSATD solver, MPI and openPMD, hint a parallel HDF5 installation in ``$HOME/sw/hdf5-parallel-1.10.4``, and only build 2D and 3D geometry: +So another sophisticated example might be: use Clang as the compiler, build with local source copies of PICSAR and AMReX, support the FFT-based solvers, MPI and openPMD, hint a parallel HDF5 installation in ``$HOME/sw/hdf5-parallel-1.10.4``, and only build 2D and 3D geometry: .. code-block:: bash - CC=$(which clang) CXX=$(which clang++) WARPX_AMREX_SRC=$PWD/../amrex WARPX_PICSAR_SRC=$PWD/../picsar WARPX_PSATD=ON WARPX_MPI=ON WARPX_DIMS="2;3" CMAKE_PREFIX_PATH=$HOME/sw/hdf5-parallel-1.10.4:$CMAKE_PREFIX_PATH python3 -m pip install --force-reinstall --no-deps -v . + CC=$(which clang) CXX=$(which clang++) WARPX_AMREX_SRC=$PWD/../amrex WARPX_PICSAR_SRC=$PWD/../picsar WARPX_FFT=ON WARPX_MPI=ON WARPX_DIMS="2;3" CMAKE_PREFIX_PATH=$HOME/sw/hdf5-parallel-1.10.4:$CMAKE_PREFIX_PATH python3 -m pip install --force-reinstall --no-deps -v . Here we wrote this all in one line, but one can also set all environment variables in a development environment and keep the pip call nice and short as in the beginning. Note that you need to use absolute paths for external source trees, because pip builds in a temporary directory, e.g. ``export WARPX_AMREX_SRC=$HOME/src/amrex``. diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst index ce9f9dca520..3bab32b7502 100644 --- a/Docs/source/install/dependencies.rst +++ b/Docs/source/install/dependencies.rst @@ -23,12 +23,13 @@ Optional dependencies include: - for on-node accelerated compute *one of either*: - `OpenMP 3.1+ `__: for threaded CPU execution or - - `CUDA Toolkit 11.7+ `__: for Nvidia GPU support (see `matching host-compilers `_) or + - `CUDA Toolkit 11.7+ `__: for Nvidia GPU support (see `matching host-compilers `__) or - `ROCm 5.2+ (5.5+ recommended) `__: for AMD GPU support -- `FFTW3 `_: for spectral solver (PSATD) support when running on CPU or SYCL +- `FFTW3 `__: for spectral solver (PSATD or IGF) support when running on CPU or SYCL - also needs the ``pkg-config`` tool on Unix -- `BLAS++ `_ and `LAPACK++ `_: for spectral solver (PSATD) support in RZ geometry +- `heFFTe 2.4.0+ `__ and `LAPACK++ `__: for spectral solver (PSATD) support in RZ geometry - `Boost 1.66.0+ `__: for QED lookup tables generation support - `openPMD-api 0.15.1+ `__: we automatically download and compile a copy of openPMD-api for openPMD I/O support diff --git a/Docs/source/install/hpc.rst b/Docs/source/install/hpc.rst index 5efdeae39a8..5b388b9d0b2 100644 --- a/Docs/source/install/hpc.rst +++ b/Docs/source/install/hpc.rst @@ -36,6 +36,7 @@ This section documents quick-start guides for a selection of supercomputers that hpc/crusher hpc/frontier hpc/fugaku + hpc/greatlakes hpc/hpc3 hpc/juwels hpc/karolina @@ -46,12 +47,12 @@ This section documents quick-start guides for a selection of supercomputers that hpc/lxplus hpc/ookami hpc/perlmutter + hpc/pitzer hpc/polaris hpc/quartz hpc/spock hpc/summit hpc/taurus - hpc/greatlakes .. tip:: diff --git a/Docs/source/install/hpc/adastra.rst b/Docs/source/install/hpc/adastra.rst index 0b984d5e2be..e47cb5f535e 100644 --- a/Docs/source/install/hpc/adastra.rst +++ b/Docs/source/install/hpc/adastra.rst @@ -100,7 +100,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $SHAREDHOMEDIR/src/warpx rm -rf build_adastra - cmake -S . -B build_adastra -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_adastra -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_adastra -j 16 The WarpX application executables are now in ``$SHAREDHOMEDIR/src/warpx/build_adastra/bin/``. @@ -110,7 +110,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_adastra_py - cmake -S . -B build_adastra_py -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_adastra_py -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_adastra_py -j 16 --target pip_install Now, you can :ref:`submit Adstra compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/crusher.rst b/Docs/source/install/hpc/crusher.rst index 1aa471ce68b..7a657e8a51c 100644 --- a/Docs/source/install/hpc/crusher.rst +++ b/Docs/source/install/hpc/crusher.rst @@ -99,7 +99,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_crusher - cmake -S . -B build_crusher -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_crusher -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_crusher -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_crusher/bin/``. @@ -109,7 +109,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_crusher_py - cmake -S . -B build_crusher_py -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_crusher_py -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_crusher_py -j 16 --target pip_install Now, you can :ref:`submit Crusher compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/frontier.rst b/Docs/source/install/hpc/frontier.rst index e9f8b687d6b..c33c2fc7401 100644 --- a/Docs/source/install/hpc/frontier.rst +++ b/Docs/source/install/hpc/frontier.rst @@ -99,7 +99,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_frontier - cmake -S . -B build_frontier -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_frontier -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_frontier -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_frontier/bin/``. @@ -109,7 +109,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_frontier_py - cmake -S . -B build_frontier_py -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_frontier_py -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_frontier_py -j 16 --target pip_install Now, you can :ref:`submit Frontier compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/greatlakes.rst b/Docs/source/install/hpc/greatlakes.rst index d238d498e5c..9691134f571 100644 --- a/Docs/source/install/hpc/greatlakes.rst +++ b/Docs/source/install/hpc/greatlakes.rst @@ -116,7 +116,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_v100 - cmake -S . -B build_v100 -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_v100 -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_v100 -j 8 The WarpX application executables are now in ``$HOME/src/warpx/build_v100/bin/``. @@ -127,7 +127,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_v100_py - cmake -S . -B build_v100_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_v100_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_v100_py -j 8 --target pip_install diff --git a/Docs/source/install/hpc/hpc3.rst b/Docs/source/install/hpc/hpc3.rst index e2f900de273..5d398b1f4f5 100644 --- a/Docs/source/install/hpc/hpc3.rst +++ b/Docs/source/install/hpc/hpc3.rst @@ -95,7 +95,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build - cmake -S . -B build -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build -j 8 The WarpX application executables are now in ``$HOME/src/warpx/build/bin/``. @@ -105,7 +105,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_py - cmake -S . -B build_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_py -j 8 --target pip_install Now, you can :ref:`submit HPC3 compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/juwels.rst b/Docs/source/install/hpc/juwels.rst index a8d820ccd46..5358e649825 100644 --- a/Docs/source/install/hpc/juwels.rst +++ b/Docs/source/install/hpc/juwels.rst @@ -57,7 +57,7 @@ Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following comman cd $HOME/src/warpx rm -rf build - cmake -S . -B build -DWarpX_DIMS="1;2;3" -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_MPI_THREAD_MULTIPLE=OFF + cmake -S . -B build -DWarpX_DIMS="1;2;3" -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_MPI_THREAD_MULTIPLE=OFF cmake --build build -j 16 The other :ref:`general compile-time options ` apply as usual. diff --git a/Docs/source/install/hpc/karolina.rst b/Docs/source/install/hpc/karolina.rst index fffb917df1c..127b81477da 100644 --- a/Docs/source/install/hpc/karolina.rst +++ b/Docs/source/install/hpc/karolina.rst @@ -80,7 +80,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $WORK/src/warpx rm -rf build_gpu - cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_gpu -j 48 The WarpX application executables are now in ``$WORK/src/warpx/build_gpu/bin/``. @@ -91,7 +91,7 @@ Additionally, the following commands will install WarpX as a Python module: cd $WORK/src/warpx rm -rf build_gpu_py - cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_gpu_py -j 48 --target pip_install Now, you can :ref:`submit Karolina compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/lassen.rst b/Docs/source/install/hpc/lassen.rst index d8565f06983..43330dcc111 100644 --- a/Docs/source/install/hpc/lassen.rst +++ b/Docs/source/install/hpc/lassen.rst @@ -24,27 +24,9 @@ If you are new to this system, **please see the following resources**: Login ----- -.. tab-set:: - - .. tab-item:: TOSS4 (RHEL8) - - Lassen is currently transitioning to RHEL8. - During this transition, first SSH into lassen and then to the updated RHEL8/TOSS4 nodes. - - .. code-block:: bash - - ssh lassen.llnl.gov - ssh eatoss4 - - Approximately October/November 2023, the new software environment on these nodes will be the new default. - - .. tab-item:: TOSS3 (RHEL7) - - .. code-block:: bash - - ssh lassen.llnl.gov +.. code-block:: bash - Approximately October/November 2023, this partition will become TOSS4 (RHEL8) as well. + ssh lassen.llnl.gov .. _building-lassen-preparation: @@ -58,135 +40,54 @@ Use the following commands to download the WarpX source code: git clone https://github.com/ECP-WarpX/WarpX.git /usr/workspace/${USER}/lassen/src/warpx -.. tab-set:: - - .. tab-item:: TOSS4 (RHEL8) - - We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx.profile``. - Create it now: - - .. code-block:: bash - - cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example $HOME/lassen_v100_warpx.profile - - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down - - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example - :language: bash - - Edit the 2nd line of this script, which sets the ``export proj=""`` variable. - For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx.profile``. - Enter the edit mode by typing ``i`` and edit line 2 to read: - - .. code-block:: bash - - export proj="nsldt" - - Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). - - .. important:: - - Now, and as the first step on future logins to lassen, activate these environment settings: - - .. code-block:: bash +We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx_toss3.profile``. +Create it now: - source $HOME/lassen_v100_warpx.profile - - .. tab-item:: TOSS3 (RHEL7) - - We use system software modules, add environment hints and further dependencies via the file ``$HOME/lassen_v100_warpx_toss3.profile``. - Create it now: - - .. code-block:: bash +.. code-block:: bash - cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example $HOME/lassen_v100_warpx_toss3.profile + cp /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example $HOME/lassen_v100_warpx_toss3.profile - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example - :language: bash + .. literalinclude:: ../../../../Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example + :language: bash - Edit the 2nd line of this script, which sets the ``export proj=""`` variable. - For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx_toss3.profile``. - Enter the edit mode by typing ``i`` and edit line 2 to read: +Edit the 2nd line of this script, which sets the ``export proj=""`` variable. +For example, if you are member of the project ``nsldt``, then run ``vi $HOME/lassen_v100_warpx_toss3.profile``. +Enter the edit mode by typing ``i`` and edit line 2 to read: - .. code-block:: bash +.. code-block:: bash - export proj="nsldt" + export proj="nsldt" - Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). +Exit the ``vi`` editor with ``Esc`` and then type ``:wq`` (write & quit). - .. important:: +.. important:: - Now, and as the first step on future logins to lassen, activate these environment settings: + Now, and as the first step on future logins to lassen, activate these environment settings: - .. code-block:: bash + .. code-block:: bash - source $HOME/lassen_v100_warpx_toss3.profile + source $HOME/lassen_v100_warpx_toss3.profile Finally, since lassen does not yet provide software modules for some of our dependencies, install them once: -.. tab-set:: - - .. tab-item:: TOSS4 (RHEL8) - - .. code-block:: bash - - bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies.sh - source /usr/workspace/${USER}/lassen/gpu/venvs/warpx-lassen/bin/activate - - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down - - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies.sh - :language: bash - - .. dropdown:: AI/ML Dependencies (Optional) - :animate: fade-in-slide-down - - If you plan to run AI/ML workflows depending on pyTorch, run the next step as well. - This will take a while and should be skipped if not needed. - - .. code-block:: bash - - runNode bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_ml.sh - - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down - - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_ml.sh - :language: bash - - For `optimas dependencies `__ (incl. scikit-learn), plan another hour of build time: - - .. code-block:: bash - - python3 -m pip install -r /usr/workspace/${USER}/lassen/src/warpx/Tools/optimas/requirements.txt - - .. tab-item:: TOSS3 (RHEL7) +.. code-block:: bash - .. code-block:: bash + bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh + source /usr/workspace/${USER}/lassen-toss3/gpu/venvs/warpx-lassen-toss3/bin/activate - bash /usr/workspace/${USER}/lassen/src/warpx/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh - source /usr/workspace/${USER}/lassen-toss3/gpu/venvs/warpx-lassen-toss3/bin/activate +.. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down - .. dropdown:: Script Details - :color: light - :icon: info - :animate: fade-in-slide-down + .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh + :language: bash - .. literalinclude:: ../../../../Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh - :language: bash .. _building-lassen-compilation: @@ -200,7 +101,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd /usr/workspace/${USER}/lassen/src/warpx rm -rf build_lassen - cmake -S . -B build_lassen -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_lassen -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_lassen -j 8 The WarpX application executables are now in ``/usr/workspace/${USER}/lassen/src/warpx/build_lassen/bin/``. @@ -210,7 +111,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_lassen_py - cmake -S . -B build_lassen_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_lassen_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_lassen_py -j 8 --target pip_install Now, you can :ref:`submit lassen compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/lawrencium.rst b/Docs/source/install/hpc/lawrencium.rst index 23be9b75431..2217c5a31ce 100644 --- a/Docs/source/install/hpc/lawrencium.rst +++ b/Docs/source/install/hpc/lawrencium.rst @@ -102,7 +102,7 @@ Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following comman cd $HOME/src/warpx rm -rf build - cmake -S . -B build -DWarpX_DIMS="1;2;RZ;3" -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON + cmake -S . -B build -DWarpX_DIMS="1;2;RZ;3" -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON cmake --build build -j 12 The general :ref:`cmake compile-time options ` apply as usual. @@ -122,14 +122,14 @@ For a *full PICMI install*, follow the :ref:`instructions for Python (PICMI) bin python3 -m pip install -r requirements.txt # compile parallel PICMI interfaces in 3D, 2D, 1D and RZ - WARPX_MPI=ON WARPX_COMPUTE=CUDA WARPX_PSATD=ON BUILD_PARALLEL=12 python3 -m pip install --force-reinstall --no-deps -v . + WARPX_MPI=ON WARPX_COMPUTE=CUDA WARPX_FFT=ON BUILD_PARALLEL=12 python3 -m pip install --force-reinstall --no-deps -v . Or, if you are *developing*, do a quick PICMI install of a *single geometry* (see: :ref:`WarpX_DIMS `) using: .. code-block:: bash # find dependencies & configure - cmake -S . -B build -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS=RZ + cmake -S . -B build -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS=RZ # build and then call "python3 -m pip install ..." cmake --build build --target pip_install -j 12 diff --git a/Docs/source/install/hpc/leonardo.rst b/Docs/source/install/hpc/leonardo.rst index 1f3c1e8fbfa..568a5612250 100644 --- a/Docs/source/install/hpc/leonardo.rst +++ b/Docs/source/install/hpc/leonardo.rst @@ -89,7 +89,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_gpu - cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_gpu -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_gpu/bin/``. @@ -100,7 +100,7 @@ Additionally, the following commands will install WarpX as a Python module: cd $HOME/src/warpx rm -rf build_gpu_py - cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_PYTHON=ON -DWarpX_APP=OFF -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_PYTHON=ON -DWarpX_APP=OFF -DWarpX_DIMS="1;2;RZ;3" cmake --build build_gpu_py -j 16 --target pip_install Now, you can :ref:`submit Leonardo compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/lumi.rst b/Docs/source/install/hpc/lumi.rst index 2bae4d21599..2d155dab222 100644 --- a/Docs/source/install/hpc/lumi.rst +++ b/Docs/source/install/hpc/lumi.rst @@ -98,7 +98,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_lumi - cmake -S . -B build_lumi -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_QED_TABLES_GEN_OMP=OFF -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_lumi -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_QED_TABLES_GEN_OMP=OFF -DWarpX_DIMS="1;2;RZ;3" cmake --build build_lumi -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_lumi/bin/``. @@ -108,7 +108,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_lumi_py - cmake -S . -B build_lumi_py -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_QED_TABLES_GEN_OMP=OFF -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_lumi_py -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_QED_TABLES_GEN_OMP=OFF -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_lumi_py -j 16 --target pip_install diff --git a/Docs/source/install/hpc/perlmutter.rst b/Docs/source/install/hpc/perlmutter.rst index 14972566abc..dc5a985e99f 100644 --- a/Docs/source/install/hpc/perlmutter.rst +++ b/Docs/source/install/hpc/perlmutter.rst @@ -153,7 +153,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_gpu - cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_gpu -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_pm_gpu/bin/``. @@ -164,7 +164,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_gpu_py - cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_gpu_py -j 16 --target pip_install .. tab-item:: CPU Nodes @@ -174,7 +174,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_cpu - cmake -S . -B build_pm_cpu -DWarpX_COMPUTE=OMP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_cpu -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_cpu -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_pm_cpu/bin/``. @@ -184,7 +184,7 @@ Use the following :ref:`cmake commands ` to compile the applicat rm -rf build_pm_cpu_py - cmake -S . -B build_pm_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_cpu_py -DWarpX_COMPUTE=OMP -DWarpX_FFT=ON -DWarpX_HEFFTE=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_cpu_py -j 16 --target pip_install Now, you can :ref:`submit Perlmutter compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/pitzer.rst b/Docs/source/install/hpc/pitzer.rst new file mode 100644 index 00000000000..4f1e53bc309 --- /dev/null +++ b/Docs/source/install/hpc/pitzer.rst @@ -0,0 +1,274 @@ +.. _building-pitzer: + +Pitzer (OSC) +============ + +The `Pitzer cluster `__ is located at the Ohio Supercomputer Center (OSC). It is currently the main CPU/GPU cluster at OSC. However, the `Cardinal cluster `__ is soon going to take over Pitzer to become the next major CPU/GPU cluster at OSC in the second half of 2024. A list of all OSC clusters can be found `here `__. + +The Pitzer cluster offers a variety of partitions suitable for different computational needs, including GPU nodes, CPU nodes, and nodes with large memory capacities. For more information on the specifications and capabilities of these partitions, visit the `Ohio Supercomputer Center's Pitzer page `__. + +Introduction +------------ + +If you are new to this system, **please see the following resources**: + +* `Pitzer user guide `__ +* Batch system: `Slurm `__ +* `Jupyter service `__ +* `Filesystems `__: + + * ``$HOME``: per-user directory, use only for inputs, source, and scripts; backed up (500GB) + * ``/fs/ess``: per-project storage directory, use for long-term storage of data and analysis; backed up (1-5TB) + * ``/fs/scratch``: per-project production directory; fast I/O for parallel jobs; not backed up (100TB) + +.. _building-pitzer-preparation: + +Preparation +----------- + +Use the following commands to download the WarpX source code: + +.. code-block:: bash + + git clone https://github.com/ECP-WarpX/WarpX.git $HOME/src/warpx + +On Pitzer, you can run either on GPU nodes with V100 GPUs or CPU nodes. + +.. tab-set:: + + .. tab-item:: V100 GPUs + + We use system software modules, add environment hints and further dependencies via the file ``$HOME/pitzer_v100_warpx.profile``. + Create it now: + + .. code-block:: bash + + cp $HOME/src/warpx/Tools/machines/pitzer-osc/pitzer_v100_warpx.profile.example $HOME/pitzer_v100_warpx.profile + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/pitzer-osc/pitzer_v100_warpx.profile.example + :language: bash + + Edit the 2nd line of this script, which sets the ``export proj=""`` variable. + For example, if you are member of the project ``pas2024``, then run ``nano $HOME/pitzer_v100_warpx.profile`` and edit line 2 to read: + + .. code-block:: bash + + export proj="pas2024" + + Exit the ``nano`` editor with ``Ctrl`` + ``O`` (save) and then ``Ctrl`` + ``X`` (exit). + + .. important:: + + Now, and as the first step on future logins to pitzer, activate these environment settings: + + .. code-block:: bash + + source $HOME/pitzer_v100_warpx.profile + + Finally, since pitzer does not yet provide software modules for some of our dependencies, install them once: + + .. code-block:: bash + + bash $HOME/src/warpx/Tools/machines/pitzer-osc/install_v100_dependencies.sh + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/pitzer-osc/install_v100_dependencies.sh + :language: bash + + + .. tab-item:: CPU Nodes + + We use system software modules, add environment hints and further dependencies via the file ``$HOME/pitzer_cpu_warpx.profile``. + Create it now: + + .. code-block:: bash + + cp $HOME/src/warpx/Tools/machines/pitzer-osc/pitzer_cpu_warpx.profile.example $HOME/pitzer_cpu_warpx.profile + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/pitzer-osc/pitzer_cpu_warpx.profile.example + :language: bash + + Edit the 2nd line of this script, which sets the ``export proj=""`` variable. + For example, if you are member of the project ``pas2024``, then run ``nano $HOME/pitzer_cpu_warpx.profile`` and edit line 2 to read: + + .. code-block:: bash + + export proj="pas2024" + + Exit the ``nano`` editor with ``Ctrl`` + ``O`` (save) and then ``Ctrl`` + ``X`` (exit). + + .. important:: + + Now, and as the first step on future logins to pitzer, activate these environment settings: + + .. code-block:: bash + + source $HOME/pitzer_cpu_warpx.profile + + Finally, since pitzer does not yet provide software modules for some of our dependencies, install them once: + + .. code-block:: bash + + bash $HOME/src/warpx/Tools/machines/pitzer-osc/install_cpu_dependencies.sh + + .. dropdown:: Script Details + :color: light + :icon: info + :animate: fade-in-slide-down + + .. literalinclude:: ../../../../Tools/machines/pitzer-osc/install_cpu_dependencies.sh + :language: bash + + +.. _building-pitzer-compilation: + +Compilation +----------- + +Use the following :ref:`cmake commands ` to compile the application executable: + +.. tab-set:: + .. tab-item:: V100 GPUs + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_v100 + + cmake -S . -B build_v100 -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_v100 -j 48 + + The WarpX application executables are now in ``$HOME/src/warpx/build_v100/bin/``. Additionally, the following commands will install WarpX as a Python module: + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_v100_py + + cmake -S . -B build_v100_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_v100_py -j 48 --target pip_install + + .. tab-item:: CPU Nodes + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build + + cmake -S . -B build -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build -j 48 + + The WarpX application executables are now in ``$HOME/src/warpx/build/bin/``. Additionally, the following commands will install WarpX as a Python module: + + .. code-block:: bash + + cd $HOME/src/warpx + rm -rf build_py + + cmake -S . -B build_py -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake --build build_py -j 48 --target pip_install + +Now, you can :ref:`submit Pitzer compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). Or, you can use the WarpX executables to submit Pitzer jobs (:ref:`example inputs `). For executables, you can reference their location in your :ref:`job script ` or copy them to a location in ``/scratch``. + +.. _building-pitzer-update: + +Update WarpX & Dependencies +--------------------------- + +If you already installed WarpX in the past and want to update it, start by getting the latest source code: + +.. code-block:: bash + + cd $HOME/src/warpx + + # read the output of this command - does it look ok? + git status + + # get the latest WarpX source code + git fetch + git pull + + # read the output of these commands - do they look ok? + git status + git log # press q to exit + +And, if needed, + +- :ref:`update the pitzer_cpu_warpx.profile file `, +- log out and into the system, activate the now updated environment profile as usual, +- :ref:`execute the dependency install scripts `. + +As a last step, clean the build directory ``rm -rf $HOME/src/warpx/build_*`` and rebuild WarpX. + +.. _running-pitzer: + +Running +------- + +.. tab-set:: + + .. tab-item:: V100 GPUs + + Pitzer's GPU partition includes: + + - 32 nodes, each equipped with two V100 (16GB) GPUs. + - 42 nodes, each with two V100 (32GB) GPUs. + - 4 large memory nodes, each with quad V100 (32GB) GPUs. + + To run a WarpX simulation on the GPU nodes, use the batch script provided below. Adjust the ``-N`` parameter in the script to match the number of nodes you intend to use. Each node in this partition supports running one MPI rank per GPU. + + .. literalinclude:: ../../../../Tools/machines/pitzer-osc/pitzer_v100.sbatch + :language: bash + :caption: Copy this file from ``$HOME/src/warpx/Tools/machines/pitzer-osc/pitzer_v100.sbatch``. + + After preparing your script, submit your job with the following command: + + .. code-block:: bash + + sbatch pitzer_v100.sbatch + + .. tab-item:: CPU Nodes + + For CPU-based computations, Pitzer offers: + + - 224 nodes, each with dual Intel Xeon Gold 6148 CPUs and 192 GB RAM. + - 340 nodes, each with dual Intel Xeon Platinum 8268 CPUs and 192 GB RAM. + - 16 large memory nodes. + + To submit a job to the CPU partition, use the provided batch script. Ensure you have copied the script to your working directory. + + .. literalinclude:: ../../../../Tools/machines/pitzer-osc/pitzer_cpu.sbatch + :language: bash + :caption: Copy this file from ``$HOME/src/warpx/Tools/machines/pitzer-osc/pitzer_cpu.sbatch``. + + Submit your job with: + + .. code-block:: bash + + sbatch pitzer_cpu.sbatch + +.. _post-processing-osc: + +Post-Processing +--------------- + +For post-processing, many users prefer to use the online `Jupyter service `__ (`documentation `__) that is directly connected to the cluster's fast filesystem. + +.. note:: + + This section is a stub and contributions are welcome. + We can document further details, e.g., which recommended post-processing Python software to install or how to customize Jupyter kernels here. diff --git a/Docs/source/install/hpc/polaris.rst b/Docs/source/install/hpc/polaris.rst index d20ecccee32..a2d95a144d6 100644 --- a/Docs/source/install/hpc/polaris.rst +++ b/Docs/source/install/hpc/polaris.rst @@ -101,7 +101,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_gpu - cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_gpu -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_gpu -j 16 The WarpX application executables are now in ``$HOME/src/warpx/build_pm_gpu/bin/``. @@ -112,7 +112,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_pm_gpu_py - cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_pm_gpu_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_pm_gpu_py -j 16 --target pip_install .. tab-item:: CPU Nodes diff --git a/Docs/source/install/hpc/quartz.rst b/Docs/source/install/hpc/quartz.rst index 03860687971..a49327e8613 100644 --- a/Docs/source/install/hpc/quartz.rst +++ b/Docs/source/install/hpc/quartz.rst @@ -93,7 +93,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_quartz - cmake -S . -B build_quartz -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_quartz -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_quartz -j 6 The WarpX application executables are now in ``$HOME/src/warpx/build_quartz/bin/``. @@ -103,7 +103,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_quartz_py - cmake -S . -B build_quartz_py -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_quartz_py -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_quartz_py -j 6 --target pip_install Now, you can :ref:`submit Quartz compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/install/hpc/spock.rst b/Docs/source/install/hpc/spock.rst index 64f307f9dae..144257289f1 100644 --- a/Docs/source/install/hpc/spock.rst +++ b/Docs/source/install/hpc/spock.rst @@ -51,7 +51,7 @@ Then, ``cd`` into the directory ``$HOME/src/warpx`` and use the following comman cd $HOME/src/warpx rm -rf build - cmake -S . -B build -DWarpX_DIMS="1;2;3" -DWarpX_COMPUTE=HIP -DWarpX_PSATD=ON -DAMReX_AMD_ARCH=gfx908 -DMPI_CXX_COMPILER=$(which CC) -DMPI_C_COMPILER=$(which cc) -DMPI_COMPILER_FLAGS="--cray-print-opts=all" + cmake -S . -B build -DWarpX_DIMS="1;2;3" -DWarpX_COMPUTE=HIP -DWarpX_FFT=ON -DAMReX_AMD_ARCH=gfx908 -DMPI_CXX_COMPILER=$(which CC) -DMPI_C_COMPILER=$(which cc) -DMPI_COMPILER_FLAGS="--cray-print-opts=all" cmake --build build -j 10 The general :ref:`cmake compile-time options ` apply as usual. diff --git a/Docs/source/install/hpc/summit.rst b/Docs/source/install/hpc/summit.rst index 918da842165..bf226df69f2 100644 --- a/Docs/source/install/hpc/summit.rst +++ b/Docs/source/install/hpc/summit.rst @@ -124,7 +124,7 @@ Use the following :ref:`cmake commands ` to compile the applicat cd $HOME/src/warpx rm -rf build_summit - cmake -S . -B build_summit -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_summit -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_summit -j 8 The WarpX application executables are now in ``$HOME/src/warpx/build_summit/bin/``. @@ -134,7 +134,7 @@ Additionally, the following commands will install WarpX as a Python module: rm -rf build_summit_py - cmake -S . -B build_summit_py -DWarpX_COMPUTE=CUDA -DWarpX_PSATD=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" + cmake -S . -B build_summit_py -DWarpX_COMPUTE=CUDA -DWarpX_FFT=ON -DWarpX_QED_TABLE_GEN=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_DIMS="1;2;RZ;3" cmake --build build_summit_py -j 8 --target pip_install Now, you can :ref:`submit Summit compute jobs ` for WarpX :ref:`Python (PICMI) scripts ` (:ref:`example scripts `). diff --git a/Docs/source/refs.bib b/Docs/source/refs.bib index 84b34b17351..130e0ce4da7 100644 --- a/Docs/source/refs.bib +++ b/Docs/source/refs.bib @@ -205,17 +205,20 @@ @article{Roedel2010 year = {2010} } -@misc{SandbergPASC24, -address = {Zuerich, Switzerland}, -author = {Ryan Sandberg and Remi Lehe and Chad E Mitchell and Marco Garten and Andrew Myers and Ji Qiang and Jean-Luc Vay and Axel Huebl}, -booktitle = {Proc. of PASC24}, -note = {accepted}, -series = {PASC'24 - Platform for Advanced Scientific Computing}, -title = {{Synthesizing Particle-in-Cell Simulations Through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines}}, -venue = {Zuerich, Switzerland}, -year = {2024}, -doi = {10.48550/arXiv.2402.17248}, -url = {https://arxiv.org/abs/2402.17248} +@inproceedings{SandbergPASC24, +author = {Sandberg, Ryan and Lehe, Remi and Mitchell, Chad and Garten, Marco and Myers, Andrew and Qiang, Ji and Vay, Jean-Luc and Huebl, Axel}, +title = {{Synthesizing Particle-In-Cell Simulations through Learning and GPU Computing for Hybrid Particle Accelerator Beamlines}}, +series = {PASC '24}, +booktitle = {Proceedings of the Platform for Advanced Scientific Computing Conference}, +location = {Zuerich, Switzerland}, +publisher = {Association for Computing Machinery}, +address = {New York, NY, USA}, +articleno = {23}, +numpages = {11}, +isbn = {9798400706394}, +doi = {10.1145/3659914.3659937}, +note = {{PASC24 Best Paper Award}}, +year = {2024} } @inproceedings{SandbergIPAC23, diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 40c16a06df5..4a8c4be0b5f 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -86,37 +86,122 @@ Overall simulation parameters * ``explicit``: Use an explicit solver, such as the standard FDTD or PSATD - * ``implicit_picard``: Use an implicit solver with exact energy conservation that uses a Picard iteration to solve the system. - Note that this method is for demonstration only. It is inefficient and does not work well when - :math:`\omega_{pe} \Delta t` is close to or greater than one. - The method is described in `Angus et al., On numerical energy conservation for an implicit particle-in-cell method coupled with a binary Monte-Carlo algorithm for Coulomb collisions `__. - The version implemented is an updated version that is relativistically correct, including the relativistic gamma factor for the particles. - For exact energy conservation, ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, - and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1`` (which is the default, in - which case charge will also be conserved). - - * ``semi_implicit_picard``: Use an energy conserving semi-implicit solver that uses a Picard iteration to solve the system. - Note that this method has the CFL limitation :math:`\Delta t < c/\sqrt( \sum_i 1/\Delta x_i^2 )`. It is inefficient and does not work well or at all when :math:`\omega_{pe} \Delta t` is close to or greater than one. + * ``theta_implicit_em``: Use a fully implicit electromagnetic solver with a time-biasing parameter theta bound between 0.5 and 1.0. Exact energy conservation is achieved using theta = 0.5. Maximal damping of high-k modes is obtained using theta = 1.0. Choices for the nonlinear solver include a Picard iteration scheme and particle-suppressed (PS) JFNK. + The algorithm itself is numerical stable for large time steps. That is, it does not require time steps that resolve the plasma period or the CFL condition for light waves. However, the practicality of using a large time step depends on the nonlinear solver. Note that the Picard solver is for demonstration only. It is inefficient and will most like not converge when + :math:`\omega_{pe} \Delta t` is close to or greater than one or when the CFL condition for light waves is violated. The PS-JFNK method must be used in order to use large time steps. However, the current implementation of PS-JFNK is still inefficient because the JFNK solver is not preconditioned and there is no use of the mass matrices to minimize the cost of a linear iteration. The time step is limited by how many cells a particle can cross in a time step (MPI-related) and by the need to resolve the relevant physics. + The Picard method is described in `Angus et al., On numerical energy conservation for an implicit particle-in-cell method coupled with a binary Monte-Carlo algorithm for Coulomb collisions `__. + The PS-JFNK method is described in `Angus et al., An implicit particle code with exact energy and charge conservation for electromagnetic studies of dense plasmas `__ . (The version implemented in WarpX is an updated version that includes the relativistic gamma factor for the particles.) Also see `Chen et al., An energy- and charge-conserving, implicit, electrostatic particle-in-cell algorithm. `__ . + Exact energy conservation requires that the interpolation stencil used for the field gather match that used for the current deposition. ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1``. If using ``algo.current_deposition = villasenor``, the corresponding field gather routine will automatically be selected and the ``interpolation.galerkin_scheme`` flag does not need to be specified. The Esirkepov and villasenor deposition schemes are charge-conserving. + + * ``semi_implicit_em``: Use an approximately energy conserving semi-implicit electromagnetic solver. Choices for the nonlinear solver include a Picard iteration scheme and particle-suppressed JFNK. + Note that this method has the CFL limitation :math:`\Delta t < c/\sqrt( \sum_i 1/\Delta x_i^2 )`. The Picard solver for this method can only be expected to work well when :math:`\omega_{pe} \Delta t` is less than one. The method is described in `Chen et al., A semi-implicit, energy- and charge-conserving particle-in-cell algorithm for the relativistic Vlasov-Maxwell equations `__. - For energy conservation, ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, - and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1`` (which is the default, in - which case charge will also be conserved). - -* ``algo.max_picard_iterations`` (`integer`, default: 10) - When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets the maximum number of Picard - itearations that are done each time step. - -* ``algo.picard_iteration_tolerance`` (`float`, default: 1.e-7) - When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets the convergence tolerance of - the iterations, the maximum of the relative change of the L2 norm of the field from one iteration to the next. - If this is set to zero, the maximum number of iterations will always be done with the change only calculated on the last - iteration (for a slight optimization). - -* ``algo.require_picard_convergence`` (`bool`, default: 1) - When `algo.evolve_scheme` is either `implicit_picard` or `semi_implicit_picard`, this sets whether the iteration each step - is required to converge. - If it is required, an abort is raised if it does not converge and the code then exits. - If not, then a warning is issued and the calculation continues. + Exact energy conservation requires that the interpolation stencil used for the field gather match that used for the current deposition. ``algo.current_deposition = direct`` must be used with ``interpolation.galerkin_scheme = 0``, and ``algo.current_deposition = Esirkepov`` must be used with ``interpolation.galerkin_scheme = 1``. If using ``algo.current_deposition = villasenor``, the corresponding field gather routine will automatically be selected and the ``interpolation.galerkin_scheme`` flag does not need to be specified. The Esirkepov and villasenor deposition schemes are charge-conserving. + +* ``implicit_evolve.theta`` (`float`, default: 0.5) + When `algo.evolve_scheme = theta_implicit_em`, the fields used on the RHS of the equations for the implicit advance + are computed as (1-theta)*E_{n} + theta*E_{n+1}. theta is bound between 0.5 and 1. The default value of theta = 0.5 + is needed for exact energy conservation. For theta > 0.5, high-k modes will be damped and the method will not be + exactly energy conserving, but the solver may perform better. + +* ``implicit_evolve.nonlinear_solver`` (`string`, default: None) + When `algo.evolve_scheme` is either `theta_implicit_em` or `semi_implicit_em`, this sets the nonlinear solver used + to advance the field-particle system in time. Options are `picard` or `newton`. + +* ``implicit_evolve.max_particle_iterations`` (`integer`, default: 21) + When `algo.evolve_scheme` is either `theta_implicit_em` or `semi_implicit_em` and `implicit_evolve.nonlinear_solver = newton` + , this sets the maximum number of iterations for the method used to obtain a self-consistent update of the particles at + each iteration in the JFNK process. + +* ``implicit_evolve.particle_tolerance`` (`float`, default: 1.e-10) + When `algo.evolve_scheme` is either `theta_implicit_em` or `semi_implicit_em` and `implicit_evolve.nonlinear_solver = newton` + , this sets the relative tolerance for the iterative method used to obtain a self-consistent update of the particles at + each iteration in the JFNK process. + +* ``picard.verbose`` (`bool`, default: 1) + When `implicit_evolve.nonlinear_solver = picard`, this sets the verbosity of the Picard solver. If true, then information + on the nonlinear error are printed to screen at each nonlinear iteration. + +* ``picard.require_convergence`` (`bool`, default: 1) + When `implicit_evolve.nonlinear_solver = picard`, this sets whether the Picard method is required to converge at each + time step. If it is required, an abort is raised if it does not converge and the code then exits. If not, then a warning + is issued and the calculation continues. + +* ``picard.maximum_iterations`` (`int`, default: 100) + When `implicit_evolve.nonlinear_solver = picard`, this sets the maximum iterations used by the Picard method. If + `picard.require_convergence = false`, the solution is considered converged if the iteration count reaches this value, + but a warning is issued. If `picard.require_convergence = true`, then an abort is raised if the iteration count reaches + this value. + +* ``picard.relative_tolerance`` (`float`, default: 1.0e-6) + When `implicit_evolve.nonlinear_solver = picard`, this sets the relative tolerance used by the Picard method for determining + convergence. The absolute error for the Picard method is the L2 norm of the difference of the solution vector between + two successive iterations. The relative error is the absolute error after iteration k > 1 divided by the absolute error + after the first iteration. The Picard method is considered converged when the relative error is below the relative tolerance. + This is the preferred means of determining convergence. + +* ``picard.absolute_tolerance`` (`float`, default: 0.0) + When `implicit_evolve.nonlinear_solver = picard`, this sets the absolute tolerance used by the Picard method for determining + convergence. The default value is 0.0, which means that the absolute tolerance is not used to determine converence. The + solution vector in the nonlinear solvers are in physical units rather than normalized ones. Thus, the absolute scale + of the problem can vary over many orders and magnitude depending on the problem. The relative tolerance is the preferred + means of determining convergence. + +* ``newton.verbose`` (`bool`, default: 1) + When `implicit_evolve.nonlinear_solver = newton`, this sets the verbosity of the Newton solver. If true, then information + on the nonlinear error are printed to screen at each nonlinear iteration. + +* ``newton.require_convergence`` (`bool`, default: 1) + When `implicit_evolve.nonlinear_solver = newton`, this sets whether the Newton method is required to converge at each + time step. If it is required, an abort is raised if it does not converge and the code then exits. If not, then a warning + is issued and the calculation continues. + +* ``newton.maximum_iterations`` (`int`, default: 100) + When `implicit_evolve.nonlinear_solver = newton`, this sets the maximum iterations used by the Newton method. If + `newton.require_convergence = false`, the solution is considered converged if the iteration count reaches this value, + but a warning is issued. If `newton.require_convergence = true`, then an abort is raised if the iteration count reaches + this value. + +* ``newton.relative_tolerance`` (`float`, default: 1.0e-6) + When `implicit_evolve.nonlinear_solver = newton`, this sets the relative tolerance used by the Newton method for determining + convergence. The absolute error for the Newton method is the L2 norm of the residual vector. The relative error is the + absolute error divided by the L2 norm of the initial residual associated with the initial guess. The Newton method is + considered converged when the relative error is below the relative tolerance. This is the preferred means of determining + convergence. + +* ``newton.absolute_tolerance`` (`float`, default: 0.0) + When `implicit_evolve.nonlinear_solver = newton`, this sets the absolute tolerance used by the Newton method for determining + convergence. The default value is 0.0, which means that the absolute tolerance is not used to determine converence. The + residual vector in the nonlinear solvers are in physical units rather than normalized ones. Thus, the absolute scale + of the problem can vary over many orders and magnitude depending on the problem. The relative tolerance is the preferred + means of determining convergence. + +* ``gmres.verbose_int`` (`int`, default: 2) + When `implicit_evolve.nonlinear_solver = newton`, this sets the verbosity of the AMReX::GMRES linear solver. The default + value of 2 gives maximumal verbosity and information about the residual are printed to the screen at each GMRES iteration. + +* ``gmres.restart_length`` (`int`, default: 30) + When `implicit_evolve.nonlinear_solver = newton`, this sets the iteration number at which to do a restart in AMReX::GMRES. + This parameter is used to save memory on building the Krylov subspace basis vectors for linear systems that are ill-conditioned + and require many iterations to converge. + +* ``gmres.relative_tolerance`` (`float`, default: 1.0e-4) + When `implicit_evolve.nonlinear_solver = newton`, this sets the relative tolerance used to determine convergence of the + AMReX::GMRES linear solver used to compute the Newton step in the JNFK process. The absolute error is the L2 norm of the + residual vector. The relative error is the absolute error divided by the L2 norm of the initial residual (typically equal + to the norm of the nonlinear residual from the end of the previous Newton iteration). The linear solver is considered + converged when the relative error is below the relative tolerance. This is the preferred means of determining convergence. + +* ``gmres.absolute_tolerance`` (`float`, default: 0.0) + When `implicit_evolve.nonlinear_solver = newton`, this sets the absolute tolerance used to determine converence of the + GMRES linear solver used to compute the Newton step in the JNFK process. The default value is 0.0, which means that the + absolute tolerance is not used to determine converence. The residual vector in the nonlinear/linear solvers are in physical + units rather than normalized ones. Thus, the absolute scale of the problem can vary over many orders and magnitude depending + on the problem. The relative tolerance is the preferred means of determining convergence. + +* ``gmres.maximum_iterations`` (`int`, default: 1000) + When `implicit_evolve.nonlinear_solver = newton`, this sets the maximum iterations used by the GMRES linear solver. The + solution to the linear system is considered converged if the iteration count reaches this value. * ``warpx.do_electrostatic`` (`string`) optional (default `none`) Specifies the electrostatic mode. When turned on, instead of updating @@ -173,7 +258,7 @@ Overall simulation parameters * ``fft``: Poisson's equation is solved using an Integrated Green Function method (which requires FFT calculations). See these references for more details :cite:t:`QiangPhysRevSTAB2006`, :cite:t:`QiangPhysRevSTAB2006err`. - It only works in 3D and it requires the compilation flag ``-DWarpX_PSATD=ON``. + It only works in 3D and it requires the compilation flag ``-DWarpX_FFT=ON``. If mesh refinement is enabled, this solver only works on the coarsest level. On the refined patches, the Poisson equation is solved with the multigrid solver. In electrostatic mode, this solver requires open field boundary conditions (``boundary.field_lo,hi = open``). @@ -415,7 +500,7 @@ Domain Boundary Conditions * ``open``: For the electrostatic Poisson solver based on a Integrated Green Function method. * ``boundary.potential_lo_x/y/z`` and ``boundary.potential_hi_x/y/z`` (default `0`) - Gives the value of the electric potential at the boundaries, for ``pec`` boundaries. With electrostatic solvers + Gives the value of the electric potential, in Volts, at the boundaries, for ``pec`` boundaries. With electrostatic solvers (i.e., with ``warpx.do_electrostatic = ...``), this is used in order to compute the potential in the simulation volume at each timestep. When using other solvers (e.g. Maxwell solver), setting these variables will trigger an electrostatic solve at ``t=0``, to compute the initial @@ -437,6 +522,10 @@ Domain Boundary Conditions The standard deviation for these distributions should be provided for each species using ``boundary..u_th``. The same standard deviation is used to sample all components. + * ``None``: No boundary conditions are applied to the particles. + When using RZ, this option must be used for the lower radial boundary, the first value of ``boundary.particle_lo``. + This should not be used in any other cases. + * ``boundary.reflect_all_velocities`` (`bool`) optional (default `false`) For a reflecting boundary condition, this flags whether the sign of only the normal velocity is changed or all velocities. @@ -492,14 +581,29 @@ Additional PML parameters Embedded Boundary Conditions ---------------------------- -* ``warpx.eb_implicit_function`` (`string`) - A function of `x`, `y`, `z` that defines the surface of the embedded - boundary. That surface lies where the function value is 0 ; - the physics simulation area is where the function value is negative ; - the interior of the embeddded boundary is where the function value is positive. +In WarpX, the embedded boundary can be defined in either of two ways: + + - **From an analytical function:** + In that case, you will need to set the following parameter in the input file. + + * ``warpx.eb_implicit_function`` (`string`) + A function of `x`, `y`, `z` that defines the surface of the embedded + boundary. That surface lies where the function value is 0 ; + the physics simulation area is where the function value is negative ; + the interior of the embeddded boundary is where the function value is positive. + + - **From an STL file:** + In that case, you will need to set the following parameters in the input file. + + * ``eb2.stl_file`` (`string`) + The path to an STL file. In addition, you also need to set ``eb2.geom_type = stl``, + in order for the file to be read by WarpX. + +Whether the embedded boundary is defined with an analytical function or an STL file, you can +additionally define the electric potential at the embedded boundary with an analytical function: * ``warpx.eb_potential(x,y,z,t)`` (`string`) - Gives the value of the electric potential at the surface of the embedded boundary, + Gives the value of the electric potential, in Volts, at the surface of the embedded boundary, as a function of `x`, `y`, `z` and `t`. With electrostatic solvers (i.e., with ``warpx.do_electrostatic = ...``), this is used in order to compute the potential in the simulation volume at each timestep. When using other solvers (e.g. Maxwell solver), @@ -1052,8 +1156,8 @@ Particle initialization * ``gaussian_parse_momentum_function``: Gaussian momentum distribution where the mean and the standard deviation are given by functions of position in the input file. Both are assumed to be non-relativistic. The mean is the normalized momentum, :math:`u_m = \gamma v_m/c`. - The standard deviation is normalized, :math:`u_th = v_th/c`. - For example, this might be `u_th = sqrt(T*q_e/mass)/clight` given the temperature (in eV) and mass. + The standard deviation is normalized, :math:`u_{th} = v_{th}/c`. + For example, this might be ``u_th = sqrt(T*q_e/mass)/clight`` given the temperature (in eV) and mass. It requires the following arguments: * ``.momentum_function_ux_m(x,y,z)``: mean :math:`u_{x}` @@ -1643,8 +1747,8 @@ are applied to the grid directly. In particular, these fields can be seen in the One can refer to input files in ``Examples/Tests/LoadExternalField`` for more information. Regarding how to prepare the openPMD data file, one can refer to the `openPMD-example-datasets `__. - Note that if both `B_ext_grid_init_style` and `E_ext_grid_init_style` are set to - `read_from_file`, the openPMD file specified by `warpx.read_fields_from_path` + Note that if both ``B_ext_grid_init_style`` and ``E_ext_grid_init_style`` are set to + ``read_from_file``, the openPMD file specified by ``warpx.read_fields_from_path`` should contain both B and E external fields data. * ``warpx.E_external_grid`` & ``warpx.B_external_grid`` (list of `3 floats`) @@ -1700,6 +1804,21 @@ are applied to the particles directly, at each timestep. As a results, these fie Note that the position is defined in Cartesian coordinates, as a function of (x,y,z), even for RZ. + * ``read_from_file``: load the external field from an openPMD file. + An additional parameter, indicating the path of an openPMD data file, ``particles.read_fields_from_path`` + must be specified, from which the external E field data can be loaded into WarpX. + One can refer to input files in ``Examples/Tests/LoadExternalField`` for more information. + Regarding how to prepare the openPMD data file, one can refer to + the `openPMD-example-datasets `__. + Note that if both ``B_ext_particle_init_style`` and ``E_ext_particle_init_style`` are set to + ``read_from_file``, the openPMD file specified by ``particles.read_fields_from_path`` + should contain both B and E external fields data. + + .. note:: + + When using ``read_from_file``, the fields loaded from the file will be interpolated + to the resolution of the grid used for the simulation. + * ``repeated_plasma_lens``: apply a series of plasma lenses. The properties of the lenses are defined in the lab frame by the input parameters: @@ -2252,7 +2371,7 @@ Maxwell solver: PSATD method * ``psatd.v_comoving`` (3 floating-point values, in units of the speed of light; default ``0. 0. 0.``) Defines the comoving velocity in the comoving PSATD scheme. - A non-zero comoving velocity selects the comoving PSATD algorithm, which suppresses the numerical Cherenkov instability (NCI) in boosted-frame simulations, under certain assumptions. This option requires that WarpX is compiled with ``USE_PSATD = TRUE``. It also requires the use of direct current deposition (``algo.current_deposition = direct``) and has not been neither implemented nor tested with other current deposition schemes. + A non-zero comoving velocity selects the comoving PSATD algorithm, which suppresses the numerical Cherenkov instability (NCI) in boosted-frame simulations, under certain assumptions. This option requires that WarpX is compiled with ``USE_FFT = TRUE``. It also requires the use of direct current deposition (``algo.current_deposition = direct``) and has not been neither implemented nor tested with other current deposition schemes. * ``psatd.do_time_averaging`` (`0` or `1`; default: 0) Whether to use an averaged Galilean PSATD algorithm or standard Galilean PSATD. @@ -2342,7 +2461,7 @@ Grid types (collocated, staggered, hybrid) For example, :math:`E_z` is gathered using ``algo.particle_shape`` along :math:`(x,y)` and ``algo.particle_shape - 1`` along :math:`z`. See equations (21)-(23) of :cite:t:`param-Godfrey2013` and associated references for details. - Default: ``interpolation.galerkin_scheme = 0`` with collocated grids and/or momentum-conserving field gathering, ``interpolation.galerkin_scheme = 1`` otherwise. + Default: ``interpolation.galerkin_scheme = 0`` with collocated grids, or momentum-conserving field gathering, or when ``algo.current_deposition = direct`` ; ``interpolation.galerkin_scheme = 1`` otherwise. .. warning:: @@ -2604,7 +2723,9 @@ In-situ capabilities can be used by turning on Sensei or Ascent (provided they a * ``.fields_to_plot`` (list of `strings`, optional) Fields written to output. - Possible scalar fields: ``part_per_cell`` ``rho`` ``phi`` ``F`` ``part_per_grid`` ``divE`` ``divB`` and ``rho_``, where ```` must match the name of one of the available particle species. Note that ``phi`` will only be written out when do_electrostatic==labframe. Also, note that for ``.diag_type = BackTransformed``, the only scalar field currently supported is ``rho``. + Possible scalar fields: ``part_per_cell`` ``rho`` ``phi`` ``F`` ``part_per_grid`` ``divE`` ``divB`` ``rho_`` and ``T_``, where ```` must match the name of one of the available particle species. + ``T_`` is the temperature in eV. + Note that ``phi`` will only be written out when do_electrostatic==labframe. Also, note that for ``.diag_type = BackTransformed``, the only scalar field currently supported is ``rho``. Possible vector field components in Cartesian geometry: ``Ex`` ``Ey`` ``Ez`` ``Bx`` ``By`` ``Bz`` ``jx`` ``jy`` ``jz``. Possible vector field components in RZ geometry: ``Er`` ``Et`` ``Ez`` ``Br`` ``Bt`` ``Bz`` ``jr`` ``jt`` ``jz``. The default is ``.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz`` in Cartesian geometry and ``.fields_to_plot = Er Et Ez Br Bt Bz jr jt jz`` in RZ geometry. diff --git a/Docs/source/usage/python.rst b/Docs/source/usage/python.rst index 3aba5f165de..38b0a31d7f3 100644 --- a/Docs/source/usage/python.rst +++ b/Docs/source/usage/python.rst @@ -53,6 +53,30 @@ Field solvers define the updates of electric and magnetic fields. .. autoclass:: pywarpx.picmi.ElectrostaticSolver +Object that allows smoothing of fields. + +.. autoclass:: pywarpx.picmi.BinomialSmoother + +Evolve Schemes +-------------- + +These define the scheme use to evolve the fields and particles. +An instance of one of these would be passed as the `evolve_scheme` into the `Simulation`. + +.. autoclass:: pywarpx.picmi.ExplicitEvolveScheme + +.. autoclass:: pywarpx.picmi.ThetaImplicitEMEvolveScheme + +.. autoclass:: pywarpx.picmi.SemiImplicitEMEvolveScheme + +There are several support classes use to specify components of the evolve schemes + +.. autoclass:: pywarpx.picmi.PicardNonlinearSolver + +.. autoclass:: pywarpx.picmi.NewtonNonlinearSolver + +.. autoclass:: pywarpx.picmi.GMRESLinearSolver + Constants --------- @@ -77,6 +101,8 @@ Instances of the classes below need to be passed to the method `add_applied_fiel .. autoclass:: pywarpx.picmi.AnalyticAppliedField +.. autoclass:: pywarpx.picmi.LoadInitialField + .. autoclass:: pywarpx.picmi.PlasmaLens .. autoclass:: pywarpx.picmi.Mirror diff --git a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py index ac3398e43fc..13bf492e203 100755 --- a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py +++ b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_3d.py @@ -55,7 +55,9 @@ electrons = picmi.Species( particle_type = 'electron', name = 'electrons', - initial_distribution = uniform_distribution) + initial_distribution = uniform_distribution, + warpx_add_int_attributes = {'regionofinterest': "(z>12.0e-6) * (z<13.0e-6)"}, + warpx_add_real_attributes = {'initialenergy': "ux*ux + uy*uy + uz*uz"}) # Particles: beam electrons q_tot = 1e-12 diff --git a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py index 9990cca01cf..7f09db8d6b3 100755 --- a/Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py +++ b/Examples/Physics_applications/laser_acceleration/PICMI_inputs_rz.py @@ -31,7 +31,7 @@ upper_bound = [rmax, zmax], lower_boundary_conditions = ['none', 'dirichlet'], upper_boundary_conditions = ['dirichlet', 'dirichlet'], - lower_boundary_conditions_particles = ['absorbing', 'absorbing'], + lower_boundary_conditions_particles = ['none', 'absorbing'], upper_boundary_conditions_particles = ['absorbing', 'absorbing'], moving_window_velocity = [0., c], warpx_max_grid_size = max_grid_size, diff --git a/Examples/Physics_applications/laser_acceleration/inputs_3d b/Examples/Physics_applications/laser_acceleration/inputs_3d index bdcfd7676a4..fb8810dd34b 100644 --- a/Examples/Physics_applications/laser_acceleration/inputs_3d +++ b/Examples/Physics_applications/laser_acceleration/inputs_3d @@ -55,7 +55,7 @@ electrons.do_continuous_injection = 1 electrons.addIntegerAttributes = regionofinterest electrons.attribute.regionofinterest(x,y,z,ux,uy,uz,t) = "(z>12.0e-6) * (z<13.0e-6)" electrons.addRealAttributes = initialenergy -electrons.attribute.initialenergy(x,y,z,ux,uy,uz,t) = " ux*ux + uy*uy + uz*uz" +electrons.attribute.initialenergy(x,y,z,ux,uy,uz,t) = "ux*ux + uy*uy + uz*uz" ################################# ############ LASER ############# diff --git a/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py b/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py index 844501992c3..9f7a2aacfca 100755 --- a/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py +++ b/Examples/Physics_applications/laser_ion/PICMI_inputs_2d.py @@ -91,18 +91,17 @@ rms_velocity=[c*ux_th, 0., c*uz_th] # thermal velocity spread in m/s ) -# TODO: add additional attributes orig_x and orig_z electrons = picmi.Species( particle_type='electron', name='electrons', initial_distribution=slab_with_ramp_dist_electrons, ) -# TODO: add additional attributes orig_x and orig_z hydrogen = picmi.Species( particle_type='proton', name='hydrogen', - initial_distribution=slab_with_ramp_dist_hydrogen + initial_distribution=slab_with_ramp_dist_hydrogen, + warpx_add_real_attributes = {"orig_x": "x", "orig_z": "z"} ) # Laser diff --git a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py index d541c2393ca..5b57b59fbe3 100644 --- a/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py +++ b/Examples/Physics_applications/spacecraft_charging/PICMI_inputs_rz.py @@ -220,7 +220,7 @@ def compute_actual_charge_on_spacecraft(): upper_bound = [rmax, zmax], lower_boundary_conditions = ['none', 'dirichlet'], upper_boundary_conditions = ['dirichlet', 'dirichlet'], - lower_boundary_conditions_particles = ['absorbing', 'reflecting'], + lower_boundary_conditions_particles = ['none', 'reflecting'], upper_boundary_conditions_particles = ['absorbing', 'reflecting'] ) diff --git a/Examples/Tests/Implicit/PICMI_inputs_vandb_jfnk_2d.py b/Examples/Tests/Implicit/PICMI_inputs_vandb_jfnk_2d.py new file mode 100755 index 00000000000..2f919124e13 --- /dev/null +++ b/Examples/Tests/Implicit/PICMI_inputs_vandb_jfnk_2d.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +# +# --- Tests the python interface to the Implicit solver + +import numpy as np + +from pywarpx import picmi + +constants = picmi.constants + +########################## +# physics parameters +########################## + +n0 = 1.e30 # m^-3 +Ti = 100. # eV +Te = 100. # eV +wpe = constants.q_e*np.sqrt(n0/(constants.m_e*constants.ep0)) +de0 = constants.c/wpe +nppcz = 10 # number of particles/cell in z +dt = 0.1/wpe # s + +vthe = np.sqrt(Te*constants.q_e/constants.m_e) +vthi = np.sqrt(Ti*constants.q_e/constants.m_p) + +########################## +# numerics parameters +########################## + +# --- Number of time steps +max_steps = 20 +diagnostic_intervals = "::20" + +# --- Grid +nx = 40 +ny = 40 + +xmin = 0. +ymin = 0. +xmax = 10.0*de0 +ymax = 10.0*de0 + +number_per_cell_each_dim = [nppcz, nppcz] + +########################## +# physics components +########################## + +electrons_uniform_plasma = picmi.UniformDistribution(density = n0, + rms_velocity = [vthe, vthe, vthe]) + +electrons = picmi.Species(particle_type='electron', name='electrons', initial_distribution=electrons_uniform_plasma) + +protons_uniform_plasma = picmi.UniformDistribution(density = n0, + rms_velocity = [vthi, vthi, vthi]) + +protons = picmi.Species(particle_type='proton', name='protons', initial_distribution=protons_uniform_plasma) + +########################## +# numerics components +########################## + +grid = picmi.Cartesian2DGrid(number_of_cells = [nx, ny], + lower_bound = [xmin, ymin], + upper_bound = [xmax, ymax], + lower_boundary_conditions = ['periodic', 'periodic'], + upper_boundary_conditions = ['periodic', 'periodic'], + warpx_max_grid_size = 8, + warpx_blocking_factor = 8) + +solver = picmi.ElectromagneticSolver(grid = grid, + method = 'Yee') + +GMRES_solver = picmi.GMRESLinearSolver(verbose_int = 2, + max_iterations = 1000, + relative_tolerance = 1.0e-8, + absolute_tolerance = 0.0) + +newton_solver = picmi.NewtonNonlinearSolver(verbose = True, + max_iterations = 20, + relative_tolerance = 1.0e-12, + absolute_tolerance = 0.0, + require_convergence = False, + linear_solver = GMRES_solver, + max_particle_iterations = 21, + particle_tolerance = 1.0e-12) + +evolve_scheme = picmi.ThetaImplicitEMEvolveScheme(theta = 0.5, + nonlinear_solver = newton_solver) + +########################## +# diagnostics +########################## + +field_diag1 = picmi.FieldDiagnostic(name = 'diag1', + grid = grid, + period = diagnostic_intervals, + data_list = ['Ex', 'Ey', 'Ez', 'Bx', 'By', 'Bz', 'Jx', 'Jy', 'Jz', 'rho', 'divE'], + write_dir = '.', + warpx_file_prefix = 'ThetaImplicitJFNK_VandB_2d_PICMI_plt') + +part_diag1 = picmi.ParticleDiagnostic(name = 'diag1', + period = diagnostic_intervals, + species = [electrons, protons], + data_list = ['weighting', 'position', 'momentum'], + write_dir = '.', + warpx_file_prefix = 'ThetaImplicitJFNK_VandB_2d_PICMI_plt') + +particle_energy_diag = picmi.ReducedDiagnostic(diag_type = 'ParticleEnergy', + name = 'particle_energy', + period = 1) + +field_energy_diag = picmi.ReducedDiagnostic(diag_type = 'FieldEnergy', + name = 'field_energy', + period = 1) + +########################## +# simulation setup +########################## + +sim = picmi.Simulation(solver = solver, + particle_shape = 2, + time_step_size = dt, + max_steps = max_steps, + verbose = 1, + warpx_evolve_scheme = evolve_scheme, + warpx_current_deposition_algo = 'villasenor', + warpx_particle_pusher_algo = 'boris', + warpx_serialize_initial_conditions = 1, + warpx_use_filter = 0) + +sim.add_species(electrons, + layout = picmi.GriddedLayout(n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid)) +sim.add_species(protons, + layout = picmi.GriddedLayout(n_macroparticle_per_cell=number_per_cell_each_dim, grid=grid)) + +sim.add_diagnostic(field_diag1) +sim.add_diagnostic(part_diag1) +sim.add_diagnostic(particle_energy_diag) +sim.add_diagnostic(field_energy_diag) + +########################## +# simulation run +########################## + +# write_inputs will create an inputs file that can be used to run +# with the compiled version. +sim.write_input_file(file_name = 'inputs2d_from_PICMI') + +# Alternatively, sim.step will run WarpX, controlling it from Python +sim.step() diff --git a/Examples/Tests/Implicit/analysis_1d.py b/Examples/Tests/Implicit/analysis_1d.py index 0f00010a505..0e20b925df5 100755 --- a/Examples/Tests/Implicit/analysis_1d.py +++ b/Examples/Tests/Implicit/analysis_1d.py @@ -31,7 +31,7 @@ if re.match('SemiImplicitPicard_1d', fn): tolerance_rel = 2.5e-5 -elif re.match('ImplicitPicard_1d', fn): +elif re.match('ThetaImplicitPicard_1d', fn): # This case should have near machine precision conservation of energy tolerance_rel = 1.e-14 diff --git a/Examples/Tests/Implicit/analysis_vandb_2d.py b/Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py similarity index 100% rename from Examples/Tests/Implicit/analysis_vandb_2d.py rename to Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py diff --git a/Examples/Tests/Implicit/inputs_1d b/Examples/Tests/Implicit/inputs_1d index 465d9dd6965..3e57689b723 100644 --- a/Examples/Tests/Implicit/inputs_1d +++ b/Examples/Tests/Implicit/inputs_1d @@ -31,10 +31,18 @@ boundary.particle_hi = periodic ############ NUMERICS ########### ################################# +warpx.verbose = 1 warpx.const_dt = dt -algo.evolve_scheme = implicit_picard -algo.max_picard_iterations = 31 -algo.picard_iteration_tolerance = 0. +algo.evolve_scheme = theta_implicit_em + +implicit_evolve.nonlinear_solver = "picard" + +picard.verbose = true +picard.max_iterations = 31 +picard.relative_tolerance = 0.0 +picard.absolute_tolerance = 0.0 +picard.require_convergence = false + algo.current_deposition = esirkepov algo.field_gathering = energy-conserving algo.particle_shape = 2 diff --git a/Examples/Tests/Implicit/inputs_1d_semiimplicit b/Examples/Tests/Implicit/inputs_1d_semiimplicit index 4008a559588..07460e08be8 100644 --- a/Examples/Tests/Implicit/inputs_1d_semiimplicit +++ b/Examples/Tests/Implicit/inputs_1d_semiimplicit @@ -31,10 +31,18 @@ boundary.particle_hi = periodic ############ NUMERICS ########### ################################# +warpx.verbose = 1 warpx.const_dt = dt -algo.evolve_scheme = semi_implicit_picard -algo.max_picard_iterations = 5 -algo.picard_iteration_tolerance = 0. +algo.evolve_scheme = semi_implicit_em + +implicit_evolve.nonlinear_solver = "picard" + +picard.verbose = true +picard.max_iterations = 5 +picard.relative_tolerance = 0.0 +picard.absolute_tolerance = 0.0 +picard.require_convergence = false + algo.current_deposition = esirkepov algo.field_gathering = energy-conserving algo.particle_shape = 2 diff --git a/Examples/Tests/Implicit/inputs_vandb_2d b/Examples/Tests/Implicit/inputs_vandb_jfnk_2d similarity index 78% rename from Examples/Tests/Implicit/inputs_vandb_2d rename to Examples/Tests/Implicit/inputs_vandb_jfnk_2d index 33ce964710a..393a9d90330 100644 --- a/Examples/Tests/Implicit/inputs_vandb_2d +++ b/Examples/Tests/Implicit/inputs_vandb_jfnk_2d @@ -38,10 +38,32 @@ warpx.const_dt = dt warpx.use_filter = 0 algo.maxwell_solver = Yee -algo.evolve_scheme = "implicit_picard" -algo.require_picard_convergence = 0 -algo.max_picard_iterations = 25 -algo.picard_iteration_tolerance = 0.0 #1.0e-12 +algo.evolve_scheme = "theta_implicit_em" +#algo.evolve_scheme = "semi_implicit_em" + +implicit_evolve.theta = 0.5 +implicit_evolve.max_particle_iterations = 21 +implicit_evolve.particle_tolerance = 1.0e-12 + +#implicit_evolve.nonlinear_solver = "picard" +#picard.verbose = true +#picard.max_iterations = 25 +#picard.relative_tolerance = 0.0 #1.0e-12 +#picard.absolute_tolerance = 0.0 #1.0e-24 +#picard.require_convergence = false + +implicit_evolve.nonlinear_solver = "newton" +newton.verbose = true +newton.max_iterations = 20 +newton.relative_tolerance = 1.0e-12 +newton.absolute_tolerance = 0.0 +newton.require_convergence = false + +gmres.verbose_int = 2 +gmres.max_iterations = 1000 +gmres.relative_tolerance = 1.0e-8 +gmres.absolute_tolerance = 0.0 + algo.particle_pusher = "boris" #algo.particle_pusher = "higuera" diff --git a/Examples/Tests/LoadExternalField/PICMI_inputs_3d.py b/Examples/Tests/LoadExternalField/PICMI_inputs_3d_grid_fields.py similarity index 96% rename from Examples/Tests/LoadExternalField/PICMI_inputs_3d.py rename to Examples/Tests/LoadExternalField/PICMI_inputs_3d_grid_fields.py index 849d4d2be08..d128d9c10e0 100644 --- a/Examples/Tests/LoadExternalField/PICMI_inputs_3d.py +++ b/Examples/Tests/LoadExternalField/PICMI_inputs_3d_grid_fields.py @@ -93,7 +93,7 @@ species=[ions], data_list = ['ux', 'uy', 'uz', 'x', 'y', 'z', 'weighting'], write_dir='.', - warpx_file_prefix='Python_LoadExternalField3D_plt' + warpx_file_prefix='Python_LoadExternalGridField3D_plt' ) field_diag = picmi.FieldDiagnostic( name='diag1', @@ -101,7 +101,7 @@ period=300, data_list = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], write_dir='.', - warpx_file_prefix='Python_LoadExternalField3D_plt' + warpx_file_prefix='Python_LoadExternalGridField3D_plt' ) ################################# diff --git a/Examples/Tests/LoadExternalField/PICMI_inputs_3d_particle_fields.py b/Examples/Tests/LoadExternalField/PICMI_inputs_3d_particle_fields.py new file mode 100644 index 00000000000..7bf7c5a084c --- /dev/null +++ b/Examples/Tests/LoadExternalField/PICMI_inputs_3d_particle_fields.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +# +# --- Input file for loading initial field from openPMD file. + +from pywarpx import picmi + +constants = picmi.constants + +################################# +####### GENERAL PARAMETERS ###### +################################# + +max_steps = 300 + +max_grid_size = 40 +nx = max_grid_size +ny = max_grid_size +nz = max_grid_size + +xmin = -1 +xmax = 1 +ymin = xmin +ymax = xmax +zmin = 0 +zmax = 5 + +number_per_cell = 200 + +################################# +############ NUMERICS ########### +################################# + +verbose = 1 +dt = 4.4e-7 +use_filter = 0 + +# Order of particle shape factors +particle_shape = 1 + +################################# +############ PLASMA ############# +################################# + +ion_dist = picmi.ParticleListDistribution( + x=0.0, + y=0.2, + z=2.5, + ux=9.5e-05*constants.c, + uy=0.0*constants.c, + uz=1.34e-4*constants.c, + weight=1.0 + ) + +ions = picmi.Species( + particle_type='H', + name='proton', charge='q_e',mass="m_p", + warpx_do_not_deposit=1, + initial_distribution=ion_dist + ) + +################################# +######## INITIAL FIELD ########## +################################# + +applied_field = picmi.LoadAppliedField( + read_fields_from_path="../../../../openPMD-example-datasets/example-femm-3d.h5", + load_E=False + ) + +################################# +###### GRID AND SOLVER ########## +################################# + +grid = picmi.Cartesian3DGrid( + number_of_cells=[nx, ny, nz], + warpx_max_grid_size=max_grid_size, + lower_bound=[xmin, ymin, zmin], + upper_bound=[xmax, ymax, zmax], + lower_boundary_conditions=['dirichlet', 'dirichlet', 'dirichlet'], + upper_boundary_conditions=['dirichlet', 'dirichlet', 'dirichlet'], + lower_boundary_conditions_particles=['absorbing', 'absorbing', 'absorbing'], + upper_boundary_conditions_particles=['absorbing', 'absorbing', 'absorbing'] + ) +solver = picmi.ElectrostaticSolver(grid=grid) + +################################# +######### DIAGNOSTICS ########### +################################# + +particle_diag = picmi.ParticleDiagnostic( + name='diag1', + period=300, + species=[ions], + data_list = ['ux', 'uy', 'uz', 'x', 'y', 'z', 'weighting'], + write_dir='.', + warpx_file_prefix='Python_LoadExternalParticleField3D_plt' + ) +field_diag = picmi.FieldDiagnostic( + name='diag1', + grid=grid, + period=300, + data_list = ['Bx', 'By', 'Bz', 'Ex', 'Ey', 'Ez', 'Jx', 'Jy', 'Jz'], + write_dir='.', + warpx_file_prefix='Python_LoadExternalParticleField3D_plt' + ) + +################################# +####### SIMULATION SETUP ######## +################################# + +sim = picmi.Simulation( + solver=solver, + max_steps=max_steps, + verbose=verbose, + warpx_serialize_initial_conditions=False, + warpx_grid_type='collocated', + warpx_do_dynamic_scheduling=False, + warpx_use_filter=use_filter, + time_step_size=dt, + particle_shape=particle_shape + ) + +sim.add_applied_field(applied_field) + +sim.add_species( + ions, + layout=picmi.PseudoRandomLayout( + n_macroparticles_per_cell=number_per_cell, grid=grid + ) + ) + +sim.add_diagnostic(field_diag) +sim.add_diagnostic(particle_diag) + +################################# +##### SIMULATION EXECUTION ###### +################################# + +#sim.write_input_file('PICMI_inputs_3d') +sim.step(max_steps) diff --git a/Examples/Tests/LoadExternalField/inputs_rz b/Examples/Tests/LoadExternalField/inputs_rz_grid_fields similarity index 100% rename from Examples/Tests/LoadExternalField/inputs_rz rename to Examples/Tests/LoadExternalField/inputs_rz_grid_fields diff --git a/Examples/Tests/LoadExternalField/inputs_rz_particle_fields b/Examples/Tests/LoadExternalField/inputs_rz_particle_fields new file mode 100644 index 00000000000..b76d4cb7efc --- /dev/null +++ b/Examples/Tests/LoadExternalField/inputs_rz_particle_fields @@ -0,0 +1,65 @@ +warpx.serialize_initial_conditions = 0 +warpx.do_dynamic_scheduling = 0 +particles.do_tiling = 0 + +particles.B_ext_particle_init_style = "read_from_file" +particles.read_fields_from_path = "../../../../openPMD-example-datasets/example-femm-thetaMode.h5" + +warpx.grid_type = collocated +warpx.do_electrostatic = labframe + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 300 +amr.n_cell = 40 40 +warpx.numprocs = 1 1 +amr.max_level = 0 +geometry.dims = RZ + +geometry.prob_lo = 0.0 0.0 +geometry.prob_hi = 1.0 5.0 + +################################# +###### Boundary Condition ####### +################################# +boundary.field_lo = none pec +boundary.field_hi = pec pec +boundary.potential_lo_x = 0 +boundary.potential_hi_x = 0 +boundary.potential_lo_y = 0 +boundary.potential_hi_y = 0 +boundary.potential_lo_z = 0 +boundary.potential_hi_z = 0 + +################################# +############ NUMERICS ########### +################################# +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.const_dt = 4.40917904849092e-7 +warpx.use_filter = 0 + +# Order of particle shape factors +algo.particle_shape = 1 + +################################# +############ PLASMA ############# +################################# +particles.species_names = proton +proton.injection_style = "SingleParticle" +proton.single_particle_pos = 0.0 0.2 2.5 +proton.single_particle_u = 9.506735958279367e-05 0.0 0.00013435537232359165 +proton.single_particle_weight = 1.0 +proton.do_not_deposit = 1 +proton.mass = m_p +proton.charge = q_e + +# Diagnostics +diagnostics.diags_names = diag1 chk +diag1.intervals = 300 +diag1.diag_type = Full + +chk.intervals = 150 +chk.diag_type = Full +chk.format = checkpoint diff --git a/Examples/Tests/collision/analysis_collision_1d.py b/Examples/Tests/collision/analysis_collision_1d.py new file mode 100755 index 00000000000..7775a476dae --- /dev/null +++ b/Examples/Tests/collision/analysis_collision_1d.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 + +# Copyright 2024 Justin Angus +# +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL +# +# This is a script that analyses the simulation results from the script `inputs_1d`. +# run locally: python analysis_vandb_1d.py diags/diag1000600/ +# +# This is a 1D intra-species Coulomb scattering relaxation test consisting +# of a low-density population streaming into a higher density population at rest. +# Both populations belong to the same carbon12 ion species. +# See test T1b from JCP 413 (2020) by D. Higginson, et al. +# +import os +import sys + +import numpy as np +import yt +from scipy.constants import e + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +# this will be the name of the plot file +fn = sys.argv[1] +ds = yt.load(fn) +data = ds.covering_grid(level = 0, left_edge = ds.domain_left_edge, dims = ds.domain_dimensions) + +# carbon 12 ion (mass = 12*amu - 6*me) +mass = 1.992100316897910e-26 + +# Separate macroparticles from group A (low weight) and group B (high weight) +# by sorting based on weight +sorted_indices = data['ions','particle_weight'].argsort() +sorted_wp = data['ions', 'particle_weight'][sorted_indices].value +sorted_px = data['ions', 'particle_momentum_x'][sorted_indices].value +sorted_py = data['ions', 'particle_momentum_y'][sorted_indices].value +sorted_pz = data['ions', 'particle_momentum_z'][sorted_indices].value + +# Find the index 'Npmin' that separates macroparticles from group A and group B +Np = len(sorted_wp) +wpmin = sorted_wp.min(); +wpmax = sorted_wp.max(); +for i in range(len(sorted_wp)): + if sorted_wp[i] > wpmin: + Npmin = i + break + +NpA = Npmin +wpA = wpmin; +NpB = Np - Npmin +wpB = wpmax; +NpAs = 0 +NpAe = Npmin +NpBs = Npmin +NpBe = Np + +############# + +sorted_px_sum = np.abs(sorted_px).sum(); +sorted_py_sum = np.abs(sorted_py).sum(); +sorted_pz_sum = np.abs(sorted_pz).sum(); +sorted_wp_sum = np.abs(sorted_wp).sum(); + +# compute mean velocities +wAtot = wpA*NpA +wBtot = wpB*NpB + +uBx = uBy = uBz = 0. +for i in range(NpBs,NpBe): + uBx += wpB*sorted_px[i] + uBy += wpB*sorted_py[i] + uBz += wpB*sorted_pz[i] +uBx /= (mass*wBtot) # [m/s] +uBy /= (mass*wBtot) # [m/s] +uBz /= (mass*wBtot) # [m/s] + +uAx = uAy = uAz = 0. +for i in range(NpAs,NpAe): + uAx += wpA*sorted_px[i] + uAy += wpA*sorted_py[i] + uAz += wpA*sorted_pz[i] +uAx /= (mass*wAtot) # [m/s] +uAy /= (mass*wAtot) # [m/s] +uAz /= (mass*wAtot) # [m/s] + +# compute temperatures +TBx = TBy = TBz = 0. +for i in range(NpBs,NpBe): + TBx += wpB*(sorted_px[i]/mass - uBx)**2 + TBy += wpB*(sorted_py[i]/mass - uBy)**2 + TBz += wpB*(sorted_pz[i]/mass - uBz)**2 +TBx *= mass/(e*wBtot) +TBy *= mass/(e*wBtot) +TBz *= mass/(e*wBtot) + +TAx = TAy = TAz = 0. +for i in range(NpAs,NpAe): + TAx += wpA*(sorted_px[i]/mass - uAx)**2 + TAy += wpA*(sorted_py[i]/mass - uAy)**2 + TAz += wpA*(sorted_pz[i]/mass - uAz)**2 +TAx *= mass/(e*wAtot) +TAy *= mass/(e*wAtot) +TAz *= mass/(e*wAtot) + +TApar = TAz +TAperp = (TAx + TAy)/2.0 +TA = (TAx + TAy + TAz)/3.0 + +TBpar = TBz +TBperp = (TBx + TBy)/2.0 +TB = (TBx + TBy + TBz)/3.0 + +TApar_30ps_soln = 6.15e3 # TA parallel solution at t = 30 ps +error = np.abs(TApar-TApar_30ps_soln)/TApar_30ps_soln +tolerance = 0.02 +print('TApar at 30ps error = ', error); +print('tolerance = ', tolerance); +assert error < tolerance + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/collision/analysis_collision_2d.py b/Examples/Tests/collision/analysis_collision_2d.py index bd013898a8a..29482909b2e 100755 --- a/Examples/Tests/collision/analysis_collision_2d.py +++ b/Examples/Tests/collision/analysis_collision_2d.py @@ -114,4 +114,4 @@ dim, species_name) test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn, do_particles=False) +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/collision/analysis_collision_3d.py b/Examples/Tests/collision/analysis_collision_3d.py index ee0c81d8f75..86a434caab2 100755 --- a/Examples/Tests/collision/analysis_collision_3d.py +++ b/Examples/Tests/collision/analysis_collision_3d.py @@ -111,4 +111,4 @@ dim, species_name) test_name = os.path.split(os.getcwd())[1] -checksumAPI.evaluate_checksum(test_name, fn, do_particles=False) +checksumAPI.evaluate_checksum(test_name, fn) diff --git a/Examples/Tests/collision/inputs_1d b/Examples/Tests/collision/inputs_1d new file mode 100644 index 00000000000..b2de17192ae --- /dev/null +++ b/Examples/Tests/collision/inputs_1d @@ -0,0 +1,90 @@ +################################# +########## CONSTANTS ############ +################################# + +my_constants.nA = 1.e25 # m^-3 +my_constants.NpA = 400 # m^-3 +my_constants.UA = 6.55e5 # m/s +my_constants.TA = 500. # eV +# +my_constants.nB = 1.e26 # m^-3 +my_constants.NpB = 400 # m^-3 +my_constants.UB = 0. # m/s +my_constants.TB = 500. # eV +# +my_constants.q_c12 = 6.*q_e +my_constants.m_c12 = 12.*m_u - 6.*m_e + +################################# +####### GENERAL PARAMETERS ###### +################################# +max_step = 600 +amr.n_cell = 180 +amr.max_level = 0 +amr.blocking_factor = 4 +geometry.dims = 1 +geometry.prob_lo = 0. +geometry.prob_hi = 0.01 + +################################# +###### Boundary Condition ####### +################################# +boundary.field_lo = periodic +boundary.field_hi = periodic + +################################# +############ NUMERICS ########### +################################# +warpx.serialize_initial_conditions = 1 +warpx.verbose = 1 +warpx.const_dt = 0.05e-12 +warpx.use_filter = 0 + +# Do not evolve the E and B fields +algo.maxwell_solver = none + +# Order of particle shape factors +algo.particle_shape = 1 + +################################# +############ PLASMA ############# +################################# +particles.species_names = ions + +ions.charge = q_c12 +ions.mass = m_c12 +ions.do_not_deposit = 1 + +ions.injection_sources = groupA groupB + +ions.groupA.injection_style = "NUniformPerCell" +ions.groupA.num_particles_per_cell_each_dim = NpA +ions.groupA.profile = constant +ions.groupA.density = nA # number per m^3 +ions.groupA.momentum_distribution_type = "gaussian" +ions.groupA.uz_m = UA/clight +ions.groupA.ux_th = sqrt(TA*q_e/m_c12)/clight +ions.groupA.uy_th = sqrt(TA*q_e/m_c12)/clight +ions.groupA.uz_th = sqrt(TA*q_e/m_c12)/clight + +ions.groupB.injection_style = "NUniformPerCell" +ions.groupB.num_particles_per_cell_each_dim = NpB +ions.groupB.profile = constant +ions.groupB.density = nB # number per m^3 +ions.groupB.momentum_distribution_type = "gaussian" +ions.groupB.uz_m = UB/clight +ions.groupB.ux_th = sqrt(TB*q_e/m_c12)/clight +ions.groupB.uy_th = sqrt(TB*q_e/m_c12)/clight +ions.groupB.uz_th = sqrt(TB*q_e/m_c12)/clight + +################################# +############ COLLISION ########## +################################# +collisions.collision_names = collision1 +collision1.species = ions ions +collision1.CoulombLog = 10.0 + +# Diagnostics +diagnostics.diags_names = diag1 +diag1.intervals = 600 +diag1.diag_type = Full diff --git a/Examples/Tests/collision/inputs_3d b/Examples/Tests/collision/inputs_3d index 3cc06061bed..11d6b150dd2 100644 --- a/Examples/Tests/collision/inputs_3d +++ b/Examples/Tests/collision/inputs_3d @@ -22,6 +22,7 @@ boundary.field_hi = periodic periodic periodic warpx.serialize_initial_conditions = 1 warpx.verbose = 1 warpx.const_dt = 1.224744871e-07 +warpx.random_seed = 2034958209 # Do not evolve the E and B fields algo.maxwell_solver = none @@ -75,7 +76,7 @@ collision3.ndt = 10 diagnostics.diags_names = diag1 diag_parser_filter diag_uniform_filter diag_random_filter diag1.intervals = 10 diag1.diag_type = Full -diag1.fields_to_plot = Ex Ey Ez Bx By Bz +diag1.fields_to_plot = Ex Ey Ez Bx By Bz T_electron T_ion ## diag_parser_filter is a diag used to test the particle filter function. diag_parser_filter.intervals = 150:150: diff --git a/Examples/Tests/electrostatic_sphere_eb/analysis_rz_mr.py b/Examples/Tests/electrostatic_sphere_eb/analysis_rz_mr.py new file mode 100755 index 00000000000..0b01b128362 --- /dev/null +++ b/Examples/Tests/electrostatic_sphere_eb/analysis_rz_mr.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python + +# Copyright 2024 Olga Shapoval, Edoardo Zoni +# +# This file is part of WarpX. +# +# License: BSD-3-Clause-LBNL + +# This script tests the embedded boundary in RZ. +# A cylindrical surface (r=0.1) has a fixed potential 1 V. +# The outer surface has 0 V fixed. +# Thus the analytical solution has the form: +# phi(r) = A+B*log(r), Er(r) = -B/r. + +import os +import sys + +import numpy as np +from openpmd_viewer import OpenPMDTimeSeries + +sys.path.insert(1, '../../../../warpx/Regression/Checksum/') +import checksumAPI + +tolerance = 0.004 +print(f'tolerance = {tolerance}') + +fn = sys.argv[1] + +def find_first_non_zero_from_bottom_left(matrix): + for i in range(matrix.shape[0]): + for j in range(matrix.shape[1]): + if (matrix[i][j] != 0) and (matrix[i][j] != np.nan): + return (i, j) + return i, j + +def find_first_non_zero_from_upper_right(matrix): + for i in range(matrix.shape[0]-1, -1, -1): + for j in range(matrix.shape[1]-1, -1, -1): + if (matrix[i][j] != 0) and (matrix[i][j] != np.nan): + return (i, j) + return i,j + +def get_fields(ts, level): + if level == 0: + Er, info = ts.get_field('E', 'r', iteration=0) + phi, info = ts.get_field('phi', iteration=0) + else: + Er, info = ts.get_field(f'E_lvl{level}', 'r', iteration=0) + phi, info = ts.get_field(f'phi_lvl{level}', iteration=0) + return Er, phi, info + +def get_error_per_lev(ts,level): + Er, phi, info = get_fields(ts, level) + + nr_half = info.r.shape[0] // 2 + dr = info.dr + + Er_patch = Er[:,nr_half:] + phi_patch = phi[:,nr_half:] + r1 = info.r[nr_half:] + patch_left_lower_i, patch_left_lower_j = find_first_non_zero_from_bottom_left(Er_patch) + patch_right_upper_i, patch_right_upper_j = find_first_non_zero_from_upper_right(Er_patch) + + # phi and Er field on the MR patch + phi_sim = phi_patch[patch_left_lower_i:patch_right_upper_i+1, patch_left_lower_j:patch_right_upper_j+1] + Er_sim = Er_patch[patch_left_lower_i:patch_right_upper_i+1, patch_left_lower_j:patch_right_upper_j+1] + r = r1[patch_left_lower_j:patch_right_upper_j+1] + + B = 1.0/np.log(0.1/0.5) + A = -B*np.log(0.5) + + # outside EB and last cutcell + rmin = np.min(np.argwhere(r >= (0.1+dr))) + rmax = -1 + r = r[rmin:rmax] + phi_sim = phi_sim[:,rmin:rmax] + Er_sim = Er_sim[:,rmin:rmax] + + phi_theory = A + B*np.log(r) + phi_theory = np.tile(phi_theory, (phi_sim.shape[0],1)) + phi_error = np.max(np.abs(phi_theory-phi_sim) / np.abs(phi_theory)) + + Er_theory = -B/r + Er_theory = np.tile(Er_theory, (Er_sim.shape[0],1)) + Er_error = np.max(np.abs(Er_theory-Er_sim) / np.abs(Er_theory)) + + print(f'max error of phi[lev={level}]: {phi_error}') + print(f'max error of Er[lev={level}]: {Er_error}') + assert(phi_error < tolerance) + assert(Er_error < tolerance) + +ts = OpenPMDTimeSeries(fn) +level_fields = [field for field in ts.avail_fields if 'lvl' in field] +nlevels = 0 if level_fields == [] else int(level_fields[-1][-1]) +for level in range(nlevels+1): + get_error_per_lev(ts,level) + +test_name = os.path.split(os.getcwd())[1] +checksumAPI.evaluate_checksum(test_name, fn, output_format="openpmd") diff --git a/Examples/Tests/electrostatic_sphere_eb/inputs_rz_mr b/Examples/Tests/electrostatic_sphere_eb/inputs_rz_mr index 3bea63d76fb..722fc916416 100644 --- a/Examples/Tests/electrostatic_sphere_eb/inputs_rz_mr +++ b/Examples/Tests/electrostatic_sphere_eb/inputs_rz_mr @@ -30,3 +30,4 @@ diagnostics.diags_names = diag1 diag1.intervals = 1 diag1.diag_type = Full diag1.fields_to_plot = Er phi +diag1.format = openpmd diff --git a/Examples/Tests/embedded_circle/inputs_2d b/Examples/Tests/embedded_circle/inputs_2d index 7100f245917..7c76c1659da 100644 --- a/Examples/Tests/embedded_circle/inputs_2d +++ b/Examples/Tests/embedded_circle/inputs_2d @@ -5,7 +5,7 @@ max_step = 11 warpx.const_dt = 3.99e-13 warpx.do_electrostatic = labframe -warpx.self_fields_required_precision = 1e-06 +warpx.self_fields_required_precision = 2e-06 warpx.eb_implicit_function = -((x-0.00005)**2+(z-0.00005)**2-1e-05**2) warpx.eb_potential(x,y,z,t) = -10 warpx.self_fields_absolute_tolerance = 0.02 diff --git a/Examples/Tests/ionization/PICMI_inputs_2d.py b/Examples/Tests/ionization/PICMI_inputs_2d.py index a076361bf50..802bf5435ac 100644 --- a/Examples/Tests/ionization/PICMI_inputs_2d.py +++ b/Examples/Tests/ionization/PICMI_inputs_2d.py @@ -47,12 +47,14 @@ fill_in = True) electrons = picmi.Species( particle_type = 'electron', - name = 'electrons') + name = 'electrons', + warpx_add_real_attributes = {'orig_z': 'z'}) ions = picmi.Species( particle_type = 'N', name = 'ions', charge_state = 2, - initial_distribution = uniform_distribution) + initial_distribution = uniform_distribution, + warpx_add_real_attributes = {'orig_z': 'z'}) # Field ionization nitrogen_ionization = picmi.FieldIonization( @@ -88,7 +90,7 @@ name = 'diag1', period = 10000, species = [electrons, ions], - data_list = ['ux', 'uy', 'uz', 'x', 'z', 'weighting'], + data_list = ['ux', 'uy', 'uz', 'x', 'z', 'weighting', 'orig_z'], write_dir = '.', warpx_file_prefix = 'Python_ionization_plt') field_diag = picmi.FieldDiagnostic( diff --git a/Examples/Tests/ionization/analysis_ionization.py b/Examples/Tests/ionization/analysis_ionization.py index 95732b03e36..90657915b50 100755 --- a/Examples/Tests/ionization/analysis_ionization.py +++ b/Examples/Tests/ionization/analysis_ionization.py @@ -52,7 +52,7 @@ ad = ds.all_data() # Plot ions with ionization levels - species = 'ions'; + species = 'ions' xi = ad[species, 'particle_position_x'].v zi = ad[species, 'particle_position_y'].v ii = ad[species, 'particle_ionizationLevel'].v @@ -75,7 +75,7 @@ plt.colorbar() # Plot electrons - species = 'electrons'; + species = 'electrons' if species in [x[0] for x in ds.field_list]: xe = ad[species, 'particle_position_x'].v ze = ad[species, 'particle_position_y'].v @@ -96,10 +96,11 @@ # Check that the user runtime component (if it exists) worked as expected try: orig_z = ad['electrons', 'particle_orig_z'].v - assert np.all( (orig_z > 0) & (orig_z < 1.5e-5) ) + print(f"orig_z: min = {np.min(orig_z)}, max = {np.max(orig_z)}") + assert np.all( (orig_z > 0.0) & (orig_z < 1.5e-5) ) print('particle_orig_z has reasonable values') except yt.utilities.exceptions.YTFieldNotFound: - pass # Some of the tested script to not have the quantity orig_z + pass # The backtransformed diagnostic version of the test does not have orig_z test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, filename) diff --git a/Examples/Tests/ionization/inputs_2d_bf_rt b/Examples/Tests/ionization/inputs_2d_bf_rt index 52af5d0d40f..8bcb66595d2 100644 --- a/Examples/Tests/ionization/inputs_2d_bf_rt +++ b/Examples/Tests/ionization/inputs_2d_bf_rt @@ -48,7 +48,7 @@ electrons.zmax = 50.e-6 electrons.profile = constant electrons.density = 2. electrons.momentum_distribution_type = at_rest -electrons.do_continuous_injection=1 +electrons.do_continuous_injection = 1 lasers.names = laser1 laser1.profile = Gaussian diff --git a/Examples/Tests/langmuir/PICMI_inputs_rz.py b/Examples/Tests/langmuir/PICMI_inputs_rz.py index 8328aba5185..8da03b00469 100755 --- a/Examples/Tests/langmuir/PICMI_inputs_rz.py +++ b/Examples/Tests/langmuir/PICMI_inputs_rz.py @@ -90,7 +90,7 @@ upper_bound = [rmax, zmax], lower_boundary_conditions = ['none', 'periodic'], upper_boundary_conditions = ['none', 'periodic'], - lower_boundary_conditions_particles = ['absorbing', 'periodic'], + lower_boundary_conditions_particles = ['none', 'periodic'], upper_boundary_conditions_particles = ['absorbing', 'periodic'], moving_window_velocity = [0.,0.], warpx_max_grid_size=64) diff --git a/Examples/Tests/langmuir/inputs_rz b/Examples/Tests/langmuir/inputs_rz index 534296759a2..3b70a91f207 100644 --- a/Examples/Tests/langmuir/inputs_rz +++ b/Examples/Tests/langmuir/inputs_rz @@ -28,6 +28,8 @@ geometry.prob_lo = 0.e-6 -20.e-6 # physical domain geometry.prob_hi = 20.e-6 20.e-6 boundary.field_lo = none periodic boundary.field_hi = none periodic +boundary.particle_lo = none periodic +boundary.particle_hi = absorbing periodic warpx.serialize_initial_conditions = 1 diff --git a/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test b/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test index adfde819531..ffc1865ee0f 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test +++ b/Examples/Tests/laser_injection_from_file/inputs.1d_boost_test @@ -47,7 +47,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.1d_test b/Examples/Tests/laser_injection_from_file/inputs.1d_test index 6124b4552a2..6c392883418 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.1d_test +++ b/Examples/Tests/laser_injection_from_file/inputs.1d_test @@ -40,7 +40,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.2d_test b/Examples/Tests/laser_injection_from_file/inputs.2d_test index a3003a4978d..e5814471753 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.2d_test +++ b/Examples/Tests/laser_injection_from_file/inputs.2d_test @@ -40,7 +40,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.3d_test b/Examples/Tests/laser_injection_from_file/inputs.3d_test index e3096bf92d9..ad8159cb650 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.3d_test +++ b/Examples/Tests/laser_injection_from_file/inputs.3d_test @@ -40,7 +40,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.RZ_test b/Examples/Tests/laser_injection_from_file/inputs.RZ_test index fe5bb675212..2a539883fec 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.RZ_test +++ b/Examples/Tests/laser_injection_from_file/inputs.RZ_test @@ -41,7 +41,7 @@ lasy_laser.e_max = 1.e14 # Maximum amplitude of the laser field (i lasy_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_laser.profile = from_file lasy_laser.time_chunk_size = 50 -lasy_laser.lasy_file_name = "gaussianlaser3d_00000.h5" +lasy_laser.lasy_file_name = "diags/gaussianlaser3d_00000.h5" lasy_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test b/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test index ca50430c21a..f92440188b7 100644 --- a/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test +++ b/Examples/Tests/laser_injection_from_file/inputs.from_RZ_file_test @@ -40,7 +40,7 @@ lasy_RZ_laser.e_max = 1.e14 # Maximum amplitude of the laser field lasy_RZ_laser.wavelength = 1.0e-6 # The wavelength of the laser (in meters) lasy_RZ_laser.profile = from_file lasy_RZ_laser.time_chunk_size = 50 -lasy_RZ_laser.lasy_file_name = "laguerrelaserRZ_00000.h5" +lasy_RZ_laser.lasy_file_name = "diags/laguerrelaserRZ_00000.h5" lasy_RZ_laser.delay = 0.0 # Diagnostics diff --git a/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py b/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py index 1268f7a02b0..e46f561f538 100755 --- a/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py +++ b/Examples/Tests/magnetostatic_eb/PICMI_inputs_rz.py @@ -69,7 +69,7 @@ upper_bound = [rmax, zmax], lower_boundary_conditions = ['none', 'dirichlet'], upper_boundary_conditions = ['neumann', 'neumann'], - lower_boundary_conditions_particles = ['reflecting', 'absorbing'], + lower_boundary_conditions_particles = ['none', 'absorbing'], upper_boundary_conditions_particles = ['absorbing', 'absorbing'], warpx_potential_lo_z = V_domain_boundary, warpx_blocking_factor=8, diff --git a/Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py b/Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py index 70178fe97a6..858df0b26a6 100755 --- a/Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py +++ b/Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py @@ -47,7 +47,9 @@ sigma_th = 2.603e-18 error = np.abs(sigma-sigma_th)/sigma_th -tolerance = 1e-2 +tolerance = 2e-2 +print('error = ', error) +print('tolerance = ', tolerance) assert error < tolerance # Compare checksums with benchmark diff --git a/Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d_intraspecies b/Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d_intraspecies index 985cc2d7612..0030ca9b945 100644 --- a/Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d_intraspecies +++ b/Examples/Tests/nuclear_fusion/inputs_deuterium_deuterium_3d_intraspecies @@ -8,6 +8,8 @@ # plasma simulations. Journal of Computational Physics, 388, pp.439-453. # DOI: https://doi.org/10.1016/j.jcp.2019.03.020 +warpx.random_seed = 1034958209 + # algo algo.particle_shape = 3 diff --git a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py index 89c5230a6ad..5bd1e3518f9 100644 --- a/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py +++ b/Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py @@ -156,7 +156,7 @@ def setup_run(self): upper_bound=[self.Lr, self.Lz/2.0], lower_boundary_conditions = ['none', 'periodic'], upper_boundary_conditions = ['dirichlet', 'periodic'], - lower_boundary_conditions_particles = ['absorbing', 'periodic'], + lower_boundary_conditions_particles = ['none', 'periodic'], upper_boundary_conditions_particles = ['reflecting', 'periodic'] ) simulation.time_step_size = self.dt diff --git a/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py b/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py index 4017a05d413..df4a4579e2f 100644 --- a/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py +++ b/Examples/Tests/particle_boundary_interaction/PICMI_inputs_rz.py @@ -39,7 +39,7 @@ upper_bound = [rmax, zmax], lower_boundary_conditions = ['none', 'dirichlet'], upper_boundary_conditions = ['dirichlet', 'dirichlet'], - lower_boundary_conditions_particles = ['absorbing', 'reflecting'], + lower_boundary_conditions_particles = ['none', 'reflecting'], upper_boundary_conditions_particles = ['absorbing', 'reflecting'] ) diff --git a/Examples/Tests/repelling_particles/analysis_repelling.py b/Examples/Tests/repelling_particles/analysis_repelling.py index ebff010fc40..bda3d74d274 100755 --- a/Examples/Tests/repelling_particles/analysis_repelling.py +++ b/Examples/Tests/repelling_particles/analysis_repelling.py @@ -36,7 +36,7 @@ # Check plotfile name specified in command line last_filename = sys.argv[1] -filename_radical = re.findall('(.*?)\d+/*$', last_filename)[0] +filename_radical = re.findall(r'(.*?)\d+/*$', last_filename)[0] # Loop through files, and extract the position and velocity of both particles x1 = [] @@ -48,10 +48,10 @@ ds = yt.load(filename) ad = ds.all_data() - x1.append( float(ad[('electron1','particle_position_x')]) ) - x2.append( float(ad[('electron2','particle_position_x')]) ) - beta1.append( float(ad[('electron1','particle_momentum_x')])/(m_e*c) ) - beta2.append( float(ad[('electron2','particle_momentum_x')])/(m_e*c) ) + x1.append( float(ad[('electron1','particle_position_x')][0]) ) + x2.append( float(ad[('electron2','particle_position_x')][0]) ) + beta1.append( float(ad[('electron1','particle_momentum_x')][0])/(m_e*c) ) + beta2.append( float(ad[('electron2','particle_momentum_x')][0])/(m_e*c) ) # Convert to numpy array x1 = np.array(x1) diff --git a/Examples/Tests/scraping/analysis_rz.py b/Examples/Tests/scraping/analysis_rz.py index 193b2575992..11d0194707f 100755 --- a/Examples/Tests/scraping/analysis_rz.py +++ b/Examples/Tests/scraping/analysis_rz.py @@ -60,6 +60,14 @@ def n_scraped_particles( iteration ): n_total = n_remaining[0] assert np.all( n_scraped+n_remaining == n_total) +# Check that the particle IDs match between the initial iteration +# (all particles in the simulation domain) and the finall iteration (particles are either scraped or still in simulation box) +id_initial, = ts_full.get_particle(['id'], iteration=0) +id_final_scrape, = ts_scraping.get_particle(['id'], iteration=ts_scraping.iterations[0]) +id_final_box, = ts_full.get_particle(['id'], iteration=ts_full.iterations[-1]) +id_final = np.concatenate( (id_final_scrape, id_final_box)) +assert np.all( np.sort(id_initial) == np.sort(id_final) ) # Sort because particles may not be in the same order + # Checksum test test_name = os.path.split(os.getcwd())[1] checksumAPI.evaluate_checksum(test_name, fn, do_particles=False) diff --git a/GNUmakefile b/GNUmakefile index 8c657025740..a332c9cc04e 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -38,8 +38,7 @@ USE_OPENPMD = FALSE WarpxBinDir = Bin -USE_PSATD = FALSE -USE_PSATD_PICSAR = FALSE +USE_FFT = FALSE USE_RZ = FALSE USE_EB = FALSE diff --git a/Python/pywarpx/Bucket.py b/Python/pywarpx/Bucket.py index 91a34b9d3d6..9dbe4def88e 100644 --- a/Python/pywarpx/Bucket.py +++ b/Python/pywarpx/Bucket.py @@ -59,8 +59,6 @@ def attrlist(self): for attr, value in self.argvattrs.items(): if value is None: continue - # --- repr is applied to value so that for floats, all of the digits are included. - # --- The strip of "'" is then needed when value is a string. if isinstance(value, str): if value.find('=') > -1: # --- Expressions with temporary variables need to be inside quotes @@ -73,11 +71,11 @@ def attrlist(self): continue # --- For lists, tuples, and arrays make a space delimited string of the values. # --- The lambda is needed in case this is a list of strings. - rhs = ' '.join(map(lambda s : repr(s).strip("'"), value)) + rhs = ' '.join(map(lambda s : f'{s}', value)) elif isinstance(value, bool): rhs = 1 if value else 0 else: rhs = value - attrstring = '{0}.{1} = {2}'.format(self.instancename, attr, repr(rhs).strip("'")) + attrstring = f'{self.instancename}.{attr} = {rhs}' result += [attrstring] return result diff --git a/Python/pywarpx/WarpX.py b/Python/pywarpx/WarpX.py index 6b24b35e918..6752b00f371 100644 --- a/Python/pywarpx/WarpX.py +++ b/Python/pywarpx/WarpX.py @@ -89,8 +89,19 @@ def create_argv_list(self, **kw): for diagnostic in reduced_diagnostics._diagnostics_dict.values(): argv += diagnostic.attrlist() + for bucket in self._bucket_dict.values(): + argv += bucket.attrlist() + return argv + def get_bucket(self, bucket_name): + try: + return self._bucket_dict[bucket_name] + except KeyError: + bucket = Bucket(bucket_name) + self._bucket_dict[bucket_name] = bucket + return bucket + def init(self, mpi_comm=None, **kw): # note: argv[0] needs to be an absolute path so it works with AMReX backtraces # https://github.com/AMReX-Codes/amrex/issues/3435 @@ -130,4 +141,4 @@ def write_inputs(self, filename='inputs', **kw): ff.write(f'{arg}\n') -warpx = WarpX('warpx') +warpx = WarpX('warpx', _bucket_dict = {}) diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index e7575f83b26..8be6d9c6212 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -137,6 +137,9 @@ class Species(picmistandard.PICMI_Species): Cells with fewer particles than this number will be skipped during resampling. + warpx_resampling_algorithm_target_weight: float + Weight that the product particles from resampling will not exceed. + warpx_resampling_trigger_intervals: bool, default=0 Timesteps at which to resample @@ -168,6 +171,14 @@ class Species(picmistandard.PICMI_Species): during grid-based merging, with `velocity_grid_type == "cartesian"`. If a single number is given the same du value will be used in all three directions. + + warpx_add_int_attributes: dict + Dictionary of extra integer particle attributes initialized from an + expression that is a function of the variables (x, y, z, ux, uy, uz, t). + + warpx_add_real_attributes: dict + Dictionary of extra real particle attributes initialized from an + expression that is a function of the variables (x, y, z, ux, uy, uz, t). """ def init(self, kw): @@ -249,6 +260,7 @@ def init(self, kw): self.resampling_min_ppc = kw.pop('warpx_resampling_min_ppc', None) self.resampling_trigger_intervals = kw.pop('warpx_resampling_trigger_intervals', None) self.resampling_triggering_max_avg_ppc = kw.pop('warpx_resampling_trigger_max_avg_ppc', None) + self.resampling_algorithm_target_weight = kw.pop('warpx_resampling_algorithm_target_weight', None) self.resampling_algorithm_velocity_grid_type = kw.pop('warpx_resampling_algorithm_velocity_grid_type', None) self.resampling_algorithm_delta_ur = kw.pop('warpx_resampling_algorithm_delta_ur', None) self.resampling_algorithm_n_theta = kw.pop('warpx_resampling_algorithm_n_theta', None) @@ -257,6 +269,10 @@ def init(self, kw): if self.resampling_algorithm_delta_u is not None and np.size(self.resampling_algorithm_delta_u) == 1: self.resampling_algorithm_delta_u = [self.resampling_algorithm_delta_u]*3 + # extra particle attributes + self.extra_int_attributes = kw.pop('warpx_add_int_attributes', None) + self.extra_real_attributes = kw.pop('warpx_add_real_attributes', None) + def species_initialize_inputs(self, layout, initialize_self_fields = False, injection_plane_position = None, @@ -298,6 +314,7 @@ def species_initialize_inputs(self, layout, resampling_min_ppc=self.resampling_min_ppc, resampling_trigger_intervals=self.resampling_trigger_intervals, resampling_trigger_max_avg_ppc=self.resampling_triggering_max_avg_ppc, + resampling_algorithm_target_weight=self.resampling_algorithm_target_weight, resampling_algorithm_velocity_grid_type=self.resampling_algorithm_velocity_grid_type, resampling_algorithm_delta_ur=self.resampling_algorithm_delta_ur, resampling_algorithm_n_theta=self.resampling_algorithm_n_theta, @@ -313,6 +330,16 @@ def species_initialize_inputs(self, layout, self.species.add_new_attr("reflection_model_zhi(E)", self.reflection_model_zhi) # self.species.add_new_attr("reflection_model_eb(E)", self.reflection_model_eb) + # extra particle attributes + if self.extra_int_attributes is not None: + self.species.addIntegerAttributes = self.extra_int_attributes.keys() + for attr, function in self.extra_int_attributes.items(): + self.species.add_new_attr('attribute.'+attr+'(x,y,z,ux,uy,uz,t)', function) + if self.extra_real_attributes is not None: + self.species.addRealAttributes = self.extra_real_attributes.keys() + for attr, function in self.extra_real_attributes.items(): + self.species.add_new_attr('attribute.'+attr+'(x,y,z,ux,uy,uz,t)', function) + pywarpx.Particles.particles_list.append(self.species) if self.initial_distribution is not None: @@ -1206,6 +1233,194 @@ def solver_initialize_inputs(self): pywarpx.warpx.do_pml_j_damping = self.do_pml_j_damping +class ExplicitEvolveScheme(picmistandard.base._ClassWithInit): + """ + Sets up the explicit evolve scheme + """ + def solver_scheme_initialize_inputs(self): + pywarpx.algo.evolve_scheme = 'explicit' + + +class ThetaImplicitEMEvolveScheme(picmistandard.base._ClassWithInit): + """ + Sets up the "theta implicit" electromagnetic evolve scheme + + Parameters + ---------- + nonlinear_solver: nonlinear solver instance + The nonlinear solver to use for the iterations + + theta: float, optional + The "theta" parameter, determining the level of implicitness + """ + def __init__(self, nonlinear_solver, theta = None): + self.nonlinear_solver = nonlinear_solver + self.theta = theta + + def solver_scheme_initialize_inputs(self): + pywarpx.algo.evolve_scheme = 'theta_implicit_em' + implicit_evolve = pywarpx.warpx.get_bucket('implicit_evolve') + implicit_evolve.theta = self.theta + + self.nonlinear_solver.nonlinear_solver_initialize_inputs() + + +class SemiImplicitEMEvolveScheme(picmistandard.base._ClassWithInit): + """ + Sets up the "semi-implicit" electromagnetic evolve scheme + + Parameters + ---------- + nonlinear_solver: nonlinear solver instance + The nonlinear solver to use for the iterations + """ + def __init__(self, nonlinear_solver): + self.nonlinear_solver = nonlinear_solver + + def solver_scheme_initialize_inputs(self): + pywarpx.algo.evolve_scheme = 'semi_implicit_em' + + self.nonlinear_solver.nonlinear_solver_initialize_inputs() + + +class PicardNonlinearSolver(picmistandard.base._ClassWithInit): + """ + Sets up the iterative Picard nonlinear solver for the implicit evolve scheme + + Parameters + ---------- + verbose: bool, default=True + Whether there is verbose output from the solver + + absolute_tolerance: float, default=0. + Absoluate tolerence of the convergence + + relative_tolerance: float, default=1.e-6 + Relative tolerance of the convergence + + max_iterations: integer, default=100 + Maximum number of iterations + + require_convergence: bool, default True + Whether convergence is required. If True and convergence is not obtained, the code will exit. + """ + def __init__(self, verbose=None, absolute_tolerance=None, relative_tolerance=None, + max_iterations=None, require_convergence=None): + self.verbose = verbose + self.absolute_tolerance = absolute_tolerance + self.relative_tolerance = relative_tolerance + self.max_iterations = max_iterations + self.require_convergence = require_convergence + + def nonlinear_solver_initialize_inputs(self): + implicit_evolve = pywarpx.warpx.get_bucket('implicit_evolve') + implicit_evolve.nonlinear_solver = 'picard' + + picard = pywarpx.warpx.get_bucket('picard') + picard.verbose = self.verbose + picard.absolute_tolerance = self.absolute_tolerance + picard.relative_tolerance = self.relative_tolerance + picard.max_iterations = self.max_iterations + picard.require_convergence = self.require_convergence + + +class NewtonNonlinearSolver(picmistandard.base._ClassWithInit): + """ + Sets up the iterative Newton nonlinear solver for the implicit evolve scheme + + Parameters + ---------- + verbose: bool, default=True + Whether there is verbose output from the solver + + absolute_tolerance: float, default=0. + Absoluate tolerence of the convergence + + relative_tolerance: float, default=1.e-6 + Relative tolerance of the convergence + + max_iterations: integer, default=100 + Maximum number of iterations + + require_convergence: bool, default True + Whether convergence is required. If True and convergence is not obtained, the code will exit. + + linear_solver: linear solver instance, optional + Specifies input arguments to the linear solver + + max_particle_iterations: integer, optional + The maximum number of particle iterations + + particle_tolerance: float, optional + The tolerance of parrticle quantities for convergence + + """ + def __init__(self, verbose=None, absolute_tolerance=None, relative_tolerance=None, + max_iterations=None, require_convergence=None, linear_solver=None, + max_particle_iterations=None, particle_tolerance=None): + self.verbose = verbose + self.absolute_tolerance = absolute_tolerance + self.relative_tolerance = relative_tolerance + self.max_iterations = max_iterations + self.require_convergence = require_convergence + self.linear_solver = linear_solver + self.max_particle_iterations = max_particle_iterations + self.particle_tolerance = particle_tolerance + + def nonlinear_solver_initialize_inputs(self): + implicit_evolve = pywarpx.warpx.get_bucket('implicit_evolve') + implicit_evolve.nonlinear_solver = 'newton' + implicit_evolve.max_particle_iterations = self.max_particle_iterations + implicit_evolve.particle_tolerance = self.particle_tolerance + + newton = pywarpx.warpx.get_bucket('newton') + newton.verbose = self.verbose + newton.absolute_tolerance = self.absolute_tolerance + newton.relative_tolerance = self.relative_tolerance + newton.max_iterations = self.max_iterations + newton.require_convergence = self.require_convergence + + self.linear_solver.linear_solver_initialize_inputs() + + +class GMRESLinearSolver(picmistandard.base._ClassWithInit): + """ + Sets up the iterative GMRES linear solver for the implicit Newton nonlinear solver + + Parameters + ---------- + verbose_int: integer, default=2 + Level of verbosity of output + + restart_length: integer, default=30 + How often to restart the GMRES iterations + + absolute_tolerance: float, default=0. + Absoluate tolerence of the convergence + + relative_tolerance: float, default=1.e-4 + Relative tolerance of the convergence + + max_iterations: integer, default=1000 + Maximum number of iterations + """ + def __init__(self, verbose_int=None, restart_length=None, absolute_tolerance=None, relative_tolerance=None, + max_iterations=None): + self.verbose_int = verbose_int + self.restart_length = restart_length + self.absolute_tolerance = absolute_tolerance + self.relative_tolerance = relative_tolerance + self.max_iterations = max_iterations + + def linear_solver_initialize_inputs(self): + gmres = pywarpx.warpx.get_bucket('gmres') + gmres.verbose_int = self.verbose_int + gmres.restart_length = self.restart_length + gmres.absolute_tolerance = self.absolute_tolerance + gmres.relative_tolerance = self.relative_tolerance + gmres.max_iterations = self.max_iterations + + class HybridPICSolver(picmistandard.base._ClassWithInit): """ Hybrid-PIC solver based on Ohm's law. @@ -1324,6 +1539,9 @@ def init(self, kw): def solver_initialize_inputs(self): + # Open BC means FieldBoundaryType::Open for electrostatic sims, rather than perfectly-matched layer + BC_map['open'] = 'open' + self.grid.grid_initialize_inputs() if self.relativistic: @@ -1344,6 +1562,8 @@ def solver_initialize_inputs(self): pywarpx.boundary.potential_hi_y = self.grid.potential_ymax pywarpx.boundary.potential_hi_z = self.grid.potential_zmax + pywarpx.warpx.poisson_solver = self.method + class GaussianLaser(picmistandard.PICMI_GaussianLaser): def laser_initialize_inputs(self): @@ -1459,6 +1679,13 @@ def applied_field_initialize_inputs(self): expression = pywarpx.my_constants.mangle_expression(expression, self.mangle_dict) pywarpx.warpx.__setattr__(f'B{sdir}_external_grid_function(x,y,z)', expression) +class LoadAppliedField(picmistandard.PICMI_LoadAppliedField): + def applied_field_initialize_inputs(self): + pywarpx.particles.read_fields_from_path = self.read_fields_from_path + if self.load_E: + pywarpx.particles.E_ext_particle_init_style = 'read_from_file' + if self.load_B: + pywarpx.particles.B_ext_particle_init_style = 'read_from_file' class ConstantAppliedField(picmistandard.PICMI_ConstantAppliedField): def applied_field_initialize_inputs(self): @@ -1844,6 +2071,9 @@ class Simulation(picmistandard.PICMI_Simulation): Parameters ---------- + warpx_evolve_scheme: solver scheme instance, optional + Which evolve scheme to use + warpx_current_deposition_algo: {'direct', 'esirkepov', and 'vay'}, optional Current deposition algorithm. The default depends on conditions. @@ -2007,6 +2237,7 @@ class Simulation(picmistandard.PICMI_Simulation): def init(self, kw): + self.evolve_scheme = kw.pop('warpx_evolve_scheme', None) self.current_deposition_algo = kw.pop('warpx_current_deposition_algo', None) self.charge_deposition_algo = kw.pop('warpx_charge_deposition_algo', None) self.field_gathering_algo = kw.pop('warpx_field_gathering_algo', None) @@ -2074,6 +2305,9 @@ def initialize_inputs(self): pywarpx.warpx.sort_idx_type = self.sort_idx_type pywarpx.warpx.sort_bin_size = self.sort_bin_size + if self.evolve_scheme is not None: + self.evolve_scheme.solver_scheme_initialize_inputs() + pywarpx.algo.current_deposition = self.current_deposition_algo pywarpx.algo.charge_deposition = self.charge_deposition_algo pywarpx.algo.field_gathering = self.field_gathering_algo @@ -2290,6 +2524,11 @@ class FieldDiagnostic(picmistandard.PICMI_FieldDiagnostic, WarpXDiagnosticBase): warpx_openpmd_backend: {bp, h5, json}, optional Openpmd backend file format + warpx_openpmd_encoding: 'v' (variable based), 'f' (file based) or 'g' (group based), optional + Only read if ``.format = openpmd``. openPMD file output encoding. + File based: one file per timestep (slower), group/variable based: one file for all steps (faster)). + Variable based is an experimental feature with ADIOS2. Default: `'f'`. + warpx_file_prefix: string, optional Prefix on the diagnostic file name @@ -2299,6 +2538,9 @@ class FieldDiagnostic(picmistandard.PICMI_FieldDiagnostic, WarpXDiagnosticBase): warpx_dump_rz_modes: bool, optional Flag whether to dump the data for all RZ modes + warpx_dump_last_timestep: bool, optional + If true, the last timestep is dumped regardless of the diagnostic period/intervals. + warpx_particle_fields_to_plot: list of ParticleFieldDiagnostics List of ParticleFieldDiagnostic classes to install in the simulation. Error checking is handled in the class itself. @@ -2316,9 +2558,11 @@ def init(self, kw): self.plot_crsepatch = kw.pop('warpx_plot_crsepatch', None) self.format = kw.pop('warpx_format', 'plotfile') self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) + self.openpmd_encoding = kw.pop('warpx_openpmd_encoding', None) self.file_prefix = kw.pop('warpx_file_prefix', None) self.file_min_digits = kw.pop('warpx_file_min_digits', None) self.dump_rz_modes = kw.pop('warpx_dump_rz_modes', None) + self.dump_last_timestep = kw.pop('warpx_dump_last_timestep', None) self.particle_fields_to_plot = kw.pop('warpx_particle_fields_to_plot', []) self.particle_fields_species = kw.pop('warpx_particle_fields_species', None) @@ -2329,8 +2573,10 @@ def diagnostic_initialize_inputs(self): self.diagnostic.diag_type = 'Full' self.diagnostic.format = self.format self.diagnostic.openpmd_backend = self.openpmd_backend + self.diagnostic.openpmd_encoding = self.openpmd_encoding self.diagnostic.file_min_digits = self.file_min_digits self.diagnostic.dump_rz_modes = self.dump_rz_modes + self.diagnostic.dump_last_timestep = self.dump_last_timestep self.diagnostic.intervals = self.period self.diagnostic.diag_lo = self.lower_bound self.diagnostic.diag_hi = self.upper_bound @@ -2384,6 +2630,9 @@ def diagnostic_initialize_inputs(self): elif dataname.startswith('rho_'): # Adds rho_species diagnostic fields_to_plot.add(dataname) + elif dataname.startswith('T_'): + # Adds T_species diagnostic + fields_to_plot.add(dataname) elif dataname == 'dive': fields_to_plot.add('divE') elif dataname == 'divb': @@ -2491,17 +2740,31 @@ class ParticleDiagnostic(picmistandard.PICMI_ParticleDiagnostic, WarpXDiagnostic warpx_openpmd_backend: {bp, h5, json}, optional Openpmd backend file format + warpx_openpmd_encoding: 'v' (variable based), 'f' (file based) or 'g' (group based), optional + Only read if ``.format = openpmd``. openPMD file output encoding. + File based: one file per timestep (slower), group/variable based: one file for all steps (faster)). + Variable based is an experimental feature with ADIOS2. Default: `'f'`. + warpx_file_prefix: string, optional Prefix on the diagnostic file name warpx_file_min_digits: integer, optional Minimum number of digits for the time step number in the file name - warpx_random_fraction: float, optional - Random fraction of particles to include in the diagnostic + warpx_random_fraction: float or dict, optional + Random fraction of particles to include in the diagnostic. If a float + is given the same fraction will be used for all species, if a dictionary + is given the keys should be species with the value specifying the random + fraction for that species. + + warpx_uniform_stride: integer or dict, optional + Stride to down select to the particles to include in the diagnostic. + If an integer is given the same stride will be used for all species, if + a dictionary is given the keys should be species with the value + specifying the stride for that species. - warpx_uniform_stride: integer, optional - Stride to down select to the particles to include in the diagnostic + warpx_dump_last_timestep: bool, optional + If true, the last timestep is dumped regardless of the diagnostic period/intervals. warpx_plot_filter_function: string, optional Analytic expression to down select the particles to in the diagnostic @@ -2510,11 +2773,13 @@ def init(self, kw): self.format = kw.pop('warpx_format', 'plotfile') self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) + self.openpmd_encoding = kw.pop('warpx_openpmd_encoding', None) self.file_prefix = kw.pop('warpx_file_prefix', None) self.file_min_digits = kw.pop('warpx_file_min_digits', None) self.random_fraction = kw.pop('warpx_random_fraction', None) self.uniform_stride = kw.pop('warpx_uniform_stride', None) self.plot_filter_function = kw.pop('warpx_plot_filter_function', None) + self.dump_last_timestep = kw.pop('warpx_dump_last_timestep', None) self.user_defined_kw = {} if self.plot_filter_function is not None: @@ -2534,7 +2799,9 @@ def diagnostic_initialize_inputs(self): self.diagnostic.diag_type = 'Full' self.diagnostic.format = self.format self.diagnostic.openpmd_backend = self.openpmd_backend + self.diagnostic.openpmd_encoding = self.openpmd_encoding self.diagnostic.file_min_digits = self.file_min_digits + self.diagnostic.dump_last_timestep = self.dump_last_timestep self.diagnostic.intervals = self.period self.diagnostic.set_or_replace_attr('write_species', True) if 'fields_to_plot' not in self.diagnostic.argvattrs: @@ -2585,6 +2852,9 @@ def diagnostic_initialize_inputs(self): ) else: variables.add(dataname) + else: + # possibly add user defined attributes + variables.add(dataname) # --- Convert the set to a sorted list so that the order # --- is the same on all processors. @@ -2599,6 +2869,22 @@ def diagnostic_initialize_inputs(self): else: species_names = [self.species.name] + # check if random fraction is specified and whether a value is given per species + random_fraction = {} + random_fraction_default = self.random_fraction + if isinstance(self.random_fraction, dict): + random_fraction_default = 1.0 + for key, val in self.random_fraction.items(): + random_fraction[key.name] = val + + # check if uniform stride is specified and whether a value is given per species + uniform_stride = {} + uniform_stride_default = self.uniform_stride + if isinstance(self.uniform_stride, dict): + uniform_stride_default = 1 + for key, val in self.uniform_stride.items(): + uniform_stride[key.name] = val + if self.mangle_dict is None: # Only do this once so that the same variables are used in this distribution # is used multiple times @@ -2607,8 +2893,8 @@ def diagnostic_initialize_inputs(self): for name in species_names: diag = pywarpx.Bucket.Bucket(self.name + '.' + name, variables = variables, - random_fraction = self.random_fraction, - uniform_stride = self.uniform_stride) + random_fraction = random_fraction.get(name, random_fraction_default), + uniform_stride = uniform_stride.get(name, uniform_stride_default)) expression = pywarpx.my_constants.mangle_expression(self.plot_filter_function, self.mangle_dict) diag.__setattr__('plot_filter_function(t,x,y,z,ux,uy,uz)', expression) self.diagnostic._species_dict[name] = diag @@ -2632,6 +2918,11 @@ class LabFrameFieldDiagnostic(picmistandard.PICMI_LabFrameFieldDiagnostic, warpx_openpmd_backend: string, optional Passed to .openpmd_backend + warpx_openpmd_encoding: 'f' (file based) or 'g' (group based), optional + Only read if ``.format = openpmd``. openPMD file output encoding. + File based: one file per timestep (slower), group/variable based: one file for all steps (faster)). + Default: `'f'`. + warpx_file_prefix: string, optional Passed to .file_prefix @@ -2656,6 +2947,7 @@ def init(self, kw): self.format = kw.pop('warpx_format', None) self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) + self.openpmd_encoding = kw.pop('warpx_openpmd_encoding', None) self.file_prefix = kw.pop('warpx_file_prefix', None) self.intervals = kw.pop('warpx_intervals', None) self.file_min_digits = kw.pop('warpx_file_min_digits', None) @@ -2670,6 +2962,7 @@ def diagnostic_initialize_inputs(self): self.diagnostic.diag_type = 'BackTransformed' self.diagnostic.format = self.format self.diagnostic.openpmd_backend = self.openpmd_backend + self.diagnostic.openpmd_encoding = self.openpmd_encoding self.diagnostic.file_min_digits = self.file_min_digits self.diagnostic.diag_lo = self.lower_bound self.diagnostic.diag_hi = self.upper_bound @@ -2741,6 +3034,11 @@ class LabFrameParticleDiagnostic(picmistandard.PICMI_LabFrameParticleDiagnostic, warpx_openpmd_backend: string, optional Passed to .openpmd_backend + warpx_openpmd_encoding: 'f' (file based) or 'g' (group based), optional + Only read if ``.format = openpmd``. openPMD file output encoding. + File based: one file per timestep (slower), group/variable based: one file for all steps (faster)). + Default: `'f'`. + warpx_file_prefix: string, optional Passed to .file_prefix @@ -2757,6 +3055,7 @@ class LabFrameParticleDiagnostic(picmistandard.PICMI_LabFrameParticleDiagnostic, def init(self, kw): self.format = kw.pop('warpx_format', None) self.openpmd_backend = kw.pop('warpx_openpmd_backend', None) + self.openpmd_encoding = kw.pop('warpx_openpmd_encoding', None) self.file_prefix = kw.pop('warpx_file_prefix', None) self.intervals = kw.pop('warpx_intervals', None) self.file_min_digits = kw.pop('warpx_file_min_digits', None) @@ -2769,6 +3068,7 @@ def diagnostic_initialize_inputs(self): self.diagnostic.diag_type = 'BackTransformed' self.diagnostic.format = self.format self.diagnostic.openpmd_backend = self.openpmd_backend + self.diagnostic.openpmd_encoding = self.openpmd_encoding self.diagnostic.file_min_digits = self.file_min_digits self.diagnostic.do_back_transformed_particles = True diff --git a/Python/setup.py b/Python/setup.py index fbf3330ada0..31f35eeceac 100644 --- a/Python/setup.py +++ b/Python/setup.py @@ -54,12 +54,12 @@ package_data = {} setup(name = 'pywarpx', - version = '24.05', + version = '24.07', packages = ['pywarpx'], package_dir = {'pywarpx': 'pywarpx'}, description = """Wrapper of WarpX""", package_data = package_data, - install_requires = ['numpy', 'picmistandard==0.28.0', 'periodictable'], + install_requires = ['numpy', 'picmistandard==0.29.0', 'periodictable'], python_requires = '>=3.8', zip_safe=False ) diff --git a/README.md b/README.md index 8ab69791bb1..9ef874930d9 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ [![Discussions](https://img.shields.io/badge/chat-discussions-turquoise.svg)](https://github.com/ECP-WarpX/WarpX/discussions) [![Supported Platforms](https://img.shields.io/badge/platforms-linux%20|%20osx%20|%20win-blue)](https://warpx.readthedocs.io/en/latest/install/users.html) [![GitHub commits since last release](https://img.shields.io/github/commits-since/ECP-WarpX/WarpX/latest/development.svg)](https://github.com/ECP-WarpX/WarpX/compare/development) -[![Exascale Computing Project](https://img.shields.io/badge/supported%20by-ECP-orange)](https://www.exascaleproject.org/research/) +[![HPSF](https://img.shields.io/badge/hosted%20by-HPSF-orange)](https://hpsf.io) [![Language: C++17](https://img.shields.io/badge/language-C%2B%2B17-orange.svg)](https://isocpp.org/) [![Language: Python](https://img.shields.io/badge/language-Python-orange.svg)](https://python.org/) [![License WarpX](https://img.shields.io/badge/license-BSD--3--Clause--LBNL-blue.svg)](https://spdx.org/licenses/BSD-3-Clause-LBNL.html) @@ -52,6 +52,14 @@ module load gsl [![SENSEI](https://img.shields.io/static/v1?label="runs%20on"&message="SENSEI"&color="blueviolet")](https://sensei-insitu.org) Our workflow is described in [CONTRIBUTING.rst](CONTRIBUTING.rst). +We invite you to contribute to WarpX in any form following our [Code of Conduct](https://warpx.readthedocs.io/en/latest/coc.html), e.g., contribute to [discussions](https://github.com/ECP-WarpX/WarpX/discussions), help each other in [issues](https://github.com/ECP-WarpX/WarpX/issues), fix bugs, or add [documentation](https://warpx.readthedocs.io/en/latest/developers/documentation.html) and new functionality! + +## Governance + +WarpX is hosted by the High Performance Computing Foundation (HPSF). +If your organization wants to help steer the evolution of the HPC software ecosystem, visit [hpsf.io](https://hpsf.io) and consider joining! + +The WarpX open governance model is described in [GOVERNANCE.rst](GOVERNANCE.rst). ## Copyright Notice diff --git a/Regression/Checksum/benchmarks_json/BTD_rz.json b/Regression/Checksum/benchmarks_json/BTD_rz.json index 01e4c687292..cf83ce01346 100644 --- a/Regression/Checksum/benchmarks_json/BTD_rz.json +++ b/Regression/Checksum/benchmarks_json/BTD_rz.json @@ -1,13 +1,13 @@ { "lev=0": { - "Br": 1.8705552264208163e-08, - "Bt": 2380179.5677922587, - "Bz": 2.4079077116116535e-08, + "Br": 1.8705569155952808e-08, + "Bt": 2380179.567792259, + "Bz": 2.4079063431898616e-08, "Er": 497571119514841.25, - "Et": 7.048225464720808, - "Ez": 137058870936728.28, + "Et": 7.048223219552, + "Ez": 137058870936728.23, "jr": 0.0, "jt": 0.0, "jz": 0.0 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json b/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json index 59818f62826..45e29874cda 100644 --- a/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json +++ b/Regression/Checksum/benchmarks_json/Deuterium_Deuterium_Fusion_3D_intraspecies.json @@ -1,34 +1,34 @@ { "lev=0": { "rho": 0.0, - "rho_deuterium": 8203144355.71195, - "rho_helium3": 10.368009592276463 + "rho_deuterium": 8203144355.767546, + "rho_helium3": 10.312416024595505 }, "neutron": { - "particle_momentum_x": 2.2543499835759282e-15, - "particle_momentum_y": 2.2526527390783875e-15, - "particle_momentum_z": 2.2619641737859965e-15, - "particle_position_x": 61.961041864660686, - "particle_position_y": 61.78141653674165, - "particle_position_z": 61.741022731492514, - "particle_weight": 505562702.7678892 + "particle_momentum_x": 2.260586487910896e-15, + "particle_momentum_y": 2.2564723094408887e-15, + "particle_momentum_z": 2.263164930227645e-15, + "particle_position_x": 62.03084683414219, + "particle_position_y": 61.95011296352698, + "particle_position_z": 62.02404756675538, + "particle_weight": 502851860.91505826 }, "deuterium": { - "particle_momentum_x": 1.3370046499332103e-14, - "particle_momentum_y": 1.3364310231320824e-14, - "particle_momentum_z": 1.3372728873714894e-14, - "particle_position_x": 2560.1613417364665, - "particle_position_y": 2560.082464065988, - "particle_position_z": 2560.0018477161034, - "particle_weight": 7.999999989888742e+17 + "particle_momentum_x": 1.3380470537895298e-14, + "particle_momentum_y": 1.3367685546044215e-14, + "particle_momentum_z": 1.3372922616599391e-14, + "particle_position_x": 2559.782513518522, + "particle_position_y": 2559.8747993471684, + "particle_position_z": 2560.2864832238383, + "particle_weight": 7.999999989942968e+17 }, "helium3": { - "particle_momentum_x": 2.2749239620327265e-15, - "particle_momentum_y": 2.268697031603961e-15, - "particle_momentum_z": 2.278045756364995e-15, - "particle_position_x": 61.961041864660686, - "particle_position_y": 61.78141653674165, - "particle_position_z": 61.741022731492514, - "particle_weight": 505562702.7678892 + "particle_momentum_x": 2.278275200450756e-15, + "particle_momentum_y": 2.2738610200497133e-15, + "particle_momentum_z": 2.2792408973056887e-15, + "particle_position_x": 62.03084683414219, + "particle_position_y": 61.95011296352698, + "particle_position_z": 62.02404756675538, + "particle_weight": 502851860.91505826 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereEB_RZ_MR.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphereEB_RZ_MR.json index ffa8d68f9d9..6bbfce0e3b3 100644 --- a/Regression/Checksum/benchmarks_json/ElectrostaticSphereEB_RZ_MR.json +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphereEB_RZ_MR.json @@ -1,10 +1,10 @@ { "lev=0": { - "Er": 8487.661571739109, - "phi": 2036.0428085225362 + "Er": 16975.32314347822, + "phi": 4072.085617045073 }, "lev=1": { - "Er": 19519.172334977942, - "phi": 3291.0262856782897 + "Er": 26818.189739547757, + "phi_lvl1": 8731.176548788893 } } diff --git a/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json b/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json index 024127a1bd2..21d5208c59a 100644 --- a/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json +++ b/Regression/Checksum/benchmarks_json/ElectrostaticSphereLabFrame_MR_emass_10.json @@ -6,15 +6,15 @@ "rho": 0.0 }, "lev=1": { - "Ex": 14.281015560380963, - "Ey": 14.281015560380965, - "Ez": 14.281015560380965, + "Ex": 7.170105936287823, + "Ey": 7.17010593628782, + "Ez": 7.170105936287821, "rho": 2.6092568008333786e-10 }, "electron": { - "particle_momentum_x": 1.80842228672388e-24, - "particle_momentum_y": 1.8084222867238806e-24, - "particle_momentum_z": 1.7598771525647628e-24, + "particle_momentum_x": 9.257577597262615e-25, + "particle_momentum_y": 9.257577597262618e-25, + "particle_momentum_z": 9.257577597262624e-25, "particle_position_x": 327.46875, "particle_position_y": 327.46875, "particle_position_z": 327.46875, diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_psatd.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_psatd.json index 378b7691898..07c990b3997 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_psatd.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_MR_psatd.json @@ -1,40 +1,40 @@ { - "electrons": { - "particle_momentum_x": 4.2387000553111037e-20, - "particle_momentum_y": 0.0, - "particle_momentum_z": 4.238700039223283e-20, - "particle_position_x": 0.6553599659707846, - "particle_position_y": 0.6553599659699976, - "particle_weight": 3200000000000000.5 - }, "lev=0": { "Bx": 0.0, - "By": 46.86760020196344, + "By": 46.65494014556924, "Bz": 0.0, - "Ex": 7589330186947.288, + "Ex": 7596470155558.35, "Ey": 0.0, - "Ez": 7589330198843.742, - "jx": 7283102663077210.0, + "Ez": 7596470167219.471, + "jx": 7277974695617658.0, "jy": 0.0, - "jz": 7283102630279985.0 + "jz": 7277974664068728.0 }, "lev=1": { "Bx": 0.0, - "By": 369.2209081024592, + "By": 371.2497742366917, "Bz": 0.0, - "Ex": 7593801203430.788, + "Ex": 7596619292118.433, "Ey": 0.0, - "Ez": 7593800988021.711, - "jx": 6597471633118175.0, + "Ez": 7596619076137.7295, + "jx": 6592788899529852.0, "jy": 0.0, - "jz": 6597471447306984.0 + "jz": 6592788734672843.0 + }, + "electrons": { + "particle_momentum_x": 4.2314133085678375e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 4.2314132920088513e-20, + "particle_position_x": 0.6553599899254421, + "particle_position_y": 0.6553599899246253, + "particle_weight": 3200000000000000.5 }, "positrons": { - "particle_momentum_x": 4.238670783273887e-20, + "particle_momentum_x": 4.2314444192249906e-20, "particle_momentum_y": 0.0, - "particle_momentum_z": 4.238670762294589e-20, - "particle_position_x": 0.6553596917214832, - "particle_position_y": 0.6553596917221793, + "particle_momentum_z": 4.231444401040395e-20, + "particle_position_x": 0.655359770013864, + "particle_position_y": 0.6553597700144888, "particle_weight": 3200000000000000.5 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd.json index 85a09296e38..c59a09cc616 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd.json @@ -1,27 +1,27 @@ { - "electrons": { - "particle_momentum_x": 5.663705661977914e-20, - "particle_momentum_y": 0.0, - "particle_momentum_z": 5.663705661977915e-20, - "particle_position_x": 0.65536, - "particle_position_y": 0.6553599999999999, - "particle_weight": 3200000000000000.5 - }, "lev=0": { - "Ex": 3771082811318.298, + "Ex": 3751590727260.198, "Ey": 0.0, - "Ez": 3771082811318.301, - "jx": 1.009365330460902e+16, + "Ez": 3751590727260.1885, + "jx": 1.0100623838390416e+16, "jy": 0.0, - "jz": 1.009365330460902e+16, + "jz": 1.0100623838390416e+16, "part_per_cell": 131072.0 }, "positrons": { - "particle_momentum_x": 5.663705661977916e-20, + "particle_momentum_x": 5.668407775012392e-20, "particle_momentum_y": 0.0, - "particle_momentum_z": 5.663705661977916e-20, + "particle_momentum_z": 5.668407775012392e-20, "particle_position_x": 0.65536, "particle_position_y": 0.65536, "particle_weight": 3200000000000000.5 + }, + "electrons": { + "particle_momentum_x": 5.668407775012391e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 5.668407775012391e-20, + "particle_position_x": 0.65536, + "particle_position_y": 0.6553600000000002, + "particle_weight": 3200000000000000.5 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_multiJ.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_multiJ.json index c342d9dcaeb..ddf519913bd 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_multiJ.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_2d_psatd_multiJ.json @@ -1,29 +1,29 @@ { - "electrons": { - "particle_momentum_x": 5.663705657675969e-20, - "particle_momentum_y": 0.0, - "particle_momentum_z": 5.663705657675969e-20, - "particle_position_x": 0.65536, - "particle_position_y": 0.65536, - "particle_weight": 3200000000000000.5 - }, "lev=0": { "Bx": 0.0, - "By": 3.4892704618136277, + "By": 5.710236645367475, "Bz": 0.0, - "Ex": 3771082786646.7104, + "Ex": 3751590785149.1333, "Ey": 0.0, - "Ez": 3771082786646.702, - "jx": 1.0093631772735916e+16, + "Ez": 3751590785149.124, + "jx": 1.0100602372860838e+16, "jy": 0.0, - "jz": 1.0093631772735912e+16 + "jz": 1.0100602372860836e+16 }, "positrons": { - "particle_momentum_x": 5.663705657675971e-20, + "particle_momentum_x": 5.668407759026453e-20, "particle_momentum_y": 0.0, - "particle_momentum_z": 5.663705657675969e-20, - "particle_position_x": 0.65536, + "particle_momentum_z": 5.668407759026454e-20, + "particle_position_x": 0.6553599999999999, "particle_position_y": 0.65536, "particle_weight": 3200000000000000.5 + }, + "electrons": { + "particle_momentum_x": 5.668407759026452e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 5.668407759026452e-20, + "particle_position_x": 0.65536, + "particle_position_y": 0.6553600000000002, + "particle_weight": 3200000000000000.5 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_div_cleaning.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_div_cleaning.json index c9339aeb1aa..7098b83dbac 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_div_cleaning.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_div_cleaning.json @@ -1,30 +1,30 @@ { - "electrons": { - "particle_momentum_x": 9.520485222844096e-20, - "particle_position_x": 2.6214400000233367, - "particle_position_y": 2.6214400000233367, - "particle_position_z": 2.6214400000000007, - "particle_weight": 128000000000.00002 - }, "lev=0": { - "Bx": 78.90261993791012, - "By": 78.90261993794903, - "Bz": 78.9041393998435, - "Ex": 84580428018688.97, - "Ey": 84580428018688.73, - "Ez": 84580441923246.2, - "F": 685.6927382678912, - "divE": 7.94184911020468e+19, - "jx": 6.007457695701311e+16, - "jy": 6.007457695701312e+16, - "jz": 6.007453636158103e+16, + "Bx": 86.03228226677659, + "By": 86.03228226679013, + "Bz": 86.033798606458, + "Ex": 84659309211473.42, + "Ey": 84659309211473.42, + "Ez": 84659323064827.52, + "F": 700.2020613746595, + "divE": 7.94984970149747e+19, + "jx": 5.999100369888243e+16, + "jy": 5.999100369888247e+16, + "jz": 5.999096310379822e+16, "part_per_cell": 524288.0, - "rho": 705198439.2596256 + "rho": 705877712.4251474 }, "positrons": { - "particle_momentum_z": 9.520479438980041e-20, - "particle_position_x": 2.6214400000233367, - "particle_position_y": 2.6214400000233358, + "particle_momentum_z": 9.508931687286131e-20, + "particle_position_x": 2.621440000034161, + "particle_position_y": 2.6214400000341618, "particle_position_z": 2.62144 + }, + "electrons": { + "particle_momentum_x": 9.508937471997448e-20, + "particle_position_x": 2.6214400000341604, + "particle_position_y": 2.621440000034161, + "particle_position_z": 2.6214399999999993, + "particle_weight": 128000000000.00002 } -} +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_multiJ.json b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_multiJ.json index 487d4a89828..e1785dcce4d 100644 --- a/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_multiJ.json +++ b/Regression/Checksum/benchmarks_json/Langmuir_multi_psatd_multiJ.json @@ -1,28 +1,28 @@ { - "electrons": { - "particle_momentum_x": 9.633869745818886e-20, - "particle_position_x": 2.621440000001177, - "particle_position_y": 2.6214400000011784, - "particle_position_z": 2.6214400000000007, - "particle_weight": 128000000000.00002 - }, "lev=0": { - "Bx": 80.96035538655111, - "By": 80.96035538657691, - "Bz": 80.96271445263956, - "Ex": 84777489275096.88, - "Ey": 84777489275096.88, - "Ez": 84777485856239.4, - "jx": 6.08447015360442e+16, - "jy": 6.084470153604425e+16, - "jz": 6.084470085554113e+16, + "Bx": 88.73949682623977, + "By": 88.73949682627456, + "Bz": 88.74185838578241, + "Ex": 84851140116479.64, + "Ey": 84851140116479.62, + "Ez": 84851136696893.72, + "jx": 6.075773528102969e+16, + "jy": 6.0757735281029736e+16, + "jz": 6.075773460167053e+16, "part_per_cell": 524288.0, - "rho": 703546536.8089281 + "rho": 704298346.5173542 }, "positrons": { - "particle_momentum_z": 9.63386961193585e-20, - "particle_position_x": 2.621440000001177, - "particle_position_y": 2.621440000001179, - "particle_position_z": 2.6214400000000007 + "particle_momentum_z": 9.621748455552108e-20, + "particle_position_x": 2.621440000001309, + "particle_position_y": 2.6214400000013076, + "particle_position_z": 2.6214399999999998 + }, + "electrons": { + "particle_momentum_x": 9.621748589257923e-20, + "particle_position_x": 2.621440000001309, + "particle_position_y": 2.6214400000013076, + "particle_position_z": 2.6214399999999998, + "particle_weight": 128000000000.00002 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/LoadExternalFieldRZ.json b/Regression/Checksum/benchmarks_json/LoadExternalFieldRZGrid.json similarity index 100% rename from Regression/Checksum/benchmarks_json/LoadExternalFieldRZ.json rename to Regression/Checksum/benchmarks_json/LoadExternalFieldRZGrid.json diff --git a/Regression/Checksum/benchmarks_json/LoadExternalFieldRZParticles.json b/Regression/Checksum/benchmarks_json/LoadExternalFieldRZParticles.json new file mode 100644 index 00000000000..dc4ffa3beb2 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/LoadExternalFieldRZParticles.json @@ -0,0 +1,22 @@ +{ + "lev=0": { + "Br": 0.6380326770459374, + "Bt": 0.0, + "Bz": 4.850698699951099, + "Er": 0.0, + "Et": 0.0, + "Ez": 0.0, + "jr": 0.0, + "jt": 0.0, + "jz": 0.0 + }, + "proton": { + "particle_momentum_x": 1.2945379007125463e-23, + "particle_momentum_y": 7.901629565307941e-23, + "particle_momentum_z": 2.0004592432172922e-23, + "particle_position_x": 0.12402004585603355, + "particle_position_y": 4.363249204658946, + "particle_theta": 0.22200801011337173, + "particle_weight": 1.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/PEC_particle.json b/Regression/Checksum/benchmarks_json/PEC_particle.json index 53253cebb0a..0c502a6d615 100644 --- a/Regression/Checksum/benchmarks_json/PEC_particle.json +++ b/Regression/Checksum/benchmarks_json/PEC_particle.json @@ -1,31 +1,31 @@ { "lev=0": { - "Bx": 6.52080713617892e-07, - "By": 5.323555640562078e-18, - "Bz": 1.1834601056649145e-06, - "Ex": 300.06725745121753, - "Ey": 320.82687281173185, - "Ez": 185.62208934593673, - "jx": 1.857782359527958e-06, - "jy": 257766.55567863808, + "Bx": 6.5208071361789e-07, + "By": 5.323579900863346e-18, + "Bz": 1.1834601056649094e-06, + "Ex": 300.0672574512165, + "Ey": 320.82687281173, + "Ez": 185.62208934593605, + "jx": 1.8577823595279576e-06, + "jy": 257766.5556786359, "jz": 0.0 }, + "electron": { + "particle_momentum_x": 1.1142179380730402e-32, + "particle_momentum_y": 4.5160903214571184e-36, + "particle_momentum_z": 5.735409609139057e-50, + "particle_position_x": 3.199800000000006e-05, + "particle_position_y": 6.267072536608953e-23, + "particle_position_z": 3.3938629027782103e-37, + "particle_weight": 1.0 + }, "proton": { - "particle_momentum_x": 1.0975605406855061e-33, + "particle_momentum_x": 1.097560540685506e-33, "particle_momentum_y": 1.002878875615427e-18, - "particle_momentum_z": 5.084424741580567e-51, + "particle_momentum_z": 8.484417024161229e-51, "particle_position_x": 3.1998e-05, "particle_position_y": 6.572670690061996e-06, - "particle_position_z": 5.681444715216788e-38, - "particle_weight": 1.0 - }, - "electron": { - "particle_momentum_x": 1.1142179380730476e-32, - "particle_momentum_y": 4.516090321457367e-36, - "particle_momentum_z": 2.2354046360787e-50, - "particle_position_x": 3.199800000000006e-05, - "particle_position_y": 6.267072536609136e-23, - "particle_position_z": 1.4145775327446577e-37, + "particle_position_z": 4.372139867361774e-39, "particle_weight": 1.0 } } diff --git a/Regression/Checksum/benchmarks_json/Python_Langmuir.json b/Regression/Checksum/benchmarks_json/Python_Langmuir.json index 950d7f9194a..46867790cb4 100644 --- a/Regression/Checksum/benchmarks_json/Python_Langmuir.json +++ b/Regression/Checksum/benchmarks_json/Python_Langmuir.json @@ -1,13 +1,13 @@ { + "lev=0": { + "Ex": 3047094224871324.5, + "jx": 3.294250379355652e+18 + }, "electrons": { - "particle_momentum_x": 1.7507165884671642e-17, - "particle_position_x": 10.350911572498216, - "particle_position_y": 10.485760000000003, - "particle_position_z": 10.485760000000003, + "particle_momentum_x": 1.740964347107983e-17, + "particle_position_x": 10.351045542291539, + "particle_position_y": 10.485759999999997, + "particle_position_z": 10.48576, "particle_weight": 320000000000.00006 - }, - "lev=0": { - "Ex": 3091324128949712.5, - "jx": 3.3015975627803766e+18 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_Langmuir_2d.json b/Regression/Checksum/benchmarks_json/Python_Langmuir_2d.json index 2d1ca2f23c5..d152ff41998 100644 --- a/Regression/Checksum/benchmarks_json/Python_Langmuir_2d.json +++ b/Regression/Checksum/benchmarks_json/Python_Langmuir_2d.json @@ -1,12 +1,12 @@ { + "lev=0": { + "Ex": 53556162492188.62, + "jx": 5.722548841990386e+16 + }, "electrons": { - "particle_momentum_x": 1.0449418710231497e-19, - "particle_position_x": 0.0831205701527011, - "particle_position_y": 0.08192, + "particle_momentum_x": 1.0581442023547547e-19, + "particle_position_x": 0.0831038802568349, + "particle_position_y": 0.08191999999999999, "particle_weight": 8000000000000001.0 - }, - "lev=0": { - "Ex": 54311177004701.516, - "jx": 5.639670983525456e+16 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json b/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json index 08969db023e..474e812480f 100644 --- a/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json +++ b/Regression/Checksum/benchmarks_json/Python_LaserAcceleration.json @@ -21,12 +21,14 @@ "particle_weight": 6241509.074460764 }, "electrons": { + "particle_initialenergy": 0.0, "particle_momentum_x": 1.7921232210868553e-20, "particle_momentum_y": 7.225819896136567e-20, "particle_momentum_z": 4.2317254599358777e-20, "particle_position_x": 0.713912262116188, "particle_position_y": 0.7150340887578024, "particle_position_z": 1.31757706006908, + "particle_regionofinterest": 1936.0, "particle_weight": 12926557617.187498 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json b/Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json index baaf29bec59..d2fd4841cc4 100644 --- a/Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json +++ b/Regression/Checksum/benchmarks_json/Python_LaserIonAcc2d.json @@ -23,6 +23,8 @@ "particle_weight": 2.6507336926909222e+17 }, "hydrogen": { + "particle_origX": 0.008198291015625001, + "particle_origZ": 0.0365664599609375, "particle_position_x": 0.008197892199782453, "particle_position_y": 0.0, "particle_position_z": 0.0365646600930625, @@ -31,4 +33,4 @@ "particle_momentum_z": 1.0873094324185116e-18, "particle_weight": 2.703612070965676e+17 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/Python_LoadExternalField3D.json b/Regression/Checksum/benchmarks_json/Python_LoadExternalGridField3D.json similarity index 100% rename from Regression/Checksum/benchmarks_json/Python_LoadExternalField3D.json rename to Regression/Checksum/benchmarks_json/Python_LoadExternalGridField3D.json diff --git a/Regression/Checksum/benchmarks_json/Python_LoadExternalParticleField3D.json b/Regression/Checksum/benchmarks_json/Python_LoadExternalParticleField3D.json new file mode 100644 index 00000000000..a34b2f136a6 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/Python_LoadExternalParticleField3D.json @@ -0,0 +1,22 @@ +{ + "lev=0": { + "Bx": 29.8448958859583, + "By": 29.844895885958305, + "Bz": 197.55124310523018, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + }, + "proton": { + "particle_momentum_x": 1.2118953253556959e-23, + "particle_momentum_y": 7.822512598009088e-23, + "particle_momentum_z": 2.2761898274347412e-23, + "particle_position_x": 0.12238072371983327, + "particle_position_y": 0.009653941154598592, + "particle_position_z": 4.317544769595657, + "particle_weight": 1.0 + } +} \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_gaussian_beam.json b/Regression/Checksum/benchmarks_json/Python_gaussian_beam.json index 2129e8af565..2b4297e3627 100644 --- a/Regression/Checksum/benchmarks_json/Python_gaussian_beam.json +++ b/Regression/Checksum/benchmarks_json/Python_gaussian_beam.json @@ -1,32 +1,32 @@ { - "electrons": { - "particle_momentum_x": 6.808479576836945e-20, - "particle_momentum_y": 6.876934757849776e-20, - "particle_momentum_z": 6.802215755177324e-20, - "particle_position_x": 6339.279719574188, - "particle_position_y": 6400.519522256783, - "particle_position_z": 6332.552838182088, - "particle_weight": 4999999954710.04 - }, "lev=0": { - "Bx": 2.18760546171463e-05, - "By": 1.777427672926153e-05, - "Bz": 1.798431779205324e-05, - "Ex": 262735.3418998487, - "Ey": 264285.9081716973, - "Ez": 262283.0576561827, - "jx": 930.8363498458478, - "jy": 940.577756171662, - "jz": 929.9701516370898, + "Bx": 2.1885709623428054e-05, + "By": 1.7781538740987698e-05, + "Bz": 1.798374460438725e-05, + "Ex": 262814.04898035893, + "Ey": 264360.66474815976, + "Ez": 262360.8533027486, + "jx": 931.6306925490601, + "jy": 941.3279496407436, + "jz": 930.7478072804985, "part_per_cell": 65536.0 }, + "electrons": { + "particle_momentum_x": 6.823819887231586e-20, + "particle_momentum_y": 6.891558962132863e-20, + "particle_momentum_z": 6.817050175691089e-20, + "particle_position_x": 6339.113170504729, + "particle_position_y": 6400.3586668158905, + "particle_position_z": 6332.387112999683, + "particle_weight": 4999999954710.04 + }, "protons": { - "particle_momentum_x": 3.177356695273816e-21, - "particle_momentum_y": 3.165603704977051e-21, - "particle_momentum_z": 3.157786057110636e-21, - "particle_position_x": 6550.923797287247, - "particle_position_y": 6537.195882086984, - "particle_position_z": 6523.379262220709, + "particle_momentum_x": 2.9968929267374238e-21, + "particle_momentum_y": 2.984482330834465e-21, + "particle_momentum_z": 2.9781449297861096e-21, + "particle_position_x": 6550.923883255297, + "particle_position_y": 6537.195968516041, + "particle_position_z": 6523.379347813018, "particle_weight": 4999999954710.04 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/Python_ionization.json b/Regression/Checksum/benchmarks_json/Python_ionization.json index 31f426aa362..7918a30244d 100644 --- a/Regression/Checksum/benchmarks_json/Python_ionization.json +++ b/Regression/Checksum/benchmarks_json/Python_ionization.json @@ -3,28 +3,30 @@ "Bx": 0.0, "By": 26296568.434868, "Bz": 0.0, - "Ex": 7878103122971888.0, + "Ex": 7878103122971890.0, "Ey": 0.0, - "Ez": 3027.995293554466, - "jx": 1.2111358330750162e+16, + "Ez": 3027.738911163427, + "jx": 1.2111358330750164e+16, "jy": 0.0, - "jz": 1.3483401471475687e-07 + "jz": 1.3523766702993914e-07 }, "electrons": { - "particle_momentum_x": 4.4206237143449475e-18, + "particle_momentum_x": 4.4237309006289776e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 2.6361297302081026e-18, - "particle_position_x": 0.11009154442846772, - "particle_position_y": 0.6414658436421568, - "particle_weight": 3.4450781249999996e-10 + "particle_momentum_z": 2.643146520891287e-18, + "particle_orig_z": 0.4306994012391905, + "particle_position_x": 0.11013574385450274, + "particle_position_y": 0.6415447480129455, + "particle_weight": 3.4456249999999997e-10 }, "ions": { "particle_ionizationLevel": 72897.0, - "particle_momentum_x": 1.76132401934254e-18, + "particle_momentum_x": 1.7613240193425407e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 3.644887053263054e-23, + "particle_momentum_z": 3.6448870533515125e-23, + "particle_orig_z": 0.128, "particle_position_x": 0.03200001189420337, "particle_position_y": 0.1280000046901387, "particle_weight": 9.999999999999999e-11 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/RepellingParticles.json b/Regression/Checksum/benchmarks_json/RepellingParticles.json index 57b36cdc90c..0331363053a 100644 --- a/Regression/Checksum/benchmarks_json/RepellingParticles.json +++ b/Regression/Checksum/benchmarks_json/RepellingParticles.json @@ -1,32 +1,32 @@ { + "lev=0": { + "Bx": 0.0, + "By": 10603.681421961119, + "Bz": 0.0, + "Ex": 12084698236273.04, + "Ey": 0.0, + "Ez": 16282812598779.527, + "F": 21124.39775810062, + "divE": 1.495787831639977e+18, + "jx": 496020344341010.9, + "jy": 0.0, + "jz": 10.641859867595102, + "rho": 6408706.535999998 + }, "electron1": { - "particle_momentum_x": 7.303686586478214e-23, + "particle_momentum_x": 7.303686586478215e-23, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.556862195127266e-36, - "particle_position_x": 1.293487226551532e-05, - "particle_position_y": 1.777889117487881e-19, + "particle_momentum_z": 1.552083282915395e-36, + "particle_position_x": 1.293487226551533e-05, + "particle_position_y": 1.7783750855614515e-19, "particle_weight": 5000000000000.0 }, "electron2": { - "particle_momentum_x": 7.303686586477884e-23, + "particle_momentum_x": 7.303686586477887e-23, "particle_momentum_y": 0.0, - "particle_momentum_z": 1.567350238825104e-36, + "particle_momentum_z": 1.5731022908002534e-36, "particle_position_x": 1.293487226551495e-05, - "particle_position_y": 1.797289839952283e-19, + "particle_position_y": 1.7989574553330234e-19, "particle_weight": 5000000000000.0 - }, - "lev=0": { - "Bx": 0.0, - "By": 10603.68142196111, - "Bz": 0.0, - "Ex": 12084698236273.04, - "Ey": 0.0, - "Ez": 16282812598779.53, - "F": 21124.39775810061, - "divE": 1.495787831639981e+18, - "jx": 496020344341011.4, - "jy": 0.0, - "jz": 10.93513947024536, - "rho": 6408706.535999998 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/ImplicitPicard_VandB_2d.json b/Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ImplicitPicard_VandB_2d.json rename to Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d.json diff --git a/Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d_PICMI.json b/Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d_PICMI.json new file mode 100644 index 00000000000..d97eb04883f --- /dev/null +++ b/Regression/Checksum/benchmarks_json/ThetaImplicitJFNK_VandB_2d_PICMI.json @@ -0,0 +1,31 @@ +{ + "lev=0": { + "Bx": 72730.70321925254, + "By": 89276.6097395453, + "Bz": 66911.00019634314, + "Ex": 92036838733000.64, + "Ey": 15583500940725.84, + "Ez": 89163420502164.97, + "divE": 8.998871921763322e+22, + "jx": 2.7748639888523993e+19, + "jy": 2.9501400595579277e+19, + "jz": 2.6976140199337787e+19, + "rho": 796777020986.2787 + }, + "protons": { + "particle_momentum_x": 2.0873315539608036e-17, + "particle_momentum_y": 2.0858882907322405e-17, + "particle_momentum_z": 2.0877345477243595e-17, + "particle_position_x": 0.004251275869323399, + "particle_position_y": 0.0042512738905209615, + "particle_weight": 2823958719279159.5 + }, + "electrons": { + "particle_momentum_x": 4.882673707817137e-19, + "particle_momentum_y": 4.879672470952739e-19, + "particle_momentum_z": 4.872329687213274e-19, + "particle_position_x": 0.004251641684258687, + "particle_position_y": 0.004251751978637919, + "particle_weight": 2823958719279159.5 + } +} diff --git a/Regression/Checksum/benchmarks_json/ImplicitPicard_1d.json b/Regression/Checksum/benchmarks_json/ThetaImplicitPicard_1d.json similarity index 100% rename from Regression/Checksum/benchmarks_json/ImplicitPicard_1d.json rename to Regression/Checksum/benchmarks_json/ThetaImplicitPicard_1d.json diff --git a/Regression/Checksum/benchmarks_json/collisionISO.json b/Regression/Checksum/benchmarks_json/collisionISO.json index 96a2f5cbdac..350848d4aee 100644 --- a/Regression/Checksum/benchmarks_json/collisionISO.json +++ b/Regression/Checksum/benchmarks_json/collisionISO.json @@ -11,12 +11,12 @@ "jz": 0.0 }, "electron": { - "particle_momentum_x": 3.5856774369040155e-19, - "particle_momentum_y": 3.5832349891760786e-19, - "particle_momentum_z": 3.5757013442445486e-19, - "particle_position_x": 1.0241562532152744, - "particle_position_y": 1.0238555904782611, - "particle_position_z": 1.0240053504978093, + "particle_momentum_x": 3.5790777034053853e-19, + "particle_momentum_y": 3.5815348106229496e-19, + "particle_momentum_z": 3.577963316718249e-19, + "particle_position_x": 1.024180253191667, + "particle_position_y": 1.023919590453571, + "particle_position_z": 1.0240653505082926, "particle_weight": 714240000000.0 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/collisionXYZ.json b/Regression/Checksum/benchmarks_json/collisionXYZ.json index 927848745a8..726573ef8b2 100644 --- a/Regression/Checksum/benchmarks_json/collisionXYZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXYZ.json @@ -5,6 +5,26 @@ "Bz": 0.0, "Ex": 0.0, "Ey": 0.0, - "Ez": 0.0 + "Ez": 0.0, + "T_electron": 351982.0169218243, + "T_ion": 349599.6939052666 + }, + "electron": { + "particle_momentum_x": 8.359982321196841e-19, + "particle_momentum_y": 8.192841151167721e-19, + "particle_momentum_z": 8.182985690701241e-19, + "particle_position_x": 21255110.08090505, + "particle_position_y": 21303488.6242626, + "particle_position_z": 21238676.122703437, + "particle_weight": 7.168263344048695e+28 + }, + "ion": { + "particle_momentum_x": 2.0034830240966893e-18, + "particle_momentum_y": 1.8323959076577197e-18, + "particle_momentum_z": 1.827953230828629e-18, + "particle_position_x": 21246214.748882487, + "particle_position_y": 21280709.710960124, + "particle_position_z": 21206153.002106402, + "particle_weight": 7.168263344048695e+28 } } diff --git a/Regression/Checksum/benchmarks_json/collisionXZ.json b/Regression/Checksum/benchmarks_json/collisionXZ.json index 927848745a8..f90c34bc86d 100644 --- a/Regression/Checksum/benchmarks_json/collisionXZ.json +++ b/Regression/Checksum/benchmarks_json/collisionXZ.json @@ -6,5 +6,21 @@ "Ex": 0.0, "Ey": 0.0, "Ez": 0.0 + }, + "ion": { + "particle_momentum_x": 2.4932317055825563e-19, + "particle_momentum_y": 2.274916403334278e-19, + "particle_momentum_z": 2.2528161767665816e-19, + "particle_position_x": 2648815.601036139, + "particle_position_y": 2662836.7581390506, + "particle_weight": 1.7256099431746894e+26 + }, + "electron": { + "particle_momentum_x": 1.0489203687862582e-19, + "particle_momentum_y": 1.0209657029567292e-19, + "particle_momentum_z": 1.0248962872393911e-19, + "particle_position_x": 2657004.8285825616, + "particle_position_y": 2670174.272797987, + "particle_weight": 1.7256099431746894e+26 } } diff --git a/Regression/Checksum/benchmarks_json/collisionZ.json b/Regression/Checksum/benchmarks_json/collisionZ.json new file mode 100644 index 00000000000..3be8d5ae893 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/collisionZ.json @@ -0,0 +1,20 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 0.0, + "Bz": 0.0, + "Ex": 0.0, + "Ey": 0.0, + "Ez": 0.0, + "jx": 0.0, + "jy": 0.0, + "jz": 0.0 + }, + "ions": { + "particle_momentum_x": 3.425400072687143e-16, + "particle_momentum_y": 3.421937133999805e-16, + "particle_momentum_z": 5.522701882677923e-16, + "particle_position_x": 720.0011611411148, + "particle_weight": 1.0999999999999999e+24 + } +} diff --git a/Regression/Checksum/benchmarks_json/ionization_boost.json b/Regression/Checksum/benchmarks_json/ionization_boost.json index 35b2db84a1a..dbde29d19f3 100644 --- a/Regression/Checksum/benchmarks_json/ionization_boost.json +++ b/Regression/Checksum/benchmarks_json/ionization_boost.json @@ -1,30 +1,30 @@ { "lev=0": { "Bx": 0.0, - "By": 18263123.342891, + "By": 18263123.342890993, "Bz": 0.0, - "Ex": 5472992180428804.0, + "Ex": 5472992180428803.0, "Ey": 0.0, - "Ez": 922.5589707939612, - "jx": 12440856508004.96, + "Ez": 922.2798874201462, + "jx": 12440856508004.957, "jy": 0.0, - "jz": 78616.0000011086 - }, - "electrons": { - "particle_momentum_x": 2.1386770170623736e-17, - "particle_momentum_y": 0.0, - "particle_momentum_z": 1.7287241262743654e-17, - "particle_position_x": 0.11064981928849067, - "particle_position_y": 1.7121826057017473, - "particle_weight": 3.0755014319061045e-09 + "jz": 78616.00000110897 }, "ions": { "particle_ionizationLevel": 52741.0, - "particle_momentum_x": 3.630619728387593e-18, + "particle_momentum_x": 3.630619728387586e-18, "particle_momentum_y": 0.0, "particle_momentum_z": 1.0432995297946715e-13, "particle_position_x": 0.021439968272741083, "particle_position_y": 0.4742770090248555, "particle_weight": 5.000948082142308e-10 + }, + "electrons": { + "particle_momentum_x": 2.138486151076505e-17, + "particle_momentum_y": 0.0, + "particle_momentum_z": 1.7285619821244245e-17, + "particle_position_x": 0.11066682695880013, + "particle_position_y": 1.712069774427624, + "particle_weight": 3.0755014319061037e-09 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/ionization_lab.json b/Regression/Checksum/benchmarks_json/ionization_lab.json index 82984141b59..99d93020532 100644 --- a/Regression/Checksum/benchmarks_json/ionization_lab.json +++ b/Regression/Checksum/benchmarks_json/ionization_lab.json @@ -3,29 +3,29 @@ "Bx": 0.0, "By": 26296568.434868, "Bz": 0.0, - "Ex": 7878103122971888.0, + "Ex": 7878103122971890.0, "Ey": 0.0, - "Ez": 3027.995293554467, - "jx": 1.2111358330750162e+16, + "Ez": 3027.7389111634266, + "jx": 1.2111358330750164e+16, "jy": 0.0, - "jz": 1.3575651149270143e-07 + "jz": 1.355962687103163e-07 + }, + "electrons": { + "particle_momentum_x": 4.4195594812814575e-18, + "particle_momentum_y": 0.0, + "particle_momentum_z": 2.6520195422151263e-18, + "particle_orig_z": 0.43004993872230357, + "particle_position_x": 0.11017033205040265, + "particle_position_y": 0.6412498318709239, + "particle_weight": 3.442265625e-10 }, "ions": { "particle_ionizationLevel": 72897.0, - "particle_momentum_x": 1.7613240052056494e-18, + "particle_momentum_x": 1.761324005205651e-18, "particle_momentum_y": 0.0, - "particle_momentum_z": 3.6186967375178554e-23, + "particle_momentum_z": 3.618696737610014e-23, "particle_position_x": 0.0320000118071032, "particle_position_y": 0.12800000462171646, "particle_weight": 9.999999999999999e-11 - }, - "electrons": { - "particle_momentum_x": 4.428135211584547e-18, - "particle_momentum_y": 0.0, - "particle_momentum_z": 2.6627518442038486e-18, - "particle_orig_z": 0.43043207622230534, - "particle_position_x": 0.11023346490802505, - "particle_position_y": 0.6417814148540429, - "particle_weight": 3.44453125e-10 } -} \ No newline at end of file +} diff --git a/Regression/Checksum/benchmarks_json/restart_psatd.json b/Regression/Checksum/benchmarks_json/restart_psatd.json index d22fb5b57e7..fd31e7057f4 100644 --- a/Regression/Checksum/benchmarks_json/restart_psatd.json +++ b/Regression/Checksum/benchmarks_json/restart_psatd.json @@ -1,15 +1,33 @@ { "lev=0": { - "Bx": 116102.13010406512, - "By": 245467.5412026496, - "Bz": 65305.35287114741, - "Ex": 118565481099326.73, - "Ey": 18640100573642.96, - "Ez": 51777969821120.38, - "jx": 2.1177246490642464e+16, - "jy": 351544815339900.3, - "jz": 4573466562652713.0, - "rho": 22294533.587530266 + "Bx": 116097.15119490701, + "By": 245135.43138545044, + "Bz": 65087.42714316293, + "Ex": 118461420042835.34, + "Ey": 18622845483678.824, + "Ez": 51793144992361.125, + "jx": 2.1162465965376624e+16, + "jy": 349487178458066.6, + "jz": 4563460427657682.0, + "rho": 22252333.818822004 + }, + "driverback": { + "particle_momentum_x": 4.813131349021332e+21, + "particle_momentum_y": 5.16548074090123e+21, + "particle_momentum_z": 3.005830430844926e+25, + "particle_position_x": 0.001649481123084974, + "particle_position_y": 0.0016172218745428432, + "particle_position_z": 0.4899808854005956, + "particle_weight": 6241509074.460762 + }, + "plasma_e": { + "particle_momentum_x": 2.084066965693142e-19, + "particle_momentum_y": 1.330336553422521e-20, + "particle_momentum_z": 2.6335518561017005e-17, + "particle_position_x": 0.4991724057788647, + "particle_position_y": 0.49919697481183745, + "particle_position_z": 0.4562433914656611, + "particle_weight": 33067341227104.625 }, "beam": { "particle_momentum_x": 4.178482505909375e-19, @@ -20,6 +38,15 @@ "particle_position_z": 1.901942694291968, "particle_weight": 3120754537.230381 }, + "plasma_p": { + "particle_momentum_x": 2.097709570232301e-19, + "particle_momentum_y": 3.402465268604418e-21, + "particle_momentum_z": 4.82960000875208e-14, + "particle_position_x": 0.49912500313484137, + "particle_position_y": 0.4991249983460674, + "particle_position_z": 0.4563845471150555, + "particle_weight": 33067341227104.625 + }, "driver": { "particle_momentum_x": 4.700436405078562e+21, "particle_momentum_y": 4.6785862113093076e+21, @@ -28,32 +55,5 @@ "particle_position_y": 0.0016212373149699414, "particle_position_z": 0.30417779498653386, "particle_weight": 6241509074.460762 - }, - "driverback": { - "particle_momentum_x": 4.813131349021332e+21, - "particle_momentum_y": 5.16548074090123e+21, - "particle_momentum_z": 3.005830430844926e+25, - "particle_position_x": 0.001649481123084974, - "particle_position_y": 0.0016172218745428432, - "particle_position_z": 0.4899808854005956, - "particle_weight": 6241509074.460762 - }, - "plasma_e": { - "particle_momentum_x": 2.0912364079892767e-19, - "particle_momentum_y": 1.3510804924903876e-20, - "particle_momentum_z": 2.6336070297356917e-17, - "particle_position_x": 0.49917333523960167, - "particle_position_y": 0.49919828074154127, - "particle_position_z": 0.45624167532314724, - "particle_weight": 33067341227104.625 - }, - "plasma_p": { - "particle_momentum_x": 2.1053025446547977e-19, - "particle_momentum_y": 3.3272838305044046e-21, - "particle_momentum_z": 4.829599953646669e-14, - "particle_position_x": 0.49912500297599893, - "particle_position_y": 0.49912499825258866, - "particle_position_z": 0.45638454709795806, - "particle_weight": 33067341227104.625 } } \ No newline at end of file diff --git a/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json b/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json index 50971f0d45e..cfd031d6e33 100644 --- a/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json +++ b/Regression/Checksum/benchmarks_json/restart_psatd_time_avg.json @@ -1,24 +1,15 @@ { "lev=0": { - "Bx": 115622.1878941125, - "By": 243403.66715242947, - "Bz": 63602.66529020351, - "Ex": 117118934931289.56, - "Ey": 18448377440588.27, - "Ez": 50821967818817.24, - "jx": 2.1044522674691452e+16, - "jy": 329314843111847.94, - "jz": 4524787623275627.0, - "rho": 22128883.53068064 - }, - "driverback": { - "particle_momentum_x": 4.813131349021332e+21, - "particle_momentum_y": 5.16548074090123e+21, - "particle_momentum_z": 3.005830430844926e+25, - "particle_position_x": 0.001649481123084974, - "particle_position_y": 0.0016172218745428432, - "particle_position_z": 0.4899808854005956, - "particle_weight": 6241509074.460762 + "Bx": 115610.21497814235, + "By": 243019.564489834, + "Bz": 63397.3416561916, + "Ex": 117012138466922.22, + "Ey": 18429606525820.12, + "Ez": 50835997822445.34, + "jx": 2.1028213761477244e+16, + "jy": 326844980054897.7, + "jz": 4514643615865121.0, + "rho": 22084564.546177678 }, "beam": { "particle_momentum_x": 4.178482505909375e-19, @@ -29,24 +20,33 @@ "particle_position_z": 1.901942694291968, "particle_weight": 3120754537.230381 }, - "plasma_e": { - "particle_momentum_x": 1.9905288525315664e-19, - "particle_momentum_y": 1.2685401564810352e-20, - "particle_momentum_z": 2.6334746597885943e-17, - "particle_position_x": 0.4991702885212871, - "particle_position_y": 0.4991949978513417, - "particle_position_z": 0.4562491611716817, + "plasma_p": { + "particle_momentum_x": 1.9959818202810056e-19, + "particle_momentum_y": 2.9455107653405726e-21, + "particle_momentum_z": 4.82960013792663e-14, + "particle_position_x": 0.4991250027534606, + "particle_position_y": 0.49912499804419774, + "particle_position_z": 0.4563845471451087, "particle_weight": 33067341227104.625 }, - "plasma_p": { - "particle_momentum_x": 2.002897995377179e-19, - "particle_momentum_y": 2.87423099731012e-21, - "particle_momentum_z": 4.8296000849128737e-14, - "particle_position_x": 0.49912500259846493, - "particle_position_y": 0.4991249979476612, - "particle_position_z": 0.45638454712861487, + "plasma_e": { + "particle_momentum_x": 1.9841122555955512e-19, + "particle_momentum_y": 1.2475587913571367e-20, + "particle_momentum_z": 2.6334235362474606e-17, + "particle_position_x": 0.49916947686056645, + "particle_position_y": 0.49919374997004357, + "particle_position_z": 0.4562507922838501, "particle_weight": 33067341227104.625 }, + "driverback": { + "particle_momentum_x": 4.813131349021332e+21, + "particle_momentum_y": 5.16548074090123e+21, + "particle_momentum_z": 3.005830430844926e+25, + "particle_position_x": 0.001649481123084974, + "particle_position_y": 0.0016172218745428432, + "particle_position_z": 0.4899808854005956, + "particle_weight": 6241509074.460762 + }, "driver": { "particle_momentum_x": 4.700436405078562e+21, "particle_momentum_y": 4.6785862113093076e+21, diff --git a/Regression/WarpX-GPU-tests.ini b/Regression/WarpX-GPU-tests.ini index 855dcff58fd..68d59d850a0 100644 --- a/Regression/WarpX-GPU-tests.ini +++ b/Regression/WarpX-GPU-tests.ini @@ -60,7 +60,7 @@ emailBody = Check https://ccse.lbl.gov/pub/GpuRegressionTesting/WarpX/ for more [AMReX] dir = /home/regtester/git/amrex/ -branch = 7ca419ebb90da60fefc01d8c1816846fff8638a5 +branch = 20e6f2eadf0c297517588ba38973ec7c7084fa31 [source] dir = /home/regtester/git/WarpX @@ -107,7 +107,7 @@ analysisRoutine = Examples/Tests/pml/analysis_pml_ckc.py #inputFile = Examples/Tests/pml/inputs_2d #runtime_params = algo.maxwell_solver=psatd warpx.do_dynamic_scheduling=0 #dim = 2 -#addToCompileString = USE_PSATD=TRUE USE_GPU=TRUE +#addToCompileString = USE_FFT=TRUE USE_GPU=TRUE #restartTest = 0 #useMPI = 1 #numprocs = 2 @@ -377,7 +377,7 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d_multi_rt runtime_params = algo.maxwell_solver=psatd warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE USE_GPU=TRUE +addToCompileString = USE_FFT=TRUE USE_GPU=TRUE restartTest = 0 useMPI = 1 numprocs = 2 @@ -395,7 +395,7 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d_multi_rt runtime_params = algo.maxwell_solver=psatd warpx.do_dynamic_scheduling=0 warpx.grid_type=collocated algo.current_deposition=direct warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE USE_GPU=TRUE +addToCompileString = USE_FFT=TRUE USE_GPU=TRUE restartTest = 0 useMPI = 1 numprocs = 2 @@ -431,7 +431,7 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d_multi_rt runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 dim = 2 -addToCompileString = USE_PSATD=TRUE USE_GPU=TRUE +addToCompileString = USE_FFT=TRUE USE_GPU=TRUE restartTest = 0 useMPI = 1 numprocs = 2 @@ -449,7 +449,7 @@ analysisOutputImage = langmuir_multi_2d_analysis.png # inputFile = Examples/Tests/langmuir/inputs_2d_multi_rt # runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=w ux uy uz diag1.positrons.variables=w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell # dim = 2 -# addToCompileString = USE_PSATD=TRUE USE_GPU=TRUE +# addToCompileString = USE_FFT=TRUE USE_GPU=TRUE # restartTest = 0 # useMPI = 1 # numprocs = 4 diff --git a/Regression/WarpX-tests.ini b/Regression/WarpX-tests.ini index ac321b8b694..96dbea5c380 100644 --- a/Regression/WarpX-tests.ini +++ b/Regression/WarpX-tests.ini @@ -59,7 +59,7 @@ emailBody = Check https://ccse.lbl.gov/pub/RegressionTesting/WarpX/ for more det [AMReX] dir = /home/regtester/AMReX_RegTesting/amrex/ -branch = 7ca419ebb90da60fefc01d8c1816846fff8638a5 +branch = 20e6f2eadf0c297517588ba38973ec7c7084fa31 [source] dir = /home/regtester/AMReX_RegTesting/warpx @@ -74,17 +74,13 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_avg_2d runtime_params = psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [averaged_galilean_2d_psatd_hybrid] @@ -92,17 +88,13 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_avg_2d runtime_params = amr.max_grid_size_x=128 amr.max_grid_size_y=64 warpx.grid_type=hybrid psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [averaged_galilean_3d_psatd] @@ -110,17 +102,13 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_avg_3d runtime_params = psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [averaged_galilean_3d_psatd_hybrid] @@ -128,17 +116,13 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_avg_3d runtime_params = warpx.grid_type=hybrid psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [background_mcc] @@ -153,10 +137,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons he_ions analysisRoutine = Examples/analysis_default_regression.py [background_mcc_dp_psp] @@ -171,10 +151,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons he_ions analysisRoutine = Examples/analysis_default_regression.py [bilinear_filter] @@ -189,8 +165,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/single_particle/analysis_bilinear_filter.py [BTD_rz] @@ -205,8 +179,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/btd_rz/analysis_BTD_laser_antenna.py [collider_diagnostics] @@ -221,10 +193,22 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/collider_relevant_diags/analysis_multiple_particles.py +[collisionZ] +buildDir = . +inputFile = Examples/Tests/collision/inputs_1d +runtime_params = +dim = 1 +addToCompileString = +cmakeSetupOpts = -DWarpX_DIMS=1 +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 1 +numthreads = 1 +analysisRoutine = Examples/Tests/collision/analysis_collision_1d.py + [collisionISO] buildDir = . inputFile = Examples/Tests/collision/inputs_3d_isotropization @@ -237,8 +221,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/collision/analysis_collision_3d_isotropization.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -247,16 +229,13 @@ buildDir = . inputFile = Examples/Tests/collision/inputs_rz runtime_params = dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=FALSE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=OFF +addToCompileString = USE_RZ=TRUE USE_FFT=FALSE +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=OFF restartTest = 0 useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/collision/analysis_collision_rz.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -272,9 +251,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/collision/analysis_collision_3d.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -290,9 +266,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/collision/analysis_collision_2d.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -301,17 +274,13 @@ buildDir = . inputFile = Examples/Tests/comoving/inputs_2d_hybrid runtime_params = psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions beam analysisRoutine = Examples/analysis_default_regression.py [Deuterium_Deuterium_Fusion_3D] @@ -326,8 +295,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_two_product_fusion.py [Deuterium_Deuterium_Fusion_3D_intraspecies] @@ -342,8 +309,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_deuterium_deuterium_3d_intraspecies.py [Deuterium_Tritium_Fusion_3D] @@ -358,8 +323,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_two_product_fusion.py [Deuterium_Tritium_Fusion_RZ] @@ -374,8 +337,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_two_product_fusion.py [dirichletbc] @@ -390,9 +351,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_dirichlet_bc/analysis.py [divb_cleaning_3d] @@ -407,9 +365,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/divb_cleaning/analysis.py [dive_cleaning_2d] @@ -423,12 +378,8 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 runtime_params = geometry.dims=2 analysisRoutine = Examples/Tests/dive_cleaning/analysis.py -analysisOutputImage = Comparison.png [dive_cleaning_3d] buildDir = . @@ -441,12 +392,8 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 runtime_params = analysisRoutine = Examples/Tests/dive_cleaning/analysis.py -analysisOutputImage = Comparison.png [ElectrostaticSphere] buildDir = . @@ -460,9 +407,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py [ElectrostaticSphereLabFrame_MR_emass_10] @@ -477,9 +421,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py [ElectrostaticSphereEB] @@ -494,9 +435,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis.py [ElectrostaticSphereEB_mixedBCs] @@ -511,9 +449,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py [EmbeddedBoundaryDiffraction] @@ -528,9 +463,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 outputFile = EmbeddedBoundaryDiffraction_plt analysisRoutine = Examples/Tests/embedded_boundary_diffraction/analysis_fields.py @@ -546,15 +478,12 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz.py [ElectrostaticSphereEB_RZ_MR] buildDir = . inputFile = Examples/Tests/electrostatic_sphere_eb/inputs_rz_mr -runtime_params = warpx.abort_on_warning_threshold = medium +runtime_params = warpx.abort_on_warning_threshold = medium amr.ref_ratio_vect = 2 2 2 dim = 2 addToCompileString = USE_EB=TRUE USE_RZ=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_EB=ON @@ -563,10 +492,8 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz.py +outputFile = ElectrostaticSphereEB_RZ_MR_plt +analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis_rz_mr.py [ElectrostaticSphereLabFrame] buildDir = . @@ -580,9 +507,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py [ElectrostaticSphereRZ] @@ -597,9 +521,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py [ElectrostaticSphereRelNodal] @@ -614,9 +535,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere/analysis_electrostatic_sphere.py [embedded_boundary_cube] @@ -631,9 +549,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/embedded_boundary_cube/analysis_fields.py [embedded_boundary_cube_2d] @@ -648,9 +563,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/embedded_boundary_cube/analysis_fields_2d.py [embedded_boundary_cube_macroscopic] @@ -665,9 +577,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/embedded_boundary_cube/analysis_fields.py [embedded_boundary_python_API] @@ -684,9 +593,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/embedded_boundary_python_api/analysis.py [embedded_boundary_rotated_cube] @@ -701,9 +607,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/embedded_boundary_rotated_cube/analysis_fields.py [embedded_boundary_rotated_cube_2d] @@ -718,9 +621,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/embedded_boundary_rotated_cube/analysis_fields_2d.py [embedded_circle] @@ -735,10 +635,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ar_ions analysisRoutine = Examples/Tests/embedded_circle/analysis.py [FieldProbe] @@ -753,9 +649,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/field_probe/analysis_field_probe.py [FluxInjection] @@ -770,10 +663,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron analysisRoutine = Examples/Tests/flux_injection/analysis_flux_injection_rz.py [FluxInjection3D] @@ -788,9 +677,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/flux_injection/analysis_flux_injection_3d.py [galilean_2d_psatd] @@ -798,17 +684,13 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_2d runtime_params = warpx.grid_type=collocated algo.current_deposition=direct psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_2d_psatd_current_correction] @@ -816,17 +698,13 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_2d runtime_params = psatd.periodic_single_box_fft=0 psatd.update_with_rho=0 psatd.current_correction=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz jx jy jz rho divE amr.max_grid_size=64 amr.blocking_factor=64 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_2d_psatd_current_correction_psb] @@ -834,17 +712,13 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_2d runtime_params = psatd.periodic_single_box_fft=1 psatd.update_with_rho=0 psatd.current_correction=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz jx jy jz rho divE dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_2d_psatd_hybrid] @@ -852,17 +726,13 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_2d_hybrid runtime_params = psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions beam analysisRoutine = Examples/analysis_default_regression.py [galilean_3d_psatd] @@ -870,17 +740,13 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_3d runtime_params = psatd.v_galilean=0. 0. 0.99498743710662 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_3d_psatd_current_correction] @@ -888,17 +754,13 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_3d runtime_params = psatd.v_galilean=0. 0. 0.99498743710662 warpx.numprocs=1 1 2 psatd.periodic_single_box_fft=0 psatd.update_with_rho=0 psatd.current_correction=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz jx jy jz rho divE dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_3d_psatd_current_correction_psb] @@ -906,17 +768,13 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_3d runtime_params = psatd.v_galilean=0. 0. 0.99498743710662 warpx.numprocs=1 1 1 psatd.periodic_single_box_fft=1 psatd.update_with_rho=0 psatd.current_correction=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz jx jy jz rho divE dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_rz_psatd] @@ -924,17 +782,13 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_rz runtime_params = electrons.random_theta=0 ions.random_theta=0 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_rz_psatd_current_correction_psb] @@ -942,17 +796,13 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_rz runtime_params = psatd.periodic_single_box_fft=1 psatd.current_correction=1 electrons.random_theta=0 ions.random_theta=0 dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [galilean_rz_psatd_current_correction] @@ -960,17 +810,13 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_rz runtime_params = psatd.periodic_single_box_fft=0 psatd.current_correction=1 electrons.random_theta=0 ions.random_theta=0 amr.max_grid_size=32 amr.blocking_factor=32 dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_galilean.py [hard_edged_plasma_lens] @@ -985,10 +831,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/plasma_lens/analysis.py [hard_edged_quadrupoles] @@ -1003,10 +845,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron analysisRoutine = Examples/Tests/AcceleratorLattice/analysis.py [hard_edged_quadrupoles_boosted] @@ -1021,10 +859,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron analysisRoutine = Examples/Tests/AcceleratorLattice/analysis.py [hard_edged_quadrupoles_moving] @@ -1039,10 +873,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron analysisRoutine = Examples/Tests/AcceleratorLattice/analysis.py [initial_distribution] @@ -1057,9 +887,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/initial_distribution/analysis_distribution.py aux1File = Tools/PostProcessing/read_raw_data.py @@ -1075,8 +902,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/ionization/analysis_ionization.py [ionization_lab] @@ -1091,8 +916,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/ionization/analysis_ionization.py [ion_stopping] @@ -1107,9 +930,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/ion_stopping/analysis_ion_stopping.py [Langmuir_multi] @@ -1124,12 +944,7 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_fluid_1D] buildDir = . @@ -1143,11 +958,7 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_1d.py -analysisOutputImage = langmuir_fluid_multi_1d_analysis.png [Langmuir_fluid_RZ] buildDir = . @@ -1161,11 +972,7 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_rz.py -analysisOutputImage = langmuir_fluid_rz_analysis.png [Langmuir_fluid_2D] buildDir = . @@ -1173,17 +980,13 @@ inputFile = Examples/Tests/langmuir_fluids/inputs_2d runtime_params = dim = 2 addToCompileString = -cmakeSetupOpts = -DWarpX_DIMS=2 -DCMAKE_BUILD_TYPE=Debug +cmakeSetupOpts = -DWarpX_DIMS=2 restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_2d.py -analysisOutputImage = langmuir_fluid_multi_2d_analysis.png [Langmuir_fluid_multi] buildDir = . @@ -1197,11 +1000,7 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/langmuir_fluids/analysis_3d.py -analysisOutputImage = langmuir_fluid_multi_analysis.png [Langmuir_multi_1d] buildDir = . @@ -1215,12 +1014,7 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_1d.py -analysisOutputImage = langmuir_multi_1d_analysis.png [Langmuir_multi_2d_MR] buildDir = . @@ -1234,12 +1028,7 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = Langmuir_multi_2d_MR.png [Langmuir_multi_2d_MR_anisotropic] buildDir = . @@ -1253,12 +1042,7 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = Langmuir_multi_2d_MR.png [Langmuir_multi_2d_MR_momentum_conserving] buildDir = . @@ -1272,31 +1056,21 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = Langmuir_multi_2d_MR_momentum_conserving.png [Langmuir_multi_2d_MR_psatd] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver = psatd warpx.use_filter = 1 amr.max_level = 1 amr.ref_ratio = 4 warpx.fine_tag_lo = -10.e-6 -10.e-6 warpx.fine_tag_hi = 10.e-6 10.e-6 diag1.electrons.variables = x z w ux uy uz diag1.positrons.variables = x z w ux uy uz psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = Langmuir_multi_2d_MR_psatd.png [Langmuir_multi_2d_nodal] buildDir = . @@ -1310,206 +1084,151 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_current_correction] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=esirkepov psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_current_correction_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=direct psatd.periodic_single_box_fft=1 psatd.current_correction=1 warpx.grid_type=collocated diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot =Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_momentum_conserving] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd algo.field_gathering=momentum-conserving diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_multiJ] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd warpx.cfl=0.7071067811865475 psatd.update_with_rho=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=2 psatd.solution_type=first-order psatd.J_in_time=linear warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = Langmuir_multi_2d_psatd_multiJ.png [Langmuir_multi_2d_psatd_multiJ_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd warpx.cfl=0.7071067811865475 psatd.update_with_rho=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=2 psatd.solution_type=first-order psatd.J_in_time=linear warpx.abort_on_warning_threshold=medium warpx.grid_type=collocated dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = Langmuir_multi_2d_psatd_multiJ_nodal.png [Langmuir_multi_2d_psatd_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=direct diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot=Ex Ey Ez jx jy jz part_per_cell warpx.cfl = 0.7071067811865475 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_Vay_deposition] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_Vay_deposition_particle_shape_4] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 algo.particle_shape=4 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_2d_psatd_Vay_deposition_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_2d runtime_params = algo.maxwell_solver=psatd amr.max_grid_size=128 warpx.grid_type=collocated algo.current_deposition=vay diag1.electrons.variables=x z w ux uy uz diag1.positrons.variables=x z w ux uy uz diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.7071067811865475 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_2d.py -analysisOutputImage = langmuir_multi_2d_analysis.png [Langmuir_multi_nodal] buildDir = . -inputFile = +inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = warpx.grid_type=collocated algo.current_deposition=direct dim = 3 addToCompileString = @@ -1519,221 +1238,161 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd] buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_current_correction] buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd algo.current_deposition=esirkepov psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho divE warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_current_correction_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd algo.current_deposition=direct psatd.periodic_single_box_fft=1 psatd.current_correction=1 warpx.grid_type=collocated diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho divE warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_div_cleaning] buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.cfl = 0.5773502691896258 psatd.update_with_rho = 1 algo.current_deposition = direct warpx.do_dive_cleaning = 1 warpx.do_divb_cleaning = 1 diag1.intervals = 0, 38:40:1 diag1.fields_to_plot = Ex Ey Ez Bx By Bz jx jy jz part_per_cell rho divE F warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_momentum_conserving] buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd algo.field_gathering=momentum-conserving warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_multiJ] buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.cfl=0.5773502691896258 algo.current_deposition=direct psatd.update_with_rho=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=2 psatd.solution_type=first-order psatd.J_in_time=linear warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = Langmuir_multi_psatd_multiJ.png [Langmuir_multi_psatd_multiJ_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.cfl=0.5773502691896258 algo.current_deposition=direct psatd.update_with_rho=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=2 psatd.solution_type=first-order psatd.J_in_time=linear warpx.abort_on_warning_threshold=medium warpx.grid_type=collocated dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = Langmuir_multi_psatd_multiJ_nodal.png [Langmuir_multi_psatd_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=direct warpx.cfl = 0.5773502691896258 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_single_precision] buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON -DWarpX_PRECISION=SINGLE +addToCompileString = USE_FFT=TRUE PRECISION=FLOAT USE_SINGLE_PRECISION_PARTICLES=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_PRECISION=SINGLE restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_Vay_deposition] buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd algo.current_deposition=vay diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_psatd_Vay_deposition_nodal] buildDir = . inputFile = Examples/Tests/langmuir/inputs_3d runtime_params = algo.maxwell_solver=psatd warpx.grid_type=collocated algo.current_deposition=vay diag1.fields_to_plot = Ex Ey Ez jx jy jz part_per_cell rho divE warpx.cfl = 0.5773502691896258 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Langmuir_multi_rz] buildDir = . @@ -1747,12 +1406,7 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/langmuir/analysis_rz.py -analysisOutputImage = Langmuir_multi_rz_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_rz_psatd] @@ -1760,19 +1414,14 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 electrons.random_theta=0 ions.random_theta=0 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/langmuir/analysis_rz.py -analysisOutputImage = Langmuir_multi_rz_psatd_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_rz_psatd_current_correction] @@ -1780,19 +1429,14 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz runtime_params = algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 amr.max_grid_size=128 psatd.periodic_single_box_fft=1 psatd.current_correction=1 diag1.fields_to_plot=jr jz Er Ez Bt rho divE electrons.random_theta=0 ions.random_theta=0 dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/langmuir/analysis_rz.py -analysisOutputImage = Langmuir_multi_rz_psatd_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_rz_psatd_multiJ] @@ -1800,19 +1444,14 @@ buildDir = . inputFile = Examples/Tests/langmuir/inputs_rz runtime_params = amr.max_grid_size=32 algo.maxwell_solver=psatd diag1.electrons.variables=x y z w ux uy uz diag1.ions.variables=x y z w ux uy uz diag1.dump_rz_modes=0 algo.current_deposition=direct warpx.do_dive_cleaning=0 psatd.update_with_rho=1 warpx.n_rz_azimuthal_modes=2 electrons.random_theta=0 electrons.num_particles_per_cell_each_dim=2 4 2 ions.random_theta=0 ions.num_particles_per_cell_each_dim=2 4 2 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=4 warpx.use_filter=1 dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE BLAS_LIB=-lblas LAPACK_LIB=-llapack +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/langmuir/analysis_rz.py -analysisOutputImage = Langmuir_multi_rz_psatd_multiJ_analysis.png aux1File = Regression/PostProcessingUtils/post_processing_utils.py [Langmuir_multi_single_precision] @@ -1827,12 +1466,7 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons positrons analysisRoutine = Examples/Tests/langmuir/analysis_3d.py -analysisOutputImage = langmuir_multi_analysis.png [Larmor] buildDir = . @@ -1846,8 +1480,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [LaserAcceleration] @@ -1862,10 +1494,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons outputFile = LaserAcceleration_plt analysisRoutine = Examples/analysis_default_openpmd_regression.py @@ -1881,10 +1509,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [LaserAcceleration_1d_fluid] @@ -1899,10 +1523,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Physics_applications/laser_acceleration/analysis_1d_fluids.py [LaserAcceleration_1d_fluid_boosted] @@ -1917,10 +1537,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Physics_applications/laser_acceleration/analysis_1d_fluids_boosted.py [LaserAccelerationBoost] @@ -1935,8 +1551,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [LaserAcceleration_BTD] @@ -1951,10 +1565,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -doComparison = 0 analysisRoutine = Examples/Tests/boosted_diags/analysis.py [LaserAccelerationMR] @@ -1969,10 +1579,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons beam analysisRoutine = Examples/analysis_default_regression.py [LaserAccelerationRZ] @@ -1987,10 +1593,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons beam analysisRoutine = Examples/analysis_default_regression.py [LaserAccelerationRZ_opmd] @@ -2005,10 +1607,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons beam outputFile = LaserAccelerationRZ_opmd_plt analysisRoutine = Examples/Tests/openpmd_rz/analysis_openpmd_rz.py @@ -2024,10 +1622,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons outputFile = LaserAcceleration_single_precision_comms_plt analysisRoutine = Examples/analysis_default_openpmd_regression.py @@ -2043,11 +1637,7 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/laser_injection/analysis_laser.py -analysisOutputImage = laser_analysis.png [LaserInjection_1d] buildDir = . @@ -2061,9 +1651,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/laser_injection/analysis_1d.py [LaserInjection_2d] @@ -2078,9 +1665,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/laser_injection/analysis_2d.py [LaserInjectionFromBINARYFile] @@ -2097,10 +1681,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserInjectionFromLASYFile] buildDir = . @@ -2116,10 +1698,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserInjectionFromLASYFile_1d] buildDir = . @@ -2135,10 +1715,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserInjectionFromLASYFile_1d_boost] buildDir = . @@ -2154,10 +1732,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserInjectionFromLASYFile_2d] buildDir = . @@ -2173,10 +1749,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserInjectionFromLASYFile_RZ] buildDir = . @@ -2192,10 +1766,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserInjectionFromRZLASYFile] buildDir = . @@ -2211,10 +1783,8 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 selfTest = 1 stSuccessString = Passed -doVis = 0 [LaserIonAcc2d] buildDir = . @@ -2229,8 +1799,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_openpmd_regression.py [LaserOnFine] @@ -2245,8 +1813,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [leveling_thinning] @@ -2261,15 +1827,27 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/resampling/analysis_leveling_thinning.py -[LoadExternalFieldRZ] +[LoadExternalFieldRZGrid] +buildDir = . +inputFile = Examples/Tests/LoadExternalField/inputs_rz_grid_fields +runtime_params = warpx.abort_on_warning_threshold=medium chk.file_prefix=LoadExternalFieldRZGrid_chk chk.file_min_digits=5 +dim = 2 +addToCompileString = USE_RZ=TRUE +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_OPENPMD=ON +restartTest = 1 +restartFileNum = 150 +useMPI = 1 +numprocs = 1 +useOMP = 1 +numthreads = 1 +analysisRoutine = Examples/Tests/LoadExternalField/analysis_rz.py + +[LoadExternalFieldRZParticles] buildDir = . -inputFile = Examples/Tests/LoadExternalField/inputs_rz -runtime_params = warpx.abort_on_warning_threshold=medium chk.file_prefix=LoadExternalFieldRZ_chk chk.file_min_digits=5 +inputFile = Examples/Tests/LoadExternalField/inputs_rz_particle_fields +runtime_params = warpx.abort_on_warning_threshold=medium chk.file_prefix=LoadExternalFieldRZParticles_chk chk.file_min_digits=5 dim = 2 addToCompileString = USE_RZ=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_OPENPMD=ON @@ -2279,10 +1857,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = proton analysisRoutine = Examples/Tests/LoadExternalField/analysis_rz.py [magnetostatic_eb_3d] @@ -2297,9 +1871,6 @@ useMPI = 1 numprocs = 1 useOMP = 2 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py [Maxwell_Hybrid_QED_solver] @@ -2307,15 +1878,13 @@ buildDir = . inputFile = Examples/Tests/maxwell_hybrid_qed/inputs_2d runtime_params = warpx.cfl=0.7071067811865475 dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/maxwell_hybrid_qed/analysis_Maxwell_QED_Hybrid.py [momentum-conserving-gather] @@ -2330,10 +1899,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = beam driver plasma_e analysisRoutine = Examples/analysis_default_regression.py [multi_J_rz_psatd] @@ -2341,17 +1906,13 @@ buildDir = . inputFile = Examples/Tests/multi_j/inputs_rz runtime_params = warpx.abort_on_warning_threshold=medium psatd.J_in_time=linear dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = driver plasma_e plasma_p analysisRoutine = Examples/analysis_default_regression.py [nci_corrector] @@ -2366,9 +1927,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -doComparison = 0 analysisRoutine = Examples/Tests/nci_fdtd_stability/analysis_ncicorr.py [nci_correctorMR] @@ -2383,9 +1941,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -doComparison = 0 analysisRoutine = Examples/Tests/nci_fdtd_stability/analysis_ncicorr.py [parabolic_channel_initialization_2d_single_precision] @@ -2399,9 +1954,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 runtime_params = analysisRoutine = Examples/Tests/initial_plasma_profile/analysis.py @@ -2417,10 +1969,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/particle_boundary_process/analysis_absorption.py [particle_boundaries_3d] @@ -2435,9 +1983,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/boundaries/analysis.py [particle_fields_diags] @@ -2453,9 +1998,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particle_fields_diags/analysis_particle_diags.py [particle_fields_diags_single_precision] @@ -2471,9 +2013,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particle_fields_diags/analysis_particle_diags_single.py [particle_pusher] @@ -2488,9 +2027,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particle_pusher/analysis_pusher.py [particle_scrape] @@ -2505,10 +2041,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/particle_boundary_scrape/analysis_scrape.py [particles_in_pml] @@ -2523,9 +2055,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py [particles_in_pml_2d] @@ -2540,9 +2069,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py [particles_in_pml_2d_MR] @@ -2557,9 +2083,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py [particles_in_pml_3d_MR] @@ -2574,9 +2097,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particles_in_pml/analysis_particles_in_pml.py [PEC_field] @@ -2591,9 +2111,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/pec/analysis_pec.py [PEC_field_mr] @@ -2608,9 +2125,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/pec/analysis_pec_mr.py [PEC_particle] @@ -2625,10 +2139,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron proton analysisRoutine = Examples/analysis_default_regression.py [Performance_works_1_uniform_rest_32ppc] @@ -2643,10 +2153,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = [Performance_works_2_uniform_rest_1ppc] @@ -2661,10 +2167,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = [Performance_works_3_uniform_drift_4ppc] @@ -2679,10 +2181,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = [Performance_works_4_labdiags_2ppc] @@ -2697,10 +2195,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = [Performance_works_5_loadimbalance] @@ -2715,10 +2209,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = [Performance_works_6_output_2ppc] @@ -2733,10 +2223,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = [photon_pusher] @@ -2751,9 +2237,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/photon_pusher/analysis_photon_pusher.py [PlasmaAccelerationBoost2d] @@ -2768,8 +2251,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [PlasmaAccelerationBoost3d] @@ -2784,8 +2265,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [PlasmaAccelerationBoost3d_hybrid] @@ -2799,8 +2278,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [PlasmaAccelerationMR] @@ -2815,10 +2292,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = beam driver plasma_e analysisRoutine = Examples/analysis_default_regression.py [Plasma_lens] @@ -2833,10 +2306,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/plasma_lens/analysis.py [Plasma_lens_boosted] @@ -2851,10 +2320,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/plasma_lens/analysis.py [Plasma_lens_short] @@ -2869,10 +2334,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/plasma_lens/analysis.py [PlasmaMirror] @@ -2887,8 +2348,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [pml_psatd_dive_divb_cleaning] @@ -2896,15 +2355,13 @@ buildDir = . inputFile = Examples/Tests/pml/inputs_3d runtime_params = warpx.do_similar_dm_pml=0 warpx.abort_on_warning_threshold=medium ablastr.fillboundary_always_sync=1 dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [pml_psatd_rz] @@ -2912,15 +2369,13 @@ buildDir = . inputFile = Examples/Tests/pml/inputs_rz runtime_params = warpx.cfl=0.7 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_RZ=TRUE USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_PSATD=ON +addToCompileString = USE_RZ=TRUE USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_psatd_rz.py [pml_x_ckc] @@ -2935,8 +2390,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_ckc.py [pml_x_galilean] @@ -2944,15 +2397,13 @@ buildDir = . inputFile = Examples/Tests/pml/inputs_2d runtime_params = algo.maxwell_solver=psatd psatd.update_with_rho=1 diag1.fields_to_plot=Ex Ey Ez Bx By Bz rho divE warpx.cfl=0.7071067811865475 warpx.do_pml_dive_cleaning=1 warpx.do_pml_divb_cleaning=1 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium psatd.v_galilean=0. 0. 0.99 warpx.grid_type=collocated dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_psatd.py [pml_x_psatd] @@ -2960,16 +2411,14 @@ buildDir = . inputFile = Examples/Tests/pml/inputs_2d runtime_params = algo.maxwell_solver=psatd psatd.update_with_rho=1 diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho divE warpx.cfl = 0.7071067811865475 warpx.do_pml_dive_cleaning=0 warpx.do_pml_divb_cleaning=0 chk.file_prefix=pml_x_psatd_chk chk.file_min_digits=5 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 1 restartFileNum = 150 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_psatd.py [pml_x_yee] @@ -2985,8 +2434,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_yee.py [pml_x_yee_eb] @@ -3002,8 +2449,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/pml/analysis_pml_yee.py [Proton_Boron_Fusion_2D] @@ -3018,8 +2463,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py [Proton_Boron_Fusion_3D] @@ -3034,8 +2477,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/nuclear_fusion/analysis_proton_boron_fusion.py [Python_background_mcc] @@ -3052,9 +2493,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_2d.py [Python_background_mcc_1d] @@ -3071,9 +2509,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py [Python_background_mcc_1d_tridiag] @@ -3090,9 +2525,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_1d.py [Python_collisionXZ] @@ -3109,9 +2541,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/collision/analysis_collision_2d.py aux1File = Regression/PostProcessingUtils/post_processing_utils.py @@ -3129,9 +2558,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_dirichlet_bc/analysis.py [Python_dsmc_1d] @@ -3148,9 +2574,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Physics_applications/capacitive_discharge/analysis_dsmc.py [Python_ElectrostaticSphereEB] @@ -3167,9 +2590,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/electrostatic_sphere_eb/analysis.py [Python_gaussian_beam] @@ -3186,10 +2606,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [Python_gaussian_beam_no_field_output] @@ -3206,10 +2622,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = [Python_gaussian_beam_opmd] @@ -3226,10 +2638,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = [Python_gaussian_beam_opmd_no_field_output] @@ -3246,10 +2654,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = [Python_id_cpu_read] @@ -3266,10 +2670,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons [Python_ionization] buildDir = . @@ -3285,8 +2685,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/ionization/analysis_ionization.py [Python_Langmuir] @@ -3303,10 +2701,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [Python_Langmuir_2d] @@ -3323,10 +2717,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [Python_Langmuir_rz_multimode] @@ -3343,10 +2733,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons protons analysisRoutine = Examples/analysis_default_regression.py [Python_LaserAcceleration] @@ -3363,10 +2749,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [Python_LaserAcceleration_1d] @@ -3383,10 +2765,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [Python_LaserAccelerationMR] @@ -3403,10 +2781,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons beam analysisRoutine = Examples/analysis_default_regression.py [Python_LaserAccelerationRZ] @@ -3423,10 +2797,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons beam analysisRoutine = Examples/analysis_default_regression.py [Python_LaserIonAcc2d] @@ -3444,15 +2814,29 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_openpmd_regression.py -[Python_LoadExternalField3D] +[Python_LoadExternalGridField3D] buildDir = . -inputFile = Examples/Tests/LoadExternalField/PICMI_inputs_3d.py +inputFile = Examples/Tests/LoadExternalField/PICMI_inputs_3d_grid_fields.py runtime_params = -customRunCmd = python3 PICMI_inputs_3d.py +customRunCmd = python3 PICMI_inputs_3d_grid_fields.py +dim = 3 +addToCompileString = USE_PYTHON_MAIN=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON +target = pip_install +restartTest = 0 +useMPI = 1 +numprocs = 1 +useOMP = 1 +numthreads = 1 +analysisRoutine = Examples/Tests/LoadExternalField/analysis_3d.py + +[Python_LoadExternalParticleField3D] +buildDir = . +inputFile = Examples/Tests/LoadExternalField/PICMI_inputs_3d_particle_fields.py +runtime_params = +customRunCmd = python3 PICMI_inputs_3d_particle_fields.py dim = 3 addToCompileString = USE_PYTHON_MAIN=TRUE cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_APP=OFF -DWarpX_PYTHON=ON -DWarpX_OPENPMD=ON @@ -3462,10 +2846,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = proton analysisRoutine = Examples/Tests/LoadExternalField/analysis_3d.py [Python_magnetostatic_eb_3d] @@ -3482,10 +2862,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 2 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons analysisRoutine = Examples/analysis_default_regression.py [Python_magnetostatic_eb_rz] @@ -3502,10 +2878,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 2 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons analysisRoutine = Examples/Tests/magnetostatic_eb/analysis_rz.py [Python_ohms_law_solver_EM_modes_1d] @@ -3522,9 +2894,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/ohm_solver_EM_modes/analysis.py [Python_ohms_law_solver_EM_modes_rz] @@ -3532,7 +2901,7 @@ buildDir = . inputFile = Examples/Tests/ohm_solver_EM_modes/PICMI_inputs_rz.py runtime_params = warpx.abort_on_warning_threshold = medium customRunCmd = python3 PICMI_inputs_rz.py --test -dim = 1 +dim = 2 addToCompileString = USE_PYTHON_MAIN=TRUE USE_OPENPMD=TRUE QED=FALSE USE_RZ=TRUE cmakeSetupOpts = -DWarpX_DIMS=RZ -DWarpX_APP=OFF -DWarpX_QED=OFF -DWarpX_PYTHON=ON target = pip_install @@ -3541,9 +2910,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/ohm_solver_EM_modes/analysis_rz.py [Python_ohms_law_solver_ion_beam_1d] @@ -3560,9 +2926,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/ohm_solver_ion_beam_instability/analysis.py [Python_ohms_law_solver_landau_damping_2d] @@ -3579,9 +2942,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/ohm_solver_ion_Landau_damping/analysis.py [Python_ohms_law_solver_magnetic_reconnection_2d] @@ -3598,9 +2958,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/ohm_solver_magnetic_reconnection/analysis.py [Python_particle_attr_access] @@ -3617,8 +2974,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/particle_data_python/analysis.py [Python_particle_attr_access_unique] @@ -3635,8 +2990,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/particle_data_python/analysis.py [Python_particle_reflection] @@ -3653,8 +3006,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/particle_boundary_process/analysis_reflection.py [Python_particle_scrape] @@ -3671,10 +3022,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/particle_boundary_scrape/analysis_scrape.py # TODO: Enable in pyAMReX, then enable lines in PICMI_inputs_2d.py again @@ -3693,10 +3040,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons # TODO: comment in again once enabled #analysisRoutine = Examples/Tests/pass_mpi_communicator/analysis.py @@ -3714,10 +3057,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = beam analysisRoutine = Examples/analysis_default_regression.py [Python_PlasmaAcceleration1d] @@ -3734,10 +3073,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = beam analysisRoutine = Examples/analysis_default_regression.py [Python_PlasmaAccelerationMR] @@ -3754,10 +3089,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = beam analysisRoutine = Examples/analysis_default_regression.py [Python_plasma_lens] @@ -3774,10 +3105,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/plasma_lens/analysis.py [Python_prev_positions] @@ -3794,9 +3121,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/analysis_default_regression.py [Python_reduced_diags_loadbalancecosts_timers] @@ -3813,9 +3137,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalancecosts.py [Python_restart_eb] @@ -3833,10 +3154,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons analysisRoutine = Examples/Tests/restart/analysis_restart.py [Python_restart_runtime_components] @@ -3854,10 +3171,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons [Python_wrappers] buildDir = . @@ -3865,17 +3178,14 @@ inputFile = Examples/Tests/python_wrappers/PICMI_inputs_2d.py runtime_params = customRunCmd = python3 PICMI_inputs_2d.py dim = 2 -addToCompileString = USE_PSATD=TRUE USE_PYTHON_MAIN=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON +addToCompileString = USE_FFT=TRUE USE_PYTHON_MAIN=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON -DWarpX_APP=OFF -DWarpX_PYTHON=ON target = pip_install restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py [qed_breit_wheeler_2d] @@ -3891,9 +3201,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/qed/breit_wheeler/analysis_yt.py [qed_breit_wheeler_2d_opmd] @@ -3909,9 +3216,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 outputFile = qed_breit_wheeler_2d_opmd_plt analysisRoutine = Examples/Tests/qed/breit_wheeler/analysis_opmd.py @@ -3928,9 +3232,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/qed/breit_wheeler/analysis_yt.py [qed_breit_wheeler_3d_opmd] @@ -3946,9 +3247,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 outputFile = qed_breit_wheeler_3d_opmd_plt analysisRoutine = Examples/Tests/qed/breit_wheeler/analysis_opmd.py @@ -3964,9 +3262,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/qed/quantum_synchrotron/analysis.py [qed_quantum_sync_3d] @@ -3981,9 +3276,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/qed/quantum_synchrotron/analysis.py [qed_schwinger1] @@ -3998,8 +3290,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/qed/schwinger/analysis_schwinger.py [qed_schwinger2] @@ -4014,8 +3304,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/qed/schwinger/analysis_schwinger.py [qed_schwinger3] @@ -4030,8 +3318,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/qed/schwinger/analysis_schwinger.py [qed_schwinger4] @@ -4046,8 +3332,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/qed/schwinger/analysis_schwinger.py [radiation_reaction] @@ -4062,9 +3346,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/radiation_reaction/test_const_B_analytical/analysis_classicalRR.py [reduced_diags] @@ -4080,9 +3361,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags.py [reduced_diags_loadbalancecosts_heuristic] @@ -4097,9 +3375,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalancecosts.py [reduced_diags_loadbalancecosts_timers] @@ -4114,9 +3389,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalancecosts.py [reduced_diags_loadbalancecosts_timers_psatd] @@ -4124,16 +3396,13 @@ buildDir = . inputFile = Examples/Tests/reduced_diags/inputs_loadbalancecosts runtime_params = algo.load_balance_costs_update=Timers dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_loadbalancecosts.py [reduced_diags_single_precision] @@ -4149,9 +3418,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/reduced_diags/analysis_reduced_diags_single.py [RefinedInjection] @@ -4166,10 +3432,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons beam analysisRoutine = Examples/Physics_applications/laser_acceleration/analysis_refined_injection.py [relativistic_space_charge_initialization] @@ -4183,12 +3445,8 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 runtime_params = analysisRoutine = Examples/Tests/relativistic_space_charge_initialization/analysis.py -analysisOutputImage = Comparison.png [RepellingParticles] buildDir = . @@ -4202,8 +3460,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/repelling_particles/analysis_repelling.py [resample_velocity_coincidence_thinning] @@ -4218,9 +3474,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py [resample_velocity_coincidence_thinning_cartesian] @@ -4235,9 +3488,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py [restart] @@ -4253,10 +3503,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = beam analysisRoutine = Examples/Tests/restart/analysis_restart.py [restart_psatd] @@ -4264,18 +3510,14 @@ buildDir = . inputFile = Examples/Tests/restart/inputs runtime_params = algo.maxwell_solver=psatd psatd.use_default_v_galilean=1 particles.use_fdtd_nci_corr=0 chk.file_prefix=restart_psatd_chk chk.file_min_digits=5 boundary.field_lo=periodic periodic damped boundary.field_hi=periodic periodic damped psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 1 restartFileNum = 5 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = beam analysisRoutine = Examples/Tests/restart/analysis_restart.py [restart_psatd_time_avg] @@ -4283,18 +3525,14 @@ buildDir = . inputFile = Examples/Tests/restart/inputs runtime_params = algo.maxwell_solver=psatd psatd.use_default_v_galilean=1 particles.use_fdtd_nci_corr=0 chk.file_prefix=restart_psatd_time_avg_chk chk.file_min_digits=5 boundary.field_lo=periodic periodic damped boundary.field_hi=periodic periodic damped psatd.do_time_averaging=1 psatd.current_correction=0 warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 1 restartFileNum = 5 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = beam analysisRoutine = Examples/Tests/restart/analysis_restart.py [RigidInjection_BTD] @@ -4309,10 +3547,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -doComparison = 0 analysisRoutine = Examples/Tests/rigid_injection/analysis_rigid_injection_BoostedFrame.py [RigidInjection_lab] @@ -4327,9 +3561,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/rigid_injection/analysis_rigid_injection_LabFrame.py [scraping] @@ -4344,9 +3575,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/scraping/analysis_rz.py [scraping_filter] @@ -4361,9 +3589,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/scraping/analysis_rz_filter.py [silver_mueller_1d] @@ -4378,8 +3603,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/silver_mueller/analysis_silver_mueller.py [silver_mueller_2d_x] @@ -4394,8 +3617,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/silver_mueller/analysis_silver_mueller.py [silver_mueller_2d_z] @@ -4410,8 +3631,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/silver_mueller/analysis_silver_mueller.py [silver_mueller_rz_z] @@ -4426,8 +3645,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/Tests/silver_mueller/analysis_silver_mueller.py [space_charge_initialization] @@ -4441,12 +3658,8 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 runtime_params = analysisRoutine = Examples/Tests/space_charge_initialization/analysis.py -analysisOutputImage = Comparison.png [space_charge_initialization_2d] buildDir = . @@ -4459,12 +3672,8 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 runtime_params = geometry.dims=2 analysisRoutine = Examples/Tests/space_charge_initialization/analysis.py -analysisOutputImage = Comparison.png [subcyclingMR] buildDir = . @@ -4478,9 +3687,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/analysis_default_regression.py [Uniform_2d] @@ -4495,8 +3701,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 analysisRoutine = Examples/analysis_default_regression.py [uniform_plasma_restart] @@ -4512,10 +3716,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons analysisRoutine = Examples/Tests/restart/analysis_restart.py [uniform_plasma_multiJ] @@ -4523,17 +3723,13 @@ buildDir = . inputFile = Examples/Tests/nci_psatd_stability/inputs_3d runtime_params = psatd.solution_type=first-order psatd.J_in_time=constant psatd.rho_in_time=constant warpx.do_dive_cleaning=1 warpx.do_divb_cleaning=1 warpx.do_multi_J=1 warpx.do_multi_J_n_depositions=1 diag1.fields_to_plot=Bx By Bz divE Ex Ey Ez F G jx jy jz rho warpx.abort_on_warning_threshold=medium dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons ions analysisRoutine = Examples/Tests/nci_psatd_stability/analysis_multiJ.py [VayDeposition2D] @@ -4541,17 +3737,13 @@ buildDir = . inputFile = Examples/Tests/vay_deposition/inputs_2d runtime_params = dim = 2 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron ion analysisRoutine = Examples/Tests/vay_deposition/analysis.py [VayDeposition3D] @@ -4559,17 +3751,13 @@ buildDir = . inputFile = Examples/Tests/vay_deposition/inputs_3d runtime_params = dim = 3 -addToCompileString = USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON +addToCompileString = USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electron ion analysisRoutine = Examples/Tests/vay_deposition/analysis.py [NodalElectrostaticSolver] @@ -4584,9 +3772,6 @@ useMPI = 1 numprocs = 1 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/nodal_electrostatic/analysis_3d.py [BeamBeamCollision] @@ -4601,8 +3786,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 outputFile = BeamBeamCollision_plt analysisRoutine = Examples/analysis_default_openpmd_regression.py @@ -4620,13 +3803,8 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons protons outputFile = spacecraft_charging_plt analysisRoutine = Examples/Physics_applications/spacecraft_charging/analysis.py -analysisOutputImage = min_phi_analysis.png [Point_of_contact_EB_3d] buildDir = . @@ -4640,10 +3818,6 @@ useMPI = 1 numprocs = 2 useOMP = 0 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons outputFile = Point_of_contact_EB_3d_plt analysisRoutine = Examples/Tests/point_of_contact_EB/analysis.py @@ -4659,14 +3833,10 @@ useMPI = 1 numprocs = 2 useOMP = 0 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -particleTypes = electrons outputFile = Point_of_contact_EB_rz_plt analysisRoutine = Examples/Tests/point_of_contact_EB/analysis.py -[ImplicitPicard_1d] +[ThetaImplicitPicard_1d] buildDir = . inputFile = Examples/Tests/Implicit/inputs_1d runtime_params = warpx.abort_on_warning_threshold=high @@ -4678,14 +3848,11 @@ useMPI = 1 numprocs = 2 useOMP = 0 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/Implicit/analysis_1d.py -[ImplicitPicard_VandB_2d] +[ThetaImplicitJFNK_VandB_2d] buildDir = . -inputFile = Examples/Tests/Implicit/inputs_vandb_2d +inputFile = Examples/Tests/Implicit/inputs_vandb_jfnk_2d runtime_params = warpx.abort_on_warning_threshold=high dim = 2 addToCompileString = @@ -4695,10 +3862,23 @@ useMPI = 1 numprocs = 2 useOMP = 0 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -analysisRoutine = Examples/Tests/Implicit/analysis_vandb_2d.py +analysisRoutine = Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py + +[ThetaImplicitJFNK_VandB_2d_PICMI] +buildDir = . +inputFile = Examples/Tests/Implicit/PICMI_inputs_vandb_jfnk_2d.py +runtime_params = +customRunCmd = python3 PICMI_inputs_vandb_jfnk_2d.py +dim = 2 +addToCompileString = USE_PYTHON_MAIN=TRUE +cmakeSetupOpts = -DWarpX_DIMS=2 -DWarpX_PYTHON=ON +target = pip_install +restartTest = 0 +useMPI = 1 +numprocs = 2 +useOMP = 0 +numthreads = 1 +analysisRoutine = Examples/Tests/Implicit/analysis_vandb_jfnk_2d.py [SemiImplicitPicard_1d] buildDir = . @@ -4712,9 +3892,6 @@ useMPI = 1 numprocs = 2 useOMP = 0 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/Implicit/analysis_1d.py [EnergyConservingThermalPlasma] @@ -4728,9 +3905,6 @@ useMPI = 1 numprocs = 2 useOMP = 0 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 analysisRoutine = Examples/Tests/energy_conserving_thermal_plasma/analysis.py [focusing_gaussian_beam] @@ -4745,10 +3919,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -doComparison = 0 analysisRoutine = Examples/Tests/gaussian_beam/analysis_focusing_beam.py [particle_boundary_interaction] @@ -4765,10 +3935,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 1 -particleTypes = electrons outputFile = particle_boundary_interaction_plt analysisRoutine = Examples/Tests/particle_boundary_interaction/analysis.py @@ -4784,9 +3950,6 @@ useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 analysisRoutine = Examples/Tests/particle_thermal_boundary/analysis_2d.py [openbc_poisson_solver] @@ -4794,15 +3957,11 @@ buildDir = . inputFile = Examples/Tests/openbc_poisson_solver/inputs_3d runtime_params = warpx.abort_on_warning_threshold = high dim = 3 -addToCompileString = USE_OPENPMD=TRUE USE_PSATD=TRUE -cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_PSATD=ON -DWarpX_OPENPMD=ON +addToCompileString = USE_OPENPMD=TRUE USE_FFT=TRUE +cmakeSetupOpts = -DWarpX_DIMS=3 -DWarpX_FFT=ON -DWarpX_OPENPMD=ON restartTest = 0 useMPI = 1 numprocs = 2 useOMP = 1 numthreads = 1 -compileTest = 0 -doVis = 0 -compareParticles = 0 -doComparison = 0 analysisRoutine = Examples/Tests/openbc_poisson_solver/analysis.py diff --git a/Regression/prepare_file_ci.py b/Regression/prepare_file_ci.py index ce7a3398b95..f7b610dace9 100644 --- a/Regression/prepare_file_ci.py +++ b/Regression/prepare_file_ci.py @@ -59,8 +59,8 @@ # always build with PSATD support (runtime controlled if used) if ci_psatd: text = re.sub('addToCompileString =', - 'addToCompileString = USE_PSATD=TRUE ', text) - text = re.sub('USE_PSATD=FALSE', + 'addToCompileString = USE_FFT=TRUE ', text) + text = re.sub('USE_FFT=FALSE', '', text) # CCache diff --git a/Source/AcceleratorLattice/LatticeElementFinder.cpp b/Source/AcceleratorLattice/LatticeElementFinder.cpp index ba2e51715fb..ec784049760 100644 --- a/Source/AcceleratorLattice/LatticeElementFinder.cpp +++ b/Source/AcceleratorLattice/LatticeElementFinder.cpp @@ -60,7 +60,7 @@ LatticeElementFinder::UpdateIndices (int const lev, amrex::MFIter const& a_mfi, // Note that the current box is used since the box may have been updated since // the initialization in InitElementFinder. const amrex::Box box = a_mfi.tilebox(); - m_zmin = WarpX::LowerCorner(box, lev, 0._rt)[2]; + m_zmin = WarpX::LowerCorner(box, lev, 0._rt).z; m_time = warpx.gett_new(lev); if (accelerator_lattice.h_quad.nelements > 0) { diff --git a/Source/BoundaryConditions/PML.H b/Source/BoundaryConditions/PML.H index 42a96f3628a..8f3ca1da1d7 100644 --- a/Source/BoundaryConditions/PML.H +++ b/Source/BoundaryConditions/PML.H @@ -13,7 +13,7 @@ #include "Utils/WarpXAlgorithmSelection.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include "FieldSolver/SpectralSolver/SpectralSolver.H" #endif @@ -140,7 +140,8 @@ public: const amrex::DistributionMapping& dm, bool do_similar_dm_pml, const amrex::Geometry* geom, const amrex::Geometry* cgeom, int ncell, int delta, amrex::IntVect ref_ratio, - amrex::Real dt, int nox_fft, int noy_fft, int noz_fft, short grid_type, + amrex::Real dt, int nox_fft, int noy_fft, int noz_fft, + ablastr::utils::enums::GridType grid_type, int do_moving_window, int pml_has_particles, int do_pml_in_domain, int psatd_solution_type, int J_in_time, int rho_in_time, bool do_pml_dive_cleaning, bool do_pml_divb_cleaning, @@ -179,7 +180,7 @@ public: return *sigba_cp; } -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT void PushPSATD (int lev); #endif @@ -245,7 +246,7 @@ private: std::unique_ptr sigba_fp; std::unique_ptr sigba_cp; -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT std::unique_ptr spectral_solver_fp; std::unique_ptr spectral_solver_cp; #endif @@ -289,7 +290,7 @@ private: static void CopyToPML (amrex::MultiFab& pml, amrex::MultiFab& reg, const amrex::Geometry& geom); }; -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT void PushPMLPSATDSinglePatch( int lev, SpectralSolver& solver, std::array,3>& pml_E, diff --git a/Source/BoundaryConditions/PML.cpp b/Source/BoundaryConditions/PML.cpp index 25b34818dd1..340005e9211 100644 --- a/Source/BoundaryConditions/PML.cpp +++ b/Source/BoundaryConditions/PML.cpp @@ -11,7 +11,7 @@ #include "BoundaryConditions/PML.H" #include "BoundaryConditions/PMLComponent.H" #include "FieldSolver/Fields.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include "FieldSolver/SpectralSolver/SpectralFieldData.H" #endif #include "Utils/TextMsg.H" @@ -22,6 +22,7 @@ #include "WarpX.H" #include +#include #include #include @@ -548,7 +549,8 @@ PML::PML (const int lev, const BoxArray& grid_ba, const DistributionMapping& grid_dm, const bool do_similar_dm_pml, const Geometry* geom, const Geometry* cgeom, int ncell, int delta, amrex::IntVect ref_ratio, - Real dt, int nox_fft, int noy_fft, int noz_fft, short grid_type, + Real dt, int nox_fft, int noy_fft, int noz_fft, + ablastr::utils::enums::GridType grid_type, int do_moving_window, int /*pml_has_particles*/, int do_pml_in_domain, const int psatd_solution_type, const int J_in_time, const int rho_in_time, const bool do_pml_dive_cleaning, const bool do_pml_divb_cleaning, @@ -631,6 +633,8 @@ PML::PML (const int lev, const BoxArray& grid_ba, } if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { + using namespace ablastr::utils::enums; + // Increase the number of guard cells, in order to fit the extent // of the stencil for the spectral solver int ngFFt_x = (grid_type == GridType::Collocated) ? nox_fft : nox_fft/2; @@ -743,7 +747,7 @@ PML::PML (const int lev, const BoxArray& grid_ba, IntVect(ncell), IntVect(delta), single_domain_box, v_sigma_sb); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT amrex::ignore_unused(lev, dt, psatd_solution_type, J_in_time, rho_in_time); # if(AMREX_SPACEDIM!=3) amrex::ignore_unused(noy_fft); @@ -858,7 +862,7 @@ PML::PML (const int lev, const BoxArray& grid_ba, cncells, cdelta, single_domain_box, v_sigma_sb); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT amrex::ignore_unused(dt); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(false, "PML: PSATD solver selected but not built."); @@ -1367,7 +1371,7 @@ PML::Restart (const std::string& dir) } } -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT void PML::PushPSATD (const int lev) { diff --git a/Source/BoundaryConditions/PML_RZ.H b/Source/BoundaryConditions/PML_RZ.H index 74d3da2e19b..c908681d8e5 100644 --- a/Source/BoundaryConditions/PML_RZ.H +++ b/Source/BoundaryConditions/PML_RZ.H @@ -12,7 +12,7 @@ #include "Utils/WarpXAlgorithmSelection.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" #endif @@ -40,7 +40,7 @@ public: std::array GetE_fp (); std::array GetB_fp (); -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT void PushPSATD (int lev); #endif @@ -62,7 +62,7 @@ private: std::array,2> pml_E_fp; std::array,2> pml_B_fp; -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT void PushPMLPSATDSinglePatchRZ ( int lev, SpectralSolverRZ& solver, std::array,2>& pml_E, diff --git a/Source/BoundaryConditions/PML_RZ.cpp b/Source/BoundaryConditions/PML_RZ.cpp index faa7609b3ab..78f3cf24987 100644 --- a/Source/BoundaryConditions/PML_RZ.cpp +++ b/Source/BoundaryConditions/PML_RZ.cpp @@ -9,7 +9,7 @@ #include "BoundaryConditions/PML_RZ.H" #include "FieldSolver/Fields.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include "FieldSolver/SpectralSolver/SpectralFieldDataRZ.H" #endif #include "Utils/WarpXConst.H" @@ -187,7 +187,7 @@ PML_RZ::Restart (const std::string& dir) } } -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT void PML_RZ::PushPSATD (const int lev) { diff --git a/Source/BoundaryConditions/WarpXEvolvePML.cpp b/Source/BoundaryConditions/WarpXEvolvePML.cpp index af721d70b6d..ec696689373 100644 --- a/Source/BoundaryConditions/WarpXEvolvePML.cpp +++ b/Source/BoundaryConditions/WarpXEvolvePML.cpp @@ -8,7 +8,7 @@ #include "WarpX.H" #include "BoundaryConditions/PML.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif #include "PML_current.H" @@ -60,7 +60,7 @@ WarpX::DampPML (const int lev, PatchType patch_type) if (!do_pml) { return; } WARPX_PROFILE("WarpX::DampPML()"); -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { pml_rz[lev]->ApplyDamping(Efield_fp[lev][1].get(), Efield_fp[lev][2].get(), Bfield_fp[lev][1].get(), Bfield_fp[lev][2].get(), diff --git a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp index a5e0004325a..2b063b99a15 100644 --- a/Source/BoundaryConditions/WarpXFieldBoundaries.cpp +++ b/Source/BoundaryConditions/WarpXFieldBoundaries.cpp @@ -201,8 +201,8 @@ WarpX::ApplyFieldBoundaryOnAxis (amrex::MultiFab* Er, amrex::MultiFab* Et, amrex // Lower corner of tile box physical domain // Note that this is done before the tilebox.grow so that // these do not include the guard cells. - const std::array& xyzmin = LowerCorner(tilebox, lev, 0._rt); - const amrex::Real rmin = xyzmin[0]; + const amrex::XDim3 xyzmin = LowerCorner(tilebox, lev, 0._rt); + const amrex::Real rmin = xyzmin.x; // Skip blocks that don't touch the axis if (rmin > 0._rt) { continue; } diff --git a/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt b/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt index 5e0eeaab73a..2a5cc87c0cb 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt +++ b/Source/Diagnostics/ComputeDiagFunctors/CMakeLists.txt @@ -13,5 +13,6 @@ foreach(D IN LISTS WarpX_DIMS) BackTransformFunctor.cpp BackTransformParticleFunctor.cpp ParticleReductionFunctor.cpp + TemperatureFunctor.cpp ) endforeach() diff --git a/Source/Diagnostics/ComputeDiagFunctors/Make.package b/Source/Diagnostics/ComputeDiagFunctors/Make.package index fd1624b8708..0fd618748e3 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/Make.package +++ b/Source/Diagnostics/ComputeDiagFunctors/Make.package @@ -9,5 +9,6 @@ CEXE_sources += RhoFunctor.cpp CEXE_sources += BackTransformFunctor.cpp CEXE_sources += BackTransformParticleFunctor.cpp CEXE_sources += ParticleReductionFunctor.cpp +CEXE_sources += TemperatureFunctor.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Diagnostics/ComputeDiagFunctors diff --git a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp index 340cc635fea..32e11903778 100644 --- a/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp +++ b/Source/Diagnostics/ComputeDiagFunctors/RhoFunctor.cpp @@ -1,7 +1,7 @@ #include "RhoFunctor.H" #include "Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) #include "FieldSolver/SpectralSolver/SpectralFieldData.H" #include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" #include "Utils/WarpXAlgorithmSelection.H" @@ -60,7 +60,7 @@ RhoFunctor::operator() ( amrex::MultiFab& mf_dst, const int dcomp, const int /*i // apply the filtering if requested. warpx.ApplyFilterandSumBoundaryRho(m_lev, m_lev, *rho, 0, rho->nComp()); -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) // Apply k-space filtering when using the PSATD solver if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { diff --git a/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.H b/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.H new file mode 100644 index 00000000000..f6c425e74d5 --- /dev/null +++ b/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.H @@ -0,0 +1,35 @@ +#ifndef WARPX_TEMPERATUREFUNCTOR_H_ +#define WARPX_TEMPERATUREFUNCTOR_H_ + +#include "ComputeDiagFunctor.H" + +#include + +/** + * \brief Functor to calculate per-cell averages of particle temperature + */ +class TemperatureFunctor final : public ComputeDiagFunctor +{ +public: + /** \brief Constructor + * \param[in] lev level of multifab + * \param[in] crse_ratio for interpolating field values from simulation MultiFabs + to the output diagnostic MultiFab mf_dst + * \param[in] ispec index of the species over which to calculate the temperature + * \param[in] ncomp Number of component of mf_src to cell-center in dst multifab + */ + TemperatureFunctor(int lev, amrex::IntVect crse_ratio, int ispec, int ncomp=1); + + /** \brief Compute the temperature in each grid cell. + * + * \param[out] mf_dst output MultiFab where the result is written + * \param[in] dcomp first component of mf_dst in which cell-centered + * data is stored + */ + void operator()(amrex::MultiFab& mf_dst, int dcomp, int /*i_buffer=0*/) const override; +private: + int const m_lev; /**< level on which mf_src is defined */ + int const m_ispec; /**< index of species to average over */ +}; + +#endif // WARPX_TEMPERATUREFUNCTOR_H_ diff --git a/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.cpp b/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.cpp new file mode 100644 index 00000000000..c42f8970d5e --- /dev/null +++ b/Source/Diagnostics/ComputeDiagFunctors/TemperatureFunctor.cpp @@ -0,0 +1,165 @@ + +#include "TemperatureFunctor.H" + +#include "Diagnostics/ComputeDiagFunctors/ComputeDiagFunctor.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/WarpXParticleContainer.H" +#include "Utils/Parser/ParserUtils.H" +#include "WarpX.H" + +#include + +#include +#include +#include +#include +#include + +TemperatureFunctor::TemperatureFunctor (const int lev, + const amrex::IntVect crse_ratio, const int ispec, const int ncomp) + : ComputeDiagFunctor(ncomp, crse_ratio), m_lev(lev), m_ispec(ispec) +{ + // Write only in one output component. + AMREX_ALWAYS_ASSERT(ncomp == 1); +} + +void +TemperatureFunctor::operator() (amrex::MultiFab& mf_dst, const int dcomp, const int /*i_buffer*/) const +{ + using namespace amrex::literals; + auto& warpx = WarpX::GetInstance(); + + // Guard cell is set to 1 for generality. However, for a cell-centered + // output Multifab, mf_dst, the guard-cell data is not needed especially considering + // the operations performend in the CoarsenAndInterpolate function. + constexpr int ng = 1; + + // Temporary cell-centered, multi-component MultiFab for storing particles sums and result + amrex::MultiFab sum_mf(warpx.boxArray(m_lev), warpx.DistributionMap(m_lev), 7, ng); + + auto& pc = warpx.GetPartContainer().GetParticleContainer(m_ispec); + amrex::Real const mass = pc.getMass(); // Note, implicit conversion from ParticleReal + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(mass > 0., + "The temperature diagnostic can not be calculated for a massless species."); + + // Calculate the averages in two steps, first the average velocity , then the + // average velocity squared >**2. This method is more robust than the + // single step using - **2 when >> u_rms. + ParticleToMesh(pc, sum_mf, m_lev, + [=] AMREX_GPU_DEVICE (const WarpXParticleContainer::SuperParticleType& p, + amrex::Array4 const& out_array, + amrex::GpuArray const& plo, + amrex::GpuArray const& dxi) + { + // Get position in AMReX convention to calculate corresponding index. + // Ideally this will be replaced with the AMReX NGP interpolator + // Always do x direction. + int ii = 0, jj = 0, kk = 0; + const amrex::ParticleReal x = p.pos(0); + const amrex::Real lx = (x - plo[0]) * dxi[0]; + ii = static_cast(amrex::Math::floor(lx)); +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + const amrex::ParticleReal y = p.pos(1); + const amrex::Real ly = (y - plo[1]) * dxi[1]; + jj = static_cast(amrex::Math::floor(ly)); +#endif +#if defined(WARPX_DIM_3D) + const amrex::ParticleReal z = p.pos(2); + const amrex::Real lz = (z - plo[2]) * dxi[2]; + kk = static_cast(amrex::Math::floor(lz)); +#endif + + const amrex::ParticleReal w = p.rdata(PIdx::w); + const amrex::ParticleReal ux = p.rdata(PIdx::ux); + const amrex::ParticleReal uy = p.rdata(PIdx::uy); + const amrex::ParticleReal uz = p.rdata(PIdx::uz); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 0), (amrex::Real)(w)); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 1), (amrex::Real)(w*ux)); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 2), (amrex::Real)(w*uy)); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 3), (amrex::Real)(w*uz)); + }); + + // Divide value by number of particles for average + for (amrex::MFIter mfi(sum_mf, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const amrex::Box& box = mfi.tilebox(); + amrex::Array4 const& out_array = sum_mf.array(mfi); + amrex::ParallelFor(box, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + if (out_array(i,j,k,0) > 0) { + const amrex::Real invsum = 1._rt/out_array(i,j,k,0); + out_array(i,j,k,1) *= invsum; + out_array(i,j,k,2) *= invsum; + out_array(i,j,k,3) *= invsum; + } + }); + } + + // Calculate the sum of the squares, subtracting the averages + // These loops must be written out since ParticleToMesh always zeros out the mf. + const auto plo = pc.Geom(m_lev).ProbLoArray(); + const auto dxi = pc.Geom(m_lev).InvCellSizeArray(); + for (WarpXParIter pti(pc, m_lev); pti.isValid(); ++pti) + { + const long np = pti.numParticles(); + amrex::ParticleReal* wp = pti.GetAttribs(PIdx::w).dataPtr(); + amrex::ParticleReal* uxp = pti.GetAttribs(PIdx::ux).dataPtr(); + amrex::ParticleReal* uyp = pti.GetAttribs(PIdx::uy).dataPtr(); + amrex::ParticleReal* uzp = pti.GetAttribs(PIdx::uz).dataPtr(); + + auto const GetPosition = GetParticlePosition(pti); + + amrex::Array4 const& out_array = sum_mf.array(pti); + + amrex::ParallelFor(np, + [=] AMREX_GPU_DEVICE (long ip) { + // --- Get particle quantities + amrex::ParticleReal xp, yp, zp; + GetPosition.AsStored(ip, xp, yp, zp); + + // Get position in AMReX convention to calculate corresponding index. + int ii = 0, jj = 0, kk = 0; + const amrex::Real lx = (xp - plo[0]) * dxi[0]; + ii = static_cast(amrex::Math::floor(lx)); +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const amrex::Real lz = (zp - plo[1]) * dxi[1]; + jj = static_cast(amrex::Math::floor(lz)); +#elif defined(WARPX_DIM_3D) + const amrex::Real ly = (yp - plo[1]) * dxi[1]; + jj = static_cast(amrex::Math::floor(ly)); + const amrex::Real lz = (zp - plo[2]) * dxi[2]; + kk = static_cast(amrex::Math::floor(lz)); +#endif + + const amrex::ParticleReal w = wp[ip]; + const amrex::ParticleReal ux = uxp[ip] - out_array(ii, jj, kk, 1); + const amrex::ParticleReal uy = uyp[ip] - out_array(ii, jj, kk, 2); + const amrex::ParticleReal uz = uzp[ip] - out_array(ii, jj, kk, 3); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 4), (amrex::Real)(w*ux*ux)); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 5), (amrex::Real)(w*uy*uy)); + amrex::Gpu::Atomic::AddNoRet(&out_array(ii, jj, kk, 6), (amrex::Real)(w*uz*uz)); + }); + } + + // Divide the squares by number of particles for average and calculate the temperature + for (amrex::MFIter mfi(sum_mf, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + const amrex::Box& box = mfi.tilebox(); + amrex::Array4 const& out_array = sum_mf.array(mfi); + amrex::ParallelFor(box, + [=] AMREX_GPU_DEVICE (int i, int j, int k) { + if (out_array(i,j,k,0) > 0) { + const amrex::Real invsum = 1._rt/out_array(i,j,k,0); + out_array(i,j,k,4) *= invsum; + out_array(i,j,k,5) *= invsum; + out_array(i,j,k,6) *= invsum; + out_array(i,j,k,0) = mass*(out_array(i,j,k,4) + out_array(i,j,k,5) + out_array(i,j,k,6))/(3._rt*PhysConst::q_e); + } + }); + } + + // Coarsen and interpolate from sum_mf to the output diagnostic MultiFab, mf_dst. + ablastr::coarsen::sample::Coarsen(mf_dst, sum_mf, dcomp, 0, nComp(), 0, m_crse_ratio); + +} diff --git a/Source/Diagnostics/Diagnostics.H b/Source/Diagnostics/Diagnostics.H index c0d2a9f0d53..20550364fb7 100644 --- a/Source/Diagnostics/Diagnostics.H +++ b/Source/Diagnostics/Diagnostics.H @@ -304,6 +304,8 @@ protected: int m_num_buffers; /** Array of species indices that dump rho per species */ amrex::Vector m_rho_per_species_index; + /** Array of species indices that dump temperature per species */ + amrex::Vector m_T_per_species_index; /** Vector of particle buffer vectors for each snapshot */ amrex::Vector< amrex::Vector > > m_particles_buffer; /** Vector of pointers to functors to compute particle output per species*/ diff --git a/Source/Diagnostics/Diagnostics.cpp b/Source/Diagnostics/Diagnostics.cpp index cb2b0981c94..96da344c805 100644 --- a/Source/Diagnostics/Diagnostics.cpp +++ b/Source/Diagnostics/Diagnostics.cpp @@ -278,6 +278,30 @@ Diagnostics::BaseReadParameters () + ".fields_to_plot does not match any species" ); } + // Check if m_varnames contains a string of the form T_ + if (var.rfind("T_", 0) == 0) { + // Extract species name from the string T_ + const std::string species = var.substr(var.find("T_") + 2); + // Boolean used to check if species name was misspelled + bool species_name_is_wrong = true; + // Loop over all species + for (int i = 0, n = int(m_all_species_names.size()); i < n; i++) { + // Check if species name extracted from the string T_ + // matches any of the species in the simulation + if (species == m_all_species_names[i]) { + // Store species index: will be used in TemperatureFunctor to dump + // T for this species + m_T_per_species_index.push_back(i); + species_name_is_wrong = false; + } + } + // If species name was misspelled, abort with error message + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !species_name_is_wrong, + "Input error: string " + var + " in " + m_diag_name + + ".fields_to_plot does not match any species" + ); + } } const bool checkpoint_compatibility = ( diff --git a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp index 27d0ad84b9c..1a3318ae0d8 100644 --- a/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp +++ b/Source/Diagnostics/FlushFormats/FlushFormatCheckpoint.cpp @@ -1,7 +1,7 @@ #include "FlushFormatCheckpoint.H" #include "BoundaryConditions/PML.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif #include "Diagnostics/ParticleDiag/ParticleDiag.H" @@ -152,7 +152,7 @@ FlushFormatCheckpoint::WriteToFile ( warpx.GetPML(lev)->CheckPoint( amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "pml")); } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (warpx.GetPML_RZ(lev)) { warpx.GetPML_RZ(lev)->CheckPoint( amrex::MultiFabFileFullPrefix(lev, checkpointname, default_level_prefix, "pml_rz")); diff --git a/Source/Diagnostics/FullDiagnostics.cpp b/Source/Diagnostics/FullDiagnostics.cpp index b25f899e29d..55af73d6408 100644 --- a/Source/Diagnostics/FullDiagnostics.cpp +++ b/Source/Diagnostics/FullDiagnostics.cpp @@ -8,6 +8,7 @@ #include "ComputeDiagFunctors/PartPerCellFunctor.H" #include "ComputeDiagFunctors/PartPerGridFunctor.H" #include "ComputeDiagFunctors/ParticleReductionFunctor.H" +#include "ComputeDiagFunctors/TemperatureFunctor.H" #include "ComputeDiagFunctors/RhoFunctor.H" #include "Diagnostics/Diagnostics.H" #include "Diagnostics/ParticleDiag/ParticleDiag.H" @@ -186,6 +187,8 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) // Species index to loop over species that dump rho per species int i = 0; + // Species index to loop over species that dump temperature per species + int i_T_species = 0; const int ncomp = ncomp_multimodefab; // This function is called multiple times, for different values of `lev` // but the `varnames` need only be updated once. @@ -303,6 +306,13 @@ FullDiagnostics::InitializeFieldFunctorsRZopenPMD (int lev) AddRZModesToOutputNames(std::string("rho_") + m_all_species_names[m_rho_per_species_index[i]], ncomp); } i++; + } else if ( m_varnames_fields[comp].rfind("T_", 0) == 0 ){ + // Initialize temperature functor to dump temperature per species + m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, m_T_per_species_index[i_T_species]); + if (update_varnames) { + AddRZModesToOutputNames(std::string("T_") + m_all_species_names[m_T_per_species_index[i_T_species]], ncomp); + } + i_T_species++; } else if ( m_varnames_fields[comp] == "F" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::F_fp, lev), lev, m_crse_ratio, false, ncomp); @@ -637,6 +647,9 @@ FullDiagnostics::InitializeFieldFunctors (int lev) // Species index to loop over species that dump rho per species int i = 0; + // Species index to loop over species that dump temperature per species + int i_T_species = 0; + const auto nvar = static_cast(m_varnames_fields.size()); const auto nspec = static_cast(m_pfield_species.size()); const auto ntot = static_cast(nvar + m_pfield_varnames.size() * nspec); @@ -666,6 +679,10 @@ FullDiagnostics::InitializeFieldFunctors (int lev) // Initialize rho functor to dump rho per species m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, true, m_rho_per_species_index[i]); i++; + } else if ( m_varnames[comp].rfind("T_", 0) == 0 ){ + // Initialize temperature functor to dump temperature per species + m_all_field_functors[lev][comp] = std::make_unique(lev, m_crse_ratio, m_T_per_species_index[i_T_species]); + i_T_species++; } else if ( m_varnames[comp] == "F" ){ m_all_field_functors[lev][comp] = std::make_unique(warpx.getFieldPointer(FieldType::F_fp, lev), lev, m_crse_ratio); } else if ( m_varnames[comp] == "G" ){ diff --git a/Source/Diagnostics/ParticleIO.cpp b/Source/Diagnostics/ParticleIO.cpp index f1f4d426a4a..bfb1867e741 100644 --- a/Source/Diagnostics/ParticleIO.cpp +++ b/Source/Diagnostics/ParticleIO.cpp @@ -18,6 +18,7 @@ #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" +#include "WarpX.H" #include diff --git a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp index ae7d5230b4c..9b5a28a3516 100644 --- a/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/BeamRelevant.cpp @@ -175,15 +175,6 @@ void BeamRelevant::ComputeDiags (int step) // inverse of speed of light squared Real constexpr inv_c2 = 1.0_rt / (PhysConst::c * PhysConst::c); - // If 2D-XZ, p.pos(1) is z, rather than p.pos(2). -#if (defined WARPX_DIM_3D) - int const index_z = 2; -#elif (defined WARPX_DIM_XZ || defined WARPX_DIM_RZ) - int const index_z = 1; -#elif (defined WARPX_DIM_1D_Z) - int const index_z = 0; -#endif - // loop over species for (int i_s = 0; i_s < nSpecies; ++i_s) { @@ -212,26 +203,14 @@ void BeamRelevant::ComputeDiags (int step) const ParticleReal p_uy = p.rdata(PIdx::uy); const ParticleReal p_uz = p.rdata(PIdx::uz); const ParticleReal p_us = p_ux*p_ux + p_uy*p_uy + p_uz*p_uz; - const ParticleReal p_pos0 = p.pos(0); const ParticleReal p_w = p.rdata(PIdx::w); -#if defined(WARPX_DIM_3D) - const ParticleReal p_pos1 = p.pos(1); - const ParticleReal p_x_mean = p_pos0*p_w; - const ParticleReal p_y_mean = p_pos1*p_w; -#elif defined(WARPX_DIM_RZ) - const ParticleReal p_theta = p.rdata(PIdx::theta); - const ParticleReal p_x_mean = p_pos0*std::cos(p_theta)*p_w; - const ParticleReal p_y_mean = p_pos0*std::sin(p_theta)*p_w; -#elif defined(WARPX_DIM_XZ) - const ParticleReal p_x_mean = p_pos0*p_w; - const ParticleReal p_y_mean = 0; -#elif defined(WARPX_DIM_1D_Z) - amrex::ignore_unused(p_pos0); - const ParticleReal p_x_mean = 0; - const ParticleReal p_y_mean = 0; -#endif - const ParticleReal p_z_mean = p.pos(index_z)*p_w; + ParticleReal p_x, p_y, p_z; + get_particle_position(p, p_x, p_y, p_z); + + const ParticleReal p_x_mean = p_x*p_w; + const ParticleReal p_y_mean = p_y*p_w; + const ParticleReal p_z_mean = p_z*p_w; const ParticleReal p_ux_mean = p_ux*p_w; const ParticleReal p_uy_mean = p_uy*p_w; @@ -292,25 +271,8 @@ void BeamRelevant::ComputeDiags (int step) const ParticleReal p_gm = std::sqrt(1.0_rt+p_us*inv_c2); const ParticleReal p_w = p.rdata(PIdx::w); -#if (defined WARPX_DIM_1D_Z) - const ParticleReal p_x = 0.0; - const ParticleReal p_y = 0.0; -#elif (defined WARPX_DIM_RZ) - const ParticleReal p_pos0 = p.pos(0); - const ParticleReal p_theta = p.rdata(PIdx::theta); - const ParticleReal p_x = p_pos0*std::cos(p_theta); - const ParticleReal p_y = p_pos0*std::sin(p_theta); -#elif (defined WARPX_DIM_XZ) - const ParticleReal p_pos0 = p.pos(0); - const ParticleReal p_x = p_pos0; - const ParticleReal p_y = 0.0; -#else - const ParticleReal p_pos0 = p.pos(0); - const ParticleReal p_pos1 = p.pos(1); - const ParticleReal p_x = p_pos0; - const ParticleReal p_y = p_pos1; -#endif - const ParticleReal p_z = p.pos(index_z); + ParticleReal p_x, p_y, p_z; + get_particle_position(p, p_x, p_y, p_z); const ParticleReal p_x_ms = (p_x-x_mean)*(p_x-x_mean)*p_w; const ParticleReal p_y_ms = (p_y-y_mean)*(p_y-y_mean)*p_w; diff --git a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp index 434ab1d7949..1b0c74cf7fa 100644 --- a/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp +++ b/Source/Diagnostics/ReducedDiags/ColliderRelevant.cpp @@ -441,8 +441,7 @@ void ColliderRelevant::ComputeDiags (int step) const int lev = 0; // define variables in preparation for field gathering - const std::array& dx = WarpX::CellSize(std::max(lev, 0)); - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(lev, 0)); const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); @@ -478,8 +477,7 @@ void ColliderRelevant::ComputeDiags (int step) amrex::Box box = pti.tilebox(); box.grow(ngEB); const amrex::Dim3 lo = amrex::lbound(box); - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, lev, 0._rt); const amrex::Array4 & ex_arr = Ex[pti].array(); const amrex::Array4 & ey_arr = Ey[pti].array(); const amrex::Array4 & ez_arr = Ez[pti].array(); @@ -515,7 +513,7 @@ void ColliderRelevant::ComputeDiags (int step) ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, + dinv, xyzmin, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); // compute chi amrex::Real chi = 0.0_rt; diff --git a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp index f4b4e2a39a1..9c0fac101a2 100644 --- a/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldEnergy.cpp @@ -98,15 +98,9 @@ void FieldEnergy::ComputeDiags (int step) const MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); - // get cell size - Geometry const & geom = warpx.Geom(lev); -#if defined(WARPX_DIM_1D_Z) - auto dV = geom.CellSize(0); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - auto dV = geom.CellSize(0) * geom.CellSize(1); -#elif defined(WARPX_DIM_3D) - auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); -#endif + // get cell volume + const std::array &dx = WarpX::CellSize(lev); + const amrex::Real dV = dx[0]*dx[1]*dx[2]; #if defined(WARPX_DIM_RZ) amrex::Real const tmpEx = ComputeNorm2RZ(Ex, lev); @@ -119,6 +113,8 @@ void FieldEnergy::ComputeDiags (int step) amrex::Real const tmpBz = ComputeNorm2RZ(Bz, lev); amrex::Real const Bs = tmpBx + tmpBy + tmpBz; #else + Geometry const & geom = warpx.Geom(lev); + // compute E squared Real const tmpEx = Ex.norm2(0,geom.periodicity()); Real const tmpEy = Ey.norm2(0,geom.periodicity()); @@ -182,10 +178,10 @@ FieldEnergy::ComputeNorm2RZ(const amrex::MultiFab& field, const int lev) amrex::Box tb = convert(tilebox, field.ixType().toIntVect()); // Lower corner of tile box physical domain - const std::array& xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); const Dim3 lo = lbound(tilebox); const Dim3 hi = ubound(tilebox); - const Real rmin = xyzmin[0] + (tb.ixType().nodeCentered(0) ? 0._rt : 0.5_rt*dr); + const Real rmin = xyzmin.x + (tb.ixType().nodeCentered(0) ? 0._rt : 0.5_rt*dr); const int irmin = lo.x; const int irmax = hi.x; diff --git a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp index f182acd5ba2..7eb16efecff 100644 --- a/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldMomentum.cpp @@ -183,15 +183,9 @@ void FieldMomentum::ComputeDiags (int step) amrex::Real ExB_z = amrex::get<2>(r); amrex::ParallelDescriptor::ReduceRealSum({ExB_x,ExB_y,ExB_z}); - // Get cell size - amrex::Geometry const & geom = warpx.Geom(lev); -#if defined(WARPX_DIM_1D_Z) - auto dV = geom.CellSize(0); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - auto dV = geom.CellSize(0) * geom.CellSize(1); -#elif defined(WARPX_DIM_3D) - auto dV = geom.CellSize(0) * geom.CellSize(1) * geom.CellSize(2); -#endif + // Get cell volume + const std::array &dx = WarpX::CellSize(lev); + const amrex::Real dV = dx[0]*dx[1]*dx[2]; // Save data (offset: 3 values for each refinement level) const int offset = lev*3; diff --git a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp index 7364d5989f1..1113075f4cd 100644 --- a/Source/Diagnostics/ReducedDiags/FieldProbe.cpp +++ b/Source/Diagnostics/ReducedDiags/FieldProbe.cpp @@ -154,7 +154,12 @@ FieldProbe::FieldProbe (const std::string& rd_name) ablastr::warn_manager::WarnPriority::low); } - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(interp_order <= WarpX::nox , + // ensure assumption holds: we read the fields in the interpolation kernel as they are, + // without further communication of guard/ghost/halo regions + int particle_shape; + const ParmParse pp_algo("algo"); + utils::parser::getWithParser(pp_algo, "particle_shape", particle_shape); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(interp_order <= particle_shape , "Field probe interp_order should be less than or equal to algo.particle_shape"); if (ParallelDescriptor::IOProcessor()) { @@ -484,11 +489,8 @@ void FieldProbe::ComputeDiags (int step) auto * const AMREX_RESTRICT idcpu = pti.GetStructOfArrays().GetIdCPUData().data(); - const auto &xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - const std::array &dx = WarpX::CellSize(lev); - - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, lev, 0._rt); + const amrex::XDim3 dinv = WarpX::InvCellSize(lev); const Dim3 lo = lbound(box); // Temporarily defining modes and interp outside ParallelFor to avoid GPU compilation errors. @@ -509,7 +511,7 @@ void FieldProbe::ComputeDiags (int step) doGatherShapeN(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, arrEx, arrEy, arrEz, arrBx, arrBy, arrBz, Extype, Eytype, Eztype, Bxtype, Bytype, Bztype, - dx_arr, xyzmin_arr, lo, temp_modes, + dinv, xyzmin, lo, temp_modes, temp_interp_order, false); //Calculate the Poynting Vector S diff --git a/Source/Diagnostics/ReducedDiags/FieldReduction.H b/Source/Diagnostics/ReducedDiags/FieldReduction.H index a15d6489e2c..f467499ad56 100644 --- a/Source/Diagnostics/ReducedDiags/FieldReduction.H +++ b/Source/Diagnostics/ReducedDiags/FieldReduction.H @@ -212,15 +212,15 @@ public: amrex::Real reduce_value = amrex::get<0>(reduce_data.value()); // MPI reduce - if (std::is_same::value) + if (std::is_same_v) { amrex::ParallelDescriptor::ReduceRealMax(reduce_value); } - if (std::is_same::value) + if (std::is_same_v) { amrex::ParallelDescriptor::ReduceRealMin(reduce_value); } - if (std::is_same::value) + if (std::is_same_v) { amrex::ParallelDescriptor::ReduceRealSum(reduce_value); // If reduction operation is a sum, multiply the value by the cell volume so that the diff --git a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp index 0cc5429be7a..811fee7a9de 100644 --- a/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp +++ b/Source/Diagnostics/ReducedDiags/ParticleExtrema.cpp @@ -51,7 +51,7 @@ #include #include -using namespace amrex; +using namespace amrex::literals; using namespace warpx::fields; // constructor @@ -59,7 +59,7 @@ ParticleExtrema::ParticleExtrema (const std::string& rd_name) : ReducedDiags{rd_name} { // read species name - const ParmParse pp_rd_name(rd_name); + const amrex::ParmParse pp_rd_name(rd_name); pp_rd_name.get("species",m_species_name); // get WarpX class object @@ -122,7 +122,7 @@ ParticleExtrema::ParticleExtrema (const std::string& rd_name) m_data.resize(all_diag_names.size()); - if (ParallelDescriptor::IOProcessor()) + if (amrex::ParallelDescriptor::IOProcessor()) { if ( m_write_header ) { @@ -165,16 +165,7 @@ void ParticleExtrema::ComputeDiags (int step) const auto species_names = mypc.GetSpeciesNames(); // inverse of speed of light squared - Real constexpr inv_c2 = 1.0_rt / (PhysConst::c * PhysConst::c); - - // If 2D-XZ, p.pos(1) is z, rather than p.pos(2). -#if (defined WARPX_DIM_3D) - int const index_z = 2; -#elif (defined WARPX_DIM_XZ || defined WARPX_DIM_RZ) - int const index_z = 1; -#elif (defined WARPX_DIM_1D_Z) - int const index_z = 0; -#endif + amrex::Real constexpr inv_c2 = 1.0_rt / (PhysConst::c * PhysConst::c); // loop over species for (int i_s = 0; i_s < nSpecies; ++i_s) @@ -193,172 +184,72 @@ void ParticleExtrema::ComputeDiags (int step) } using PType = typename WarpXParticleContainer::SuperParticleType; - - // xmin -#if (defined WARPX_DIM_RZ) - Real xmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(0)*std::cos(p.rdata(PIdx::theta)); }); -#elif (defined WARPX_DIM_1D_Z) - Real xmin = 0.0_rt; -#else - Real xmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(0); }); -#endif - - // xmax -#if (defined WARPX_DIM_RZ) - Real xmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(0)*std::cos(p.rdata(PIdx::theta)); }); -#elif (defined WARPX_DIM_1D_Z) - Real xmax = 0.0_rt; -#else - Real xmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(0); }); -#endif - - // ymin -#if (defined WARPX_DIM_RZ) - Real ymin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(0)*std::sin(p.rdata(PIdx::theta)); }); -#elif (defined WARPX_DIM_XZ || WARPX_DIM_1D_Z) - Real ymin = 0.0_rt; -#else - Real ymin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(1); }); -#endif - - // ymax -#if (defined WARPX_DIM_RZ) - Real ymax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(0)*std::sin(p.rdata(PIdx::theta)); }); -#elif (defined WARPX_DIM_XZ || WARPX_DIM_1D_Z) - Real ymax = 0.0_rt; -#else - Real ymax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(1); }); -#endif - - // zmin - Real zmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(index_z); }); - - // zmax - Real zmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.pos(index_z); }); - - // uxmin - Real uxmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::ux); }); - - // uxmax - Real uxmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::ux); }); - - // uymin - Real uymin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::uy); }); - - // uymax - Real uymax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::uy); }); - - // uzmin - Real uzmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::uz); }); - - // uzmax - Real uzmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::uz); }); - - // gmin - Real gmin = 0.0_rt; - if ( is_photon ) { - gmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) + using OpMin = amrex::ReduceOpMin; + using OpMax = amrex::ReduceOpMax; + + amrex::ReduceOps reduce_ops; + auto posminmax = amrex::ParticleReduce>( + myspc, + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple { - const Real ux = p.rdata(PIdx::ux); - const Real uy = p.rdata(PIdx::uy); - const Real uz = p.rdata(PIdx::uz); - const Real us = ux*ux + uy*uy + uz*uz; - return std::sqrt(us*inv_c2); - }); - } else { - gmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) + amrex::ParticleReal x, y, z; + get_particle_position(p, x, y, z); + amrex::Real const w = p.rdata(PIdx::w); + return {w, x, y, z, w, x, y, z}; + }, + reduce_ops); + + amrex::Real wmin = amrex::get<0>(posminmax); + amrex::Real xmin = amrex::get<1>(posminmax); + amrex::Real ymin = amrex::get<2>(posminmax); + amrex::Real zmin = amrex::get<3>(posminmax); + amrex::Real wmax = amrex::get<4>(posminmax); + amrex::Real xmax = amrex::get<5>(posminmax); + amrex::Real ymax = amrex::get<6>(posminmax); + amrex::Real zmax = amrex::get<7>(posminmax); + + amrex::Real const gfactor = (is_photon ? 0._rt : 1._rt); + auto uminmax = amrex::ParticleReduce>( + myspc, + [=] AMREX_GPU_DEVICE(const PType& p) noexcept -> amrex::GpuTuple { - const Real ux = p.rdata(PIdx::ux); - const Real uy = p.rdata(PIdx::uy); - const Real uz = p.rdata(PIdx::uz); - const Real us = ux*ux + uy*uy + uz*uz; - return std::sqrt(1.0_rt + us*inv_c2); - }); - } - - // gmax - Real gmax = 0.0_rt; - if ( is_photon ) { - gmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { - const Real ux = p.rdata(PIdx::ux); - const Real uy = p.rdata(PIdx::uy); - const Real uz = p.rdata(PIdx::uz); - const Real us = ux*ux + uy*uy + uz*uz; - return std::sqrt(us*inv_c2); - }); - } else { - gmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { - const Real ux = p.rdata(PIdx::ux); - const Real uy = p.rdata(PIdx::uy); - const Real uz = p.rdata(PIdx::uz); - const Real us = ux*ux + uy*uy + uz*uz; - return std::sqrt(1.0_rt + us*inv_c2); - }); - } - - // wmin - Real wmin = ReduceMin( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::w); }); - - // wmax - Real wmax = ReduceMax( myspc, - [=] AMREX_GPU_HOST_DEVICE (const PType& p) - { return p.rdata(PIdx::w); }); - - ParallelDescriptor::ReduceRealMin({xmin,ymin,zmin,uxmin,uymin,uzmin,gmin,wmin}); - ParallelDescriptor::ReduceRealMax({xmax,ymax,zmax,uxmax,uymax,uzmax,gmax,wmax}); + amrex::Real const ux = p.rdata(PIdx::ux); + amrex::Real const uy = p.rdata(PIdx::uy); + amrex::Real const uz = p.rdata(PIdx::uz); + amrex::Real const g = std::sqrt(gfactor + (ux*ux + uy*uy + uz*uz)*inv_c2); + return {g, ux, uy, uz, g, ux, uy, uz}; + }, + reduce_ops); + + amrex::Real gmin = amrex::get<0>(uminmax); + amrex::Real uxmin = amrex::get<1>(uminmax); + amrex::Real uymin = amrex::get<2>(uminmax); + amrex::Real uzmin = amrex::get<3>(uminmax); + amrex::Real gmax = amrex::get<4>(uminmax); + amrex::Real uxmax = amrex::get<5>(uminmax); + amrex::Real uymax = amrex::get<6>(uminmax); + amrex::Real uzmax = amrex::get<7>(uminmax); + + amrex::ParallelDescriptor::ReduceRealMin({xmin,ymin,zmin,uxmin,uymin,uzmin,gmin,wmin}); + amrex::ParallelDescriptor::ReduceRealMax({xmax,ymax,zmax,uxmax,uymax,uzmax,gmax,wmax}); #if (defined WARPX_QED) // get number of level (int) const auto level_number = WarpX::GetInstance().finestLevel(); // compute chimin and chimax - Real chimin_f = 0.0_rt; - Real chimax_f = 0.0_rt; + amrex::Real chimin_f = 0.0_rt; + amrex::Real chimax_f = 0.0_rt; if (myspc.DoQED()) { // declare chi arrays - std::vector chimin, chimax; + std::vector chimin, chimax; chimin.resize(level_number+1,0.0_rt); chimax.resize(level_number+1,0.0_rt); @@ -373,18 +264,18 @@ void ParticleExtrema::ComputeDiags (int step) for (int lev = 0; lev <= level_number; ++lev) { // define variables in preparation for field gathering - const std::array& dx = WarpX::CellSize(std::max(lev, 0)); - const GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); - const MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); - const MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); - const MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); - const MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); - const MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(lev, 0)); + + const amrex::MultiFab & Ex = warpx.getField(FieldType::Efield_aux, lev,0); + const amrex::MultiFab & Ey = warpx.getField(FieldType::Efield_aux, lev,1); + const amrex::MultiFab & Ez = warpx.getField(FieldType::Efield_aux, lev,2); + const amrex::MultiFab & Bx = warpx.getField(FieldType::Bfield_aux, lev,0); + const amrex::MultiFab & By = warpx.getField(FieldType::Bfield_aux, lev,1); + const amrex::MultiFab & Bz = warpx.getField(FieldType::Bfield_aux, lev,2); // declare reduce_op - ReduceOps reduce_op; - ReduceData reduce_data(reduce_op); + amrex::ReduceOps reduce_op; + amrex::ReduceData reduce_data(reduce_op); using ReduceTuple = typename decltype(reduce_data)::Type; // Loop over boxes @@ -408,28 +299,27 @@ void ParticleExtrema::ComputeDiags (int step) // define variables in preparation for field gathering amrex::Box box = pti.tilebox(); box.grow(ngEB); - const Dim3 lo = amrex::lbound(box); - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - const GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + const amrex::Dim3 lo = amrex::lbound(box); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, lev, 0._rt); const auto& ex_arr = Ex[pti].array(); const auto& ey_arr = Ey[pti].array(); const auto& ez_arr = Ez[pti].array(); const auto& bx_arr = Bx[pti].array(); const auto& by_arr = By[pti].array(); const auto& bz_arr = Bz[pti].array(); - const IndexType ex_type = Ex[pti].box().ixType(); - const IndexType ey_type = Ey[pti].box().ixType(); - const IndexType ez_type = Ez[pti].box().ixType(); - const IndexType bx_type = Bx[pti].box().ixType(); - const IndexType by_type = By[pti].box().ixType(); - const IndexType bz_type = Bz[pti].box().ixType(); + const amrex::IndexType ex_type = Ex[pti].box().ixType(); + const amrex::IndexType ey_type = Ey[pti].box().ixType(); + const amrex::IndexType ez_type = Ez[pti].box().ixType(); + const amrex::IndexType bx_type = Bx[pti].box().ixType(); + const amrex::IndexType by_type = By[pti].box().ixType(); + const amrex::IndexType bz_type = Bz[pti].box().ixType(); // evaluate reduce_op reduce_op.eval(pti.numParticles(), reduce_data, [=] AMREX_GPU_DEVICE (int i) -> ReduceTuple { // get external fields - ParticleReal xp, yp, zp; + amrex::ParticleReal xp, yp, zp; GetPosition(i, xp, yp, zp); amrex::ParticleReal ex = Ex_external_particle; amrex::ParticleReal ey = Ey_external_particle; @@ -446,10 +336,10 @@ void ParticleExtrema::ComputeDiags (int step) ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, + dinv, xyzmin, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); // compute chi - Real chi = 0.0_rt; + amrex::Real chi = 0.0_rt; if ( is_photon ) { chi = QedUtils::chi_photon(ux[i]*m, uy[i]*m, uz[i]*m, ex, ey, ez, bx, by, bz); @@ -461,13 +351,13 @@ void ParticleExtrema::ComputeDiags (int step) }); } auto val = reduce_data.value(); - chimin[lev] = get<0>(val); - chimax[lev] = get<1>(val); + chimin[lev] = amrex::get<0>(val); + chimax[lev] = amrex::get<1>(val); } chimin_f = *std::min_element(chimin.begin(), chimin.end()); chimax_f = *std::max_element(chimax.begin(), chimax.end()); - ParallelDescriptor::ReduceRealMin(chimin_f, ParallelDescriptor::IOProcessorNumber()); - ParallelDescriptor::ReduceRealMax(chimax_f, ParallelDescriptor::IOProcessorNumber()); + amrex::ParallelDescriptor::ReduceRealMin(chimin_f, amrex::ParallelDescriptor::IOProcessorNumber()); + amrex::ParallelDescriptor::ReduceRealMax(chimax_f, amrex::ParallelDescriptor::IOProcessorNumber()); } #endif diff --git a/Source/Diagnostics/SliceDiagnostic.cpp b/Source/Diagnostics/SliceDiagnostic.cpp index 14a2d633a0c..97af967f2be 100644 --- a/Source/Diagnostics/SliceDiagnostic.cpp +++ b/Source/Diagnostics/SliceDiagnostic.cpp @@ -153,7 +153,7 @@ CreateSlice( const MultiFab& mf, const Vector &dom_geom, ablastr::utils::communication::ParallelCopy(*smf, mf, 0, 0, ncomp, nghost_vect, nghost_vect, WarpX::do_single_precision_comms); // interpolate if required on refined slice // - if (interpolate == 1 ) { + if (interpolate) { InterpolateSliceValues( *smf, interp_lo, slice_cc_nd_box, dom_geom, ncomp, nghost, slice_lo, slice_hi, SliceType, real_box); } diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index 91409bc294d..a17abe04178 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -8,7 +8,7 @@ * License: BSD-3-Clause-LBNL */ #include "BoundaryConditions/PML.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif #include "FieldIO.H" @@ -385,7 +385,7 @@ WarpX::InitFromCheckpoint () if (pml[lev]) { pml[lev]->Restart(amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml")); } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { pml_rz[lev]->Restart(amrex::MultiFabFileFullPrefix(lev, restart_chkfile, level_prefix, "pml_rz")); } diff --git a/Source/Diagnostics/WarpXOpenPMD.cpp b/Source/Diagnostics/WarpXOpenPMD.cpp index 5afb49db597..e3eaef33c44 100644 --- a/Source/Diagnostics/WarpXOpenPMD.cpp +++ b/Source/Diagnostics/WarpXOpenPMD.cpp @@ -913,33 +913,34 @@ WarpXOpenPMDPlot::SaveRealProperty (ParticleIter& pti, { auto const real_counter = std::min(write_real_comp.size(), real_comp_names.size()); +#if defined(WARPX_DIM_RZ) // reconstruct Cartesian positions for RZ simulations // r,z,theta -> x,y,z -#if defined(WARPX_DIM_RZ) - auto const * const r = soa.GetRealData(PIdx::x).data(); - auto const * const theta = soa.GetRealData(PIdx::theta).data(); + // If each comp is being written, create a temporary array, otherwise create an empty array. + std::shared_ptr const x( + new amrex::ParticleReal[(write_real_comp[0] ? numParticleOnTile : 0)], + [](amrex::ParticleReal const *p) { delete[] p; } + ); + std::shared_ptr const y( + new amrex::ParticleReal[(write_real_comp[1] ? numParticleOnTile : 0)], + [](amrex::ParticleReal const *p) { delete[] p; } + ); + const auto& tile = pti.GetParticleTile(); + const auto& ptd = tile.getConstParticleTileData(); + + for (int i = 0; i < numParticleOnTile; ++i) { + const auto& p = ptd.getSuperParticle(i); + amrex::ParticleReal xp, yp, zp; + get_particle_position(p, xp, yp, zp); + if (write_real_comp[0]) { x.get()[i] = xp; } + if (write_real_comp[1]) { y.get()[i] = yp; } + } if (write_real_comp[0]) { - std::shared_ptr const x( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p) { delete[] p; } - ); - for (int i = 0; i < numParticleOnTile; ++i) { - x.get()[i] = r[i] * std::cos(theta[i]); - } - getComponentRecord(real_comp_names[0]).storeChunk( - x, {offset}, {numParticleOnTile64}); + getComponentRecord(real_comp_names[0]).storeChunk(x, {offset}, {numParticleOnTile64}); } if (write_real_comp[1]) { - std::shared_ptr const y( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p) { delete[] p; } - ); - for (int i = 0; i < numParticleOnTile; ++i) { - y.get()[i] = r[i] * std::sin(theta[i]); - } - getComponentRecord(real_comp_names[1]).storeChunk( - y, {offset}, {numParticleOnTile64}); + getComponentRecord(real_comp_names[1]).storeChunk(y, {offset}, {numParticleOnTile64}); } #endif diff --git a/Source/Evolve/CMakeLists.txt b/Source/Evolve/CMakeLists.txt index a84d6e1e42b..3ef0dae5e8e 100644 --- a/Source/Evolve/CMakeLists.txt +++ b/Source/Evolve/CMakeLists.txt @@ -4,6 +4,5 @@ foreach(D IN LISTS WarpX_DIMS) PRIVATE WarpXEvolve.cpp WarpXComputeDt.cpp - WarpXOneStepImplicitPicard.cpp ) endforeach() diff --git a/Source/Evolve/Make.package b/Source/Evolve/Make.package index a9a877475b9..275b8bfde5a 100644 --- a/Source/Evolve/Make.package +++ b/Source/Evolve/Make.package @@ -1,5 +1,4 @@ CEXE_sources += WarpXEvolve.cpp CEXE_sources += WarpXComputeDt.cpp -CEXE_sources += WarpXOneStepImplicitPicard.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Evolve diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 392302e59de..72ef84b8f75 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -15,7 +15,7 @@ #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" #include "Evolve/WarpXDtType.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # ifdef WARPX_DIM_RZ # include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" # else @@ -125,10 +125,8 @@ WarpX::Evolve (int numsteps) ExecutePythonCallback("particleinjection"); - // TODO: move out - if (evolve_scheme == EvolveScheme::ImplicitPicard || - evolve_scheme == EvolveScheme::SemiImplicitPicard) { - OneStep_ImplicitPicard(cur_time); + if (m_implicit_solver) { + m_implicit_solver->OneStep(cur_time, dt[0], step); } else if ( electromagnetic_solver_id == ElectromagneticSolverAlgo::None || electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC ) @@ -144,7 +142,7 @@ WarpX::Evolve (int numsteps) OneStep_multiJ(cur_time); } // Electromagnetic case: no subcycling or no mesh refinement - else if (do_subcycling == 0 || finest_level == 0) + else if ( !do_subcycling || (finest_level == 0)) { OneStep_nosub(cur_time); // E: guard cells are up-to-date @@ -152,7 +150,7 @@ WarpX::Evolve (int numsteps) // F: guard cells are NOT up-to-date } // Electromagnetic case: subcycling with one level of mesh refinement - else if (do_subcycling == 1 && finest_level == 1) + else if (do_subcycling && (finest_level == 1)) { OneStep_sub1(cur_time); } @@ -248,7 +246,12 @@ WarpX::Evolve (int numsteps) // This is currently a lab frame calculation. ComputeMagnetostaticField(); } - AddExternalFields(); + // Since the fields were reset above, the external fields are added + // back on to the fine patch fields. This make it so that the net fields + // are the sum of the field solution and any external field. + for (int lev = 0; lev <= max_level; ++lev) { + AddExternalFields(lev); + } } else if (electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { // Hybrid-PIC case: // The particles are now at p^{n+1/2} and x^{n+1}. The fields @@ -308,6 +311,9 @@ WarpX::Evolve (int numsteps) multi_diags->FilterComputePackFlushLastTimestep( istep[0] ); if (m_exit_loop_due_to_interrupt_signal) { ExecutePythonCallback("onbreaksignal"); } } + + amrex::Print() << + ablastr::warn_manager::GetWMInstance().PrintGlobalWarnings("THE END"); } /* /brief Perform one PIC iteration, without subcycling @@ -603,7 +609,7 @@ void WarpX::SyncCurrentAndRho () void WarpX::OneStep_multiJ (const amrex::Real cur_time) { -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT WARPX_ALWAYS_ASSERT_WITH_MESSAGE( WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD, @@ -766,7 +772,7 @@ WarpX::OneStep_multiJ (const amrex::Real cur_time) amrex::ignore_unused(cur_time); WARPX_ABORT_WITH_MESSAGE( "multi-J algorithm not implemented for FDTD"); -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT } /* /brief Perform one PIC iteration, with subcycling diff --git a/Source/Evolve/WarpXOneStepImplicitPicard.cpp b/Source/Evolve/WarpXOneStepImplicitPicard.cpp deleted file mode 100644 index 22d42dfe89d..00000000000 --- a/Source/Evolve/WarpXOneStepImplicitPicard.cpp +++ /dev/null @@ -1,423 +0,0 @@ -/* Copyright 2022 David Grote - * - * This file is part of WarpX. - * - * License: BSD-3-Clause-LBNL - */ -#include "WarpX.H" - -#include "BoundaryConditions/PML.H" -#include "Diagnostics/MultiDiagnostics.H" -#include "Diagnostics/ReducedDiags/MultiReducedDiags.H" -#include "Evolve/WarpXDtType.H" -#include "Evolve/WarpXPushType.H" -#ifdef WARPX_USE_PSATD -# ifdef WARPX_DIM_RZ -# include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" -# else -# include "FieldSolver/SpectralSolver/SpectralSolver.H" -# endif -#endif -#include "Parallelization/GuardCellManager.H" -#include "Particles/MultiParticleContainer.H" -#include "Particles/ParticleBoundaryBuffer.H" -#include "Python/callbacks.H" -#include "Utils/TextMsg.H" -#include "Utils/WarpXAlgorithmSelection.H" -#include "Utils/WarpXUtil.H" -#include "Utils/WarpXConst.H" -#include "Utils/WarpXProfilerWrapper.H" - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include - -void -WarpX::EvolveImplicitPicardInit (int lev) -{ - - if (lev == 0) { - // Add space to save the positions and velocities at the start of the time steps - for (auto const& pc : *mypc) { -#if (AMREX_SPACEDIM >= 2) - pc->AddRealComp("x_n"); -#endif -#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - pc->AddRealComp("y_n"); -#endif - pc->AddRealComp("z_n"); - pc->AddRealComp("ux_n"); - pc->AddRealComp("uy_n"); - pc->AddRealComp("uz_n"); - } - } - - // Initialize MultiFabs to hold the E and B fields at the start of the time steps - // Only one refinement level is supported - const int nlevs_max = maxLevel() + 1; - Efield_n.resize(nlevs_max); - Efield_save.resize(nlevs_max); - if (evolve_scheme == EvolveScheme::ImplicitPicard) { - Bfield_n.resize(nlevs_max); - Bfield_save.resize(nlevs_max); - } - - // The Efield_n and Bfield_n will hold the fields at the start of the time step. - // This is needed since in each iteration the fields are advanced from the values - // at the start of the step. - // The Efield_save and Bfield_save will hold the fields from the previous iteration, - // to check the change in the fields after the iterations to check for convergence. - // The Efiel_fp and Bfield_fp will hole the n+theta during the iterations and then - // advance to the n+1 time level after the iterations complete. - AllocInitMultiFabFromModel(Efield_n[lev][0], *Efield_fp[0][0], lev, "Efield_n[0]"); - AllocInitMultiFabFromModel(Efield_n[lev][1], *Efield_fp[0][1], lev, "Efield_n[1]"); - AllocInitMultiFabFromModel(Efield_n[lev][2], *Efield_fp[0][2], lev, "Efield_n[2]"); - AllocInitMultiFabFromModel(Efield_save[lev][0], *Efield_fp[0][0], lev, "Efield_save[0]"); - AllocInitMultiFabFromModel(Efield_save[lev][1], *Efield_fp[0][1], lev, "Efield_save[1]"); - AllocInitMultiFabFromModel(Efield_save[lev][2], *Efield_fp[0][2], lev, "Efield_save[2]"); - - if (evolve_scheme == EvolveScheme::ImplicitPicard) { - AllocInitMultiFabFromModel(Bfield_n[lev][0], *Bfield_fp[0][0], lev, "Bfield_n[0]"); - AllocInitMultiFabFromModel(Bfield_n[lev][1], *Bfield_fp[0][1], lev, "Bfield_n[1]"); - AllocInitMultiFabFromModel(Bfield_n[lev][2], *Bfield_fp[0][2], lev, "Bfield_n[2]"); - AllocInitMultiFabFromModel(Bfield_save[lev][0], *Bfield_fp[0][0], lev, "Bfield_save[0]"); - AllocInitMultiFabFromModel(Bfield_save[lev][1], *Bfield_fp[0][1], lev, "Bfield_save[1]"); - AllocInitMultiFabFromModel(Bfield_save[lev][2], *Bfield_fp[0][2], lev, "Bfield_save[2]"); - } - -} - -void -WarpX::OneStep_ImplicitPicard(amrex::Real cur_time) -{ - using namespace amrex::literals; - - // We have E^{n}. - // Particles have p^{n} and x^{n}. - // With full implicit, B^{n} - // With semi-implicit, B^{n-1/2} - - // Save the values at the start of the time step, - // copying particle data to x_n etc. - for (auto const& pc : *mypc) { - SaveParticlesAtImplicitStepStart (*pc, 0); - } - - // Save the fields at the start of the step - amrex::MultiFab::Copy(*Efield_n[0][0], *Efield_fp[0][0], 0, 0, ncomps, Efield_fp[0][0]->nGrowVect()); - amrex::MultiFab::Copy(*Efield_n[0][1], *Efield_fp[0][1], 0, 0, ncomps, Efield_fp[0][1]->nGrowVect()); - amrex::MultiFab::Copy(*Efield_n[0][2], *Efield_fp[0][2], 0, 0, ncomps, Efield_fp[0][2]->nGrowVect()); - - if (evolve_scheme == EvolveScheme::ImplicitPicard) { - amrex::MultiFab::Copy(*Bfield_n[0][0], *Bfield_fp[0][0], 0, 0, ncomps, Bfield_fp[0][0]->nGrowVect()); - amrex::MultiFab::Copy(*Bfield_n[0][1], *Bfield_fp[0][1], 0, 0, ncomps, Bfield_fp[0][1]->nGrowVect()); - amrex::MultiFab::Copy(*Bfield_n[0][2], *Bfield_fp[0][2], 0, 0, ncomps, Bfield_fp[0][2]->nGrowVect()); - } else if (evolve_scheme == EvolveScheme::SemiImplicitPicard) { - // This updates Bfield_fp so it holds the new B at n+1/2 - EvolveB(dt[0], DtType::Full); - // WarpX::sync_nodal_points is used to avoid instability - FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); - } - - // Start the iterations - amrex::Real deltaE = 1._rt; - amrex::Real deltaB = 1._rt; - int iteration_count = 0; - while (iteration_count < max_picard_iterations && - (deltaE > picard_iteration_tolerance || deltaB > picard_iteration_tolerance)) { - iteration_count++; - - // Advance the particle positions by 1/2 dt, - // particle velocities by dt, then take average of old and new v, - // deposit currents, giving J at n+1/2 - // This uses Efield_fp and Bfield_fp, the field at n+1/2 from the previous iteration. - const bool skip_current = false; - const PushType push_type = PushType::Implicit; - PushParticlesandDeposit(cur_time, skip_current, push_type); - - SyncCurrentAndRho(); - - if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { - // Save the E at n+1/2 from the previous iteration so that the change - // in this iteration can be calculated - amrex::MultiFab::Copy(*Efield_save[0][0], *Efield_fp[0][0], 0, 0, ncomps, 0); - amrex::MultiFab::Copy(*Efield_save[0][1], *Efield_fp[0][1], 0, 0, ncomps, 0); - amrex::MultiFab::Copy(*Efield_save[0][2], *Efield_fp[0][2], 0, 0, ncomps, 0); - } - - // Copy Efield_n into Efield_fp since EvolveE updates Efield_fp in place - amrex::MultiFab::Copy(*Efield_fp[0][0], *Efield_n[0][0], 0, 0, ncomps, Efield_n[0][0]->nGrowVect()); - amrex::MultiFab::Copy(*Efield_fp[0][1], *Efield_n[0][1], 0, 0, ncomps, Efield_n[0][1]->nGrowVect()); - amrex::MultiFab::Copy(*Efield_fp[0][2], *Efield_n[0][2], 0, 0, ncomps, Efield_n[0][2]->nGrowVect()); - - // Updates Efield_fp so it holds the new E at n+1/2 - EvolveE(0.5_rt*dt[0]); - // WarpX::sync_nodal_points is used to avoid instability - FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - ApplyEfieldBoundary(0, PatchType::fine); - - if (evolve_scheme == EvolveScheme::ImplicitPicard) { - if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { - // Save the B at n+1/2 from the previous iteration so that the change - // in this iteration can be calculated - amrex::MultiFab::Copy(*Bfield_save[0][0], *Bfield_fp[0][0], 0, 0, ncomps, 0); - amrex::MultiFab::Copy(*Bfield_save[0][1], *Bfield_fp[0][1], 0, 0, ncomps, 0); - amrex::MultiFab::Copy(*Bfield_save[0][2], *Bfield_fp[0][2], 0, 0, ncomps, 0); - } - - // Copy Bfield_n into Bfield_fp since EvolveB updates Bfield_fp in place - amrex::MultiFab::Copy(*Bfield_fp[0][0], *Bfield_n[0][0], 0, 0, ncomps, Bfield_n[0][0]->nGrowVect()); - amrex::MultiFab::Copy(*Bfield_fp[0][1], *Bfield_n[0][1], 0, 0, ncomps, Bfield_n[0][1]->nGrowVect()); - amrex::MultiFab::Copy(*Bfield_fp[0][2], *Bfield_n[0][2], 0, 0, ncomps, Bfield_n[0][2]->nGrowVect()); - - // This updates Bfield_fp so it holds the new B at n+1/2 - EvolveB(0.5_rt*dt[0], DtType::Full); - // WarpX::sync_nodal_points is used to avoid instability - FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); - } - - // The B field update needs - if (num_mirrors>0){ - applyMirrors(cur_time); - // E : guard cells are NOT up-to-date from the mirrors - // B : guard cells are NOT up-to-date from the mirrors - } - - if (picard_iteration_tolerance > 0. || iteration_count == max_picard_iterations) { - // Calculate the change in E and B from this iteration - // deltaE = abs(Enew - Eold)/max(abs(Enew)) - Efield_save[0][0]->minus(*Efield_fp[0][0], 0, ncomps, 0); - Efield_save[0][1]->minus(*Efield_fp[0][1], 0, ncomps, 0); - Efield_save[0][2]->minus(*Efield_fp[0][2], 0, ncomps, 0); - const amrex::Real maxE0 = std::max(1._rt, Efield_fp[0][0]->norm0(0, 0)); - const amrex::Real maxE1 = std::max(1._rt, Efield_fp[0][1]->norm0(0, 0)); - const amrex::Real maxE2 = std::max(1._rt, Efield_fp[0][2]->norm0(0, 0)); - const amrex::Real deltaE0 = Efield_save[0][0]->norm0(0, 0)/maxE0; - const amrex::Real deltaE1 = Efield_save[0][1]->norm0(0, 0)/maxE1; - const amrex::Real deltaE2 = Efield_save[0][2]->norm0(0, 0)/maxE2; - deltaE = std::max(std::max(deltaE0, deltaE1), deltaE2); - if (evolve_scheme == EvolveScheme::ImplicitPicard) { - Bfield_save[0][0]->minus(*Bfield_fp[0][0], 0, ncomps, 0); - Bfield_save[0][1]->minus(*Bfield_fp[0][1], 0, ncomps, 0); - Bfield_save[0][2]->minus(*Bfield_fp[0][2], 0, ncomps, 0); - const amrex::Real maxB0 = std::max(1._rt, Bfield_fp[0][0]->norm0(0, 0)); - const amrex::Real maxB1 = std::max(1._rt, Bfield_fp[0][1]->norm0(0, 0)); - const amrex::Real maxB2 = std::max(1._rt, Bfield_fp[0][2]->norm0(0, 0)); - const amrex::Real deltaB0 = Bfield_save[0][0]->norm0(0, 0)/maxB0; - const amrex::Real deltaB1 = Bfield_save[0][1]->norm0(0, 0)/maxB1; - const amrex::Real deltaB2 = Bfield_save[0][2]->norm0(0, 0)/maxB2; - deltaB = std::max(std::max(deltaB0, deltaB1), deltaB2); - } else { - deltaB = 0.; - } - amrex::Print() << "Max delta " << iteration_count << " " << deltaE << " " << deltaB << "\n"; - } - - // Now, the particle positions and velocities and the Efield_fp and Bfield_fp hold - // the new values at n+1/2 - } - - amrex::Print() << "Picard iterations = " << iteration_count << ", Eerror = " << deltaE << ", Berror = " << deltaB << "\n"; - if (picard_iteration_tolerance > 0. && iteration_count == max_picard_iterations) { - std::stringstream convergenceMsg; - convergenceMsg << "The Picard implicit solver failed to converge after " << iteration_count << " iterations, with Eerror = " << deltaE << ", Berror = " << deltaB << " with a tolerance of " << picard_iteration_tolerance; - if (require_picard_convergence) { - WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); - } else { - ablastr::warn_manager::WMRecordWarning("PicardSolver", convergenceMsg.str()); - } - } - - // Advance particles to step n+1 - for (auto const& pc : *mypc) { - FinishImplicitParticleUpdate(*pc, 0); - } - - // Advance fields to step n+1 - // WarpX::sync_nodal_points is used to avoid instability - FinishImplicitFieldUpdate(Efield_fp, Efield_n); - FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - if (evolve_scheme == EvolveScheme::ImplicitPicard) { - FinishImplicitFieldUpdate(Bfield_fp, Bfield_n); - FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); - } - -} - -void -WarpX::SaveParticlesAtImplicitStepStart (WarpXParticleContainer& pc, int lev) -{ - -#ifdef AMREX_USE_OMP -#pragma omp parallel -#endif - { - - auto particle_comps = pc.getParticleComps(); - - for (WarpXParIter pti(pc, lev); pti.isValid(); ++pti) { - - const auto getPosition = GetParticlePosition(pti); - - auto& attribs = pti.GetAttribs(); - amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); - amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); - amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); - -#if (AMREX_SPACEDIM >= 2) - amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); -#endif -#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); -#endif - amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); - amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); - amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); - amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); - - const long np = pti.numParticles(); - - amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) - { - amrex::ParticleReal xp, yp, zp; - getPosition(ip, xp, yp, zp); - -#if (AMREX_SPACEDIM >= 2) - x_n[ip] = xp; -#endif -#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - y_n[ip] = yp; -#endif - z_n[ip] = zp; - - ux_n[ip] = ux[ip]; - uy_n[ip] = uy[ip]; - uz_n[ip] = uz[ip]; - - }); - - } - } -} - -void -WarpX::FinishImplicitParticleUpdate (WarpXParticleContainer& pc, int lev) -{ - using namespace amrex::literals; - -#ifdef AMREX_USE_OMP -#pragma omp parallel -#endif - { - - auto particle_comps = pc.getParticleComps(); - - for (WarpXParIter pti(pc, lev); pti.isValid(); ++pti) { - - const auto getPosition = GetParticlePosition(pti); - const auto setPosition = SetParticlePosition(pti); - - auto& attribs = pti.GetAttribs(); - amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); - amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); - amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); - -#if (AMREX_SPACEDIM >= 2) - amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); -#endif -#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); -#endif - amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); - amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); - amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); - amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); - - const long np = pti.numParticles(); - - amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) - { - amrex::ParticleReal xp, yp, zp; - getPosition(ip, xp, yp, zp); - -#if (AMREX_SPACEDIM >= 2) - xp = 2._rt*xp - x_n[ip]; -#endif -#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) - yp = 2._rt*yp - y_n[ip]; -#endif - zp = 2._rt*zp - z_n[ip]; - - ux[ip] = 2._rt*ux[ip] - ux_n[ip]; - uy[ip] = 2._rt*uy[ip] - uy_n[ip]; - uz[ip] = 2._rt*uz[ip] - uz_n[ip]; - - setPosition(ip, xp, yp, zp); - }); - - } - } -} - -void -WarpX::FinishImplicitFieldUpdate(amrex::Vector, 3 > >& Field_fp, - amrex::Vector, 3 > >& Field_n) -{ - using namespace amrex::literals; - - for (int lev = 0; lev <= finest_level; ++lev) { - -#ifdef AMREX_USE_OMP -#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) -#endif - for ( amrex::MFIter mfi(*Field_fp[lev][0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi ) - { - - amrex::Array4 const& Fx = Field_fp[lev][0]->array(mfi); - amrex::Array4 const& Fy = Field_fp[lev][1]->array(mfi); - amrex::Array4 const& Fz = Field_fp[lev][2]->array(mfi); - - amrex::Array4 const& Fx_n = Field_n[lev][0]->array(mfi); - amrex::Array4 const& Fy_n = Field_n[lev][1]->array(mfi); - amrex::Array4 const& Fz_n = Field_n[lev][2]->array(mfi); - - amrex::Box const tbx = mfi.tilebox(Field_fp[lev][0]->ixType().toIntVect()); - amrex::Box const tby = mfi.tilebox(Field_fp[lev][1]->ixType().toIntVect()); - amrex::Box const tbz = mfi.tilebox(Field_fp[lev][2]->ixType().toIntVect()); - - amrex::ParallelFor( - tbx, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) - { - Fx(i,j,k,n) = 2._rt*Fx(i,j,k,n) - Fx_n(i,j,k,n); - }, - tby, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) - { - Fy(i,j,k,n) = 2._rt*Fy(i,j,k,n) - Fy_n(i,j,k,n); - }, - tbz, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) - { - Fz(i,j,k,n) = 2._rt*Fz(i,j,k,n) - Fz_n(i,j,k,n); - }); - } - } -} diff --git a/Source/FieldSolver/CMakeLists.txt b/Source/FieldSolver/CMakeLists.txt index 58dd10f41cf..859117eb214 100644 --- a/Source/FieldSolver/CMakeLists.txt +++ b/Source/FieldSolver/CMakeLists.txt @@ -11,6 +11,7 @@ endforeach() add_subdirectory(FiniteDifferenceSolver) add_subdirectory(MagnetostaticSolver) -if(WarpX_PSATD) +add_subdirectory(ImplicitSolvers) +if(WarpX_FFT) add_subdirectory(SpectralSolver) endif() diff --git a/Source/FieldSolver/ElectrostaticSolver.cpp b/Source/FieldSolver/ElectrostaticSolver.cpp index a74beae3c91..189f2f2bb0a 100644 --- a/Source/FieldSolver/ElectrostaticSolver.cpp +++ b/Source/FieldSolver/ElectrostaticSolver.cpp @@ -382,8 +382,8 @@ WarpX::computePhi (const amrex::Vector >& rho, const std::optional > eb_farray_box_factory; #endif - bool const is_solver_multigrid = - WarpX::poisson_solver_id != PoissonSolverAlgo::IntegratedGreenFunction; + bool const is_solver_igf_on_lev0 = + WarpX::poisson_solver_id == PoissonSolverAlgo::IntegratedGreenFunction; ablastr::fields::computePhi( sorted_rho, @@ -396,8 +396,9 @@ WarpX::computePhi (const amrex::Vector >& rho, this->geom, this->dmap, this->grids, + WarpX::grid_type, this->m_poisson_boundary_handler, - is_solver_multigrid, + is_solver_igf_on_lev0, WarpX::do_single_precision_comms, this->ref_ratio, post_phi_calculation, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp b/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp index 3cc05c58068..0702b264874 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/ComputeDivE.cpp @@ -52,7 +52,7 @@ void FiniteDifferenceSolver::ComputeDivE ( ComputeDivECylindrical ( Efield, divEfield ); #else - if (m_grid_type == GridType::Collocated) { + if (m_grid_type == ablastr::utils::enums::GridType::Collocated) { ComputeDivECartesian ( Efield, divEfield ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp index 1fb2dd85e60..0ad2c8d6802 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/EvolveBPML.cpp @@ -53,7 +53,7 @@ void FiniteDifferenceSolver::EvolveBPML ( WARPX_ABORT_WITH_MESSAGE( "PML are not implemented in cylindrical geometry."); #else - if (m_grid_type == GridType::Collocated) { + if (m_grid_type == ablastr::utils::enums::GridType::Collocated) { EvolveBPMLCartesian (Bfield, Efield, dt, dive_cleaning); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H index a7a4f10a713..00e87525a75 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H @@ -17,6 +17,8 @@ #include "HybridPICModel/HybridPICModel_fwd.H" #include "MacroscopicProperties/MacroscopicProperties_fwd.H" +#include + #include #include @@ -47,7 +49,7 @@ class FiniteDifferenceSolver FiniteDifferenceSolver ( int fdtd_algo, std::array cell_size, - short grid_type ); + ablastr::utils::enums::GridType grid_type ); void EvolveB ( std::array< std::unique_ptr, 3 >& Bfield, std::array< std::unique_ptr, 3 > const& Efield, @@ -178,7 +180,7 @@ class FiniteDifferenceSolver private: int m_fdtd_algo; - short m_grid_type; + ablastr::utils::enums::GridType m_grid_type; #ifdef WARPX_DIM_RZ amrex::Real m_dr, m_rmin; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp index 1bbc6c9b337..9af610031c0 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.cpp @@ -30,7 +30,7 @@ FiniteDifferenceSolver::FiniteDifferenceSolver ( int const fdtd_algo, std::array cell_size, - short grid_type): + ablastr::utils::enums::GridType grid_type): // Register the type of finite-difference algorithm m_fdtd_algo{fdtd_algo}, m_grid_type{grid_type} @@ -63,7 +63,7 @@ FiniteDifferenceSolver::FiniteDifferenceSolver ( "FiniteDifferenceSolver: Unknown algorithm"); } #else - if (grid_type == GridType::Collocated) { + if (grid_type == ablastr::utils::enums::GridType::Collocated) { CartesianNodalAlgorithm::InitializeStencilCoefficients( cell_size, m_h_stencil_coefs_x, m_h_stencil_coefs_y, m_h_stencil_coefs_z ); diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H index 5c0c2bcc96a..15208727559 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H @@ -22,6 +22,8 @@ #include #include +#include + /** * \brief This class contains the parameters needed to evaluate hybrid field * solutions (kinetic ions with fluid electrons). diff --git a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp index 8979036fbea..6a72bb3569c 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.cpp @@ -10,6 +10,7 @@ #include "HybridPICModel.H" #include "FieldSolver/Fields.H" +#include "WarpX.H" using namespace amrex; using namespace warpx::fields; diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H index 129b2e2f782..ca9c77334ea 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H @@ -38,20 +38,26 @@ public: void ReadParameters (); /** - * \brief Initialize multifabs storing macroscopic multifabs + * \brief Allocate multifabs storing macroscopic multifabs * * @param[in] ba the box array associated to the multifabs E and B * @param[in] dmap the distribution mapping * @param[in] ng_EB_alloc guard cells allocated for multifabs E and B + */ + void AllocateLevelMFs ( + const amrex::BoxArray& ba, + const amrex::DistributionMapping& dm, + const amrex::IntVect& ng_EB_alloc ); + + /** + * \brief Initialize multifabs storing macroscopic multifabs + * * @param[in] geom the geometry * @param[in] Ex_stag staggering of the Ex field * @param[in] Ey_stag staggering of the Ey field * @param[in] Ez_stag staggering of the Ez field */ void InitData ( - const amrex::BoxArray& ba, - const amrex::DistributionMapping& dmap, - const amrex::IntVect& ng_EB_alloc, const amrex::Geometry& geom, const amrex::IntVect& Ex_stag, const amrex::IntVect& Ey_stag, diff --git a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp index 1467c7a8c0c..a6a389fe056 100644 --- a/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp +++ b/Source/FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.cpp @@ -119,24 +119,26 @@ MacroscopicProperties::ReadParameters () } void -MacroscopicProperties::InitData ( +MacroscopicProperties::AllocateLevelMFs ( const amrex::BoxArray& ba, const amrex::DistributionMapping& dmap, - const amrex::IntVect& ng_EB_alloc, - const amrex::Geometry& geom, - const amrex::IntVect& Ex_stag, - const amrex::IntVect& Ey_stag, - const amrex::IntVect& Ez_stag) + const amrex::IntVect& ng_EB_alloc ) { - amrex::Print() << Utils::TextMsg::Info("we are in init data of macro"); - - // Define material property multifabs using ba and dmap from WarpX instance // sigma is cell-centered MultiFab m_sigma_mf = std::make_unique(ba, dmap, 1, ng_EB_alloc); // epsilon is cell-centered MultiFab m_eps_mf = std::make_unique(ba, dmap, 1, ng_EB_alloc); // mu is cell-centered MultiFab m_mu_mf = std::make_unique(ba, dmap, 1, ng_EB_alloc); +} + +void +MacroscopicProperties::InitData ( + const amrex::Geometry& geom, + const amrex::IntVect& Ex_stag, + const amrex::IntVect& Ey_stag, + const amrex::IntVect& Ez_stag) +{ // Initialize sigma if (m_sigma_s == "constant") { diff --git a/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt b/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt new file mode 100644 index 00000000000..6e16f19084c --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/CMakeLists.txt @@ -0,0 +1,10 @@ +foreach(D IN LISTS WarpX_DIMS) + warpx_set_suffix_dims(SD ${D}) + target_sources(lib_${SD} + PRIVATE + SemiImplicitEM.cpp + ThetaImplicitEM.cpp + WarpXImplicitOps.cpp + WarpXSolverVec.cpp + ) +endforeach() diff --git a/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H b/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H new file mode 100644 index 00000000000..88ad6a058fd --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/ImplicitSolver.H @@ -0,0 +1,145 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef Implicit_Solver_H_ +#define Implicit_Solver_H_ + +#include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" +#include "NonlinearSolvers/NonlinearSolverLibrary.H" + +#include +#include + +/** + * \brief Base class for implicit time solvers. The base functions are those + * needed by an implicit solver to be used through WarpX and those needed + * to use the nonlinear solvers, such as Picard or Newton (i.e., JFNK). + */ + +class WarpX; +class ImplicitSolver +{ +public: + + ImplicitSolver() = default; + + virtual ~ImplicitSolver() = default; + + // Prohibit Move and Copy operations + ImplicitSolver(const ImplicitSolver&) = delete; + ImplicitSolver& operator=(const ImplicitSolver&) = delete; + ImplicitSolver(ImplicitSolver&&) = delete; + ImplicitSolver& operator=(ImplicitSolver&&) = delete; + + // + // the following routines are called by WarpX + // + + /** + * \brief Read user-provided parameters that control the implicit solver. + * Allocate internal arrays for intermediate field values needed by the solver. + */ + virtual void Define ( WarpX* a_WarpX ) = 0; + + [[nodiscard]] bool IsDefined () const { return m_is_defined; } + + virtual void PrintParameters () const = 0; + + void GetParticleSolverParams (int& a_max_particle_iter, + amrex::ParticleReal& a_particle_tol ) const + { + a_max_particle_iter = m_max_particle_iterations; + a_particle_tol = m_particle_tolerance; + } + + /** + * \brief Advance fields and particles by one time step using the specified implicit algorithm + */ + virtual void OneStep ( amrex::Real a_time, + amrex::Real a_dt, + int a_step ) = 0; + + // + // the following routines are called by the linear and nonlinear solvers + // + + /** + * \brief Computes the RHS of the equation corresponding to the specified implicit algorithm. + * The discrete equations corresponding to numerical integration of ODEs are often + * written in the form U = b + RHS(U), where U is the variable to be solved for (e.g., + * the solution at the next time step), b is a constant (i.e., the solution from the + * previous time step), and RHS(U) is the right-hand-side of the equation. Iterative + * solvers, such as Picard and Newton, and higher-order Runge-Kutta methods, need to + * compute RHS(U) multiple times for different values of U. Thus, a routine that + * returns this value is needed. + * e.g., Ebar - E^n = cvac^2*0.5*dt*(curl(Bbar) - mu0*Jbar(Ebar,Bbar)) + * Here, U = Ebar, b = E^n, and the expression on the right is RHS(U). + */ + virtual void ComputeRHS ( WarpXSolverVec& a_RHS, + const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt, + int a_nl_iter, + bool a_from_jacobian ) = 0; + +protected: + + /** + * \brief Pointer back to main WarpX class + */ + WarpX* m_WarpX; + + bool m_is_defined = false; + + /** + * \brief Nonlinear solver type and object + */ + NonlinearSolverType m_nlsolver_type; + std::unique_ptr> m_nlsolver; + + /** + * \brief tolerance used by the iterative method used to obtain a self-consistent + * update of the particle positions and velocities for given E and B on the grid + */ + amrex::ParticleReal m_particle_tolerance = 1.0e-10; + + /** + * \brief maximum iterations for the iterative method used to obtain a self-consistent + * update of the particle positions and velocities for given E and B on the grid + */ + int m_max_particle_iterations = 21; + + /** + * \brief parse nonlinear solver parameters (if one is used) + */ + void parseNonlinearSolverParams( const amrex::ParmParse& pp ) + { + + std::string nlsolver_type_str; + pp.get("nonlinear_solver", nlsolver_type_str); + + if (nlsolver_type_str=="picard") { + m_nlsolver_type = NonlinearSolverType::Picard; + m_nlsolver = std::make_unique>(); + m_max_particle_iterations = 1; + m_particle_tolerance = 0.0; + } + else if (nlsolver_type_str=="newton") { + m_nlsolver_type = NonlinearSolverType::Newton; + m_nlsolver = std::make_unique>(); + pp.query("max_particle_iterations", m_max_particle_iterations); + pp.query("particle_tolerance", m_particle_tolerance); + } + else { + WARPX_ABORT_WITH_MESSAGE( + "invalid nonlinear_solver specified. Valid options are picard and newton."); + } + + } + +}; + +#endif diff --git a/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H b/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H new file mode 100644 index 00000000000..423957ef061 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H @@ -0,0 +1,13 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef IMPLICIT_SOLVER_LIBRARY_H_ +#define IMPLICIT_SOLVER_LIBRARY_H_ + +#include "SemiImplicitEM.H" // IWYU pragma: export +#include "ThetaImplicitEM.H" // IWYU pragma: export + +#endif diff --git a/Source/FieldSolver/ImplicitSolvers/Make.package b/Source/FieldSolver/ImplicitSolvers/Make.package new file mode 100644 index 00000000000..a4543f94dd3 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/Make.package @@ -0,0 +1,6 @@ +CEXE_sources += SemiImplicitEM.cpp +CEXE_sources += ThetaImplicitEM.cpp +CEXE_sources += WarpXImplicitOps.cpp +CEXE_sources += WarpXSolverVec.cpp + +VPATH_LOCATIONS += $(WARPX_HOME)/Source/FieldSolver/ImplicitSolvers diff --git a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H new file mode 100644 index 00000000000..6e3e5db2c74 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.H @@ -0,0 +1,80 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef SEMI_IMPLICIT_EM_H_ +#define SEMI_IMPLICIT_EM_H_ + +#include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" + +#include +#include +#include + +#include "ImplicitSolver.H" + +/** @file + * Semi-implicit electromagnetic time solver class. The electric field and the + * particles are implicitly coupled in this algorithm, but the magnetic field + * is advanced in the standard explicit leap-frog manner (hence semi-implicit). + * + * The time stencil is + * Eg^{n+1} = Eg^n + c^2*dt*( curlBg^{n+1/2} - mu0*Jg^{n+1/2} ) + * Bg^{n+3/2} = Bg^{n+1/2} - dt*curlEg^{n+1} + * xp^{n+1} = xp^n + dt*up^{n+1/2}/(0.5(gammap^n + gammap^{n+1})) + * up^{n+1} = up^n + dt*qp/mp*(Ep^{n+1/2} + up^{n+1/2}/gammap^{n+1/2} x Bp^{n+1/2}) + * where f^{n+1/2} = (f^{n+1} + f^n)/2.0, for all but Bg, which lives at half steps + * + * This algorithm is approximately energy conserving. It is exactly energy conserving + * using a non-standard definition for the magnetic field energy. The advantage of + * this method over the exactly energy-conserving theta-implicit EM method is that + * light wave dispersion is captured much better. However, the CFL condition for light + * waves has to be satisifed for numerical stability (and for the modified definition + * of the magnetic field energy to be well-posed). + * + * See G. Chen, L. Chacon, L. Yin, B.J. Albright, D.J. Stark, R.F. Bird, + * "A semi-implicit energy- and charge-conserving particle-in-cell algorithm for the + * relativistic Vlasov-Maxwell equations.", JCP 407 (2020). + */ + +class SemiImplicitEM : public ImplicitSolver +{ +public: + + SemiImplicitEM() = default; + + ~SemiImplicitEM() override = default; + + // Prohibit Move and Copy operations + SemiImplicitEM(const SemiImplicitEM&) = delete; + SemiImplicitEM& operator=(const SemiImplicitEM&) = delete; + SemiImplicitEM(SemiImplicitEM&&) = delete; + SemiImplicitEM& operator=(SemiImplicitEM&&) = delete; + + void Define ( WarpX* a_WarpX ) override; + + void PrintParameters () const override; + + void OneStep ( amrex::Real a_time, + amrex::Real a_dt, + int a_step ) override; + + void ComputeRHS ( WarpXSolverVec& a_RHS, + const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt, + int a_nl_iter, + bool a_from_jacobian ) override; + +private: + + /** + * \brief Solver vectors for E and Eold + */ + WarpXSolverVec m_E, m_Eold; + +}; + +#endif diff --git a/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp new file mode 100644 index 00000000000..897bd5c07f3 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/SemiImplicitEM.cpp @@ -0,0 +1,117 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "SemiImplicitEM.H" +#include "WarpX.H" + +using namespace warpx::fields; +using namespace amrex::literals; + +void SemiImplicitEM::Define ( WarpX* a_WarpX ) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !m_is_defined, + "SemiImplicitEM object is already defined!"); + + // Retain a pointer back to main WarpX class + m_WarpX = a_WarpX; + + // Define E and Eold vectors + m_E.Define( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); + m_Eold.Define( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); + + // Need to define the WarpXSolverVec owned dot_mask to do dot + // product correctly for linear and nonlinear solvers + const amrex::Vector& Geom = m_WarpX->Geom(); + m_E.SetDotMask(Geom); + + // Parse implicit solver parameters + const amrex::ParmParse pp("implicit_evolve"); + parseNonlinearSolverParams( pp ); + + // Define the nonlinear solver + m_nlsolver->Define(m_E, this); + m_is_defined = true; + +} + +void SemiImplicitEM::PrintParameters () const +{ + if (!m_WarpX->Verbose()) { return; } + amrex::Print() << std::endl; + amrex::Print() << "-----------------------------------------------------------" << std::endl; + amrex::Print() << "----------- SEMI IMPLICIT EM SOLVER PARAMETERS ------------" << std::endl; + amrex::Print() << "-----------------------------------------------------------" << std::endl; + amrex::Print() << "max particle iterations: " << m_max_particle_iterations << std::endl; + amrex::Print() << "particle tolerance: " << m_particle_tolerance << std::endl; + if (m_nlsolver_type==NonlinearSolverType::Picard) { + amrex::Print() << "Nonlinear solver type: Picard" << std::endl; + } + else if (m_nlsolver_type==NonlinearSolverType::Newton) { + amrex::Print() << "Nonlinear solver type: Newton" << std::endl; + } + m_nlsolver->PrintParams(); + amrex::Print() << "-----------------------------------------------------------" << std::endl; + amrex::Print() << std::endl; +} + +void SemiImplicitEM::OneStep ( amrex::Real a_time, + amrex::Real a_dt, + int a_step ) +{ + amrex::ignore_unused(a_step); + + // Fields have E^{n}, B^{n-1/2} + // Particles have p^{n} and x^{n}. + + // Save the values at the start of the time step, + m_WarpX->SaveParticlesAtImplicitStepStart ( ); + + // Save the fields at the start of the step + m_Eold.Copy( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); + m_E.Copy(m_Eold); // initial guess for E + + // Compute Bfield at time n+1/2 + m_WarpX->EvolveB(a_dt, DtType::Full); + m_WarpX->ApplyMagneticFieldBCs(); + + const amrex::Real half_time = a_time + 0.5_rt*a_dt; + + // Solve nonlinear system for E at t_{n+1/2} + // Particles will be advanced to t_{n+1/2} + m_nlsolver->Solve( m_E, m_Eold, half_time, a_dt ); + + // Update WarpX owned Efield_fp to t_{n+1/2} + m_WarpX->SetElectricFieldAndApplyBCs( m_E ); + + // Advance particles from time n+1/2 to time n+1 + m_WarpX->FinishImplicitParticleUpdate(); + + // Advance E fields from time n+1/2 to time n+1 + // Eg^{n+1} = 2.0*E_g^{n+1/2} - E_g^n + m_E.linComb( 2._rt, m_E, -1._rt, m_Eold ); + m_WarpX->SetElectricFieldAndApplyBCs( m_E ); + +} + +void SemiImplicitEM::ComputeRHS ( WarpXSolverVec& a_RHS, + const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt, + int a_nl_iter, + bool a_from_jacobian ) +{ + // update WarpX-owned Efield_fp using current state of E from + // the nonlinear solver at time n+theta + m_WarpX->SetElectricFieldAndApplyBCs( a_E ); + + // Self consistently update particle positions and velocities using the + // current state of the fields E and B. Deposit current density at time n+1/2. + m_WarpX->ImplicitPreRHSOp( a_time, a_dt, a_nl_iter, a_from_jacobian ); + + // RHS = cvac^2*0.5*dt*( curl(B^{n+1/2}) - mu0*J^{n+1/2} ) + m_WarpX->ImplicitComputeRHSE(0.5_rt*a_dt, a_RHS); +} diff --git a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H new file mode 100644 index 00000000000..009c2c7e546 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.H @@ -0,0 +1,124 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef THETA_IMPLICIT_EM_H_ +#define THETA_IMPLICIT_EM_H_ + +#include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" + +#include +#include +#include + +#include "ImplicitSolver.H" + +/** @file + * Theta-implicit electromagnetic time solver class. This is a fully implicit + * algorithm where both the fields and particles are treated implicitly. + * + * The time stencil is + * Eg^{n+1} = Eg^n + c^2*dt*( curlBg^{n+theta} - mu0*Jg^{n+1/2} ) + * Bg^{n+1} = Bg^n - dt*curlEg^{n+theta} + * xp^{n+1} = xp^n + dt*up^{n+1/2}/(0.5*(gammap^n + gammap^{n+1})) + * up^{n+1} = up^n + dt*qp/mp*(Ep^{n+theta} + up^{n+1/2}/gammap^{n+1/2} x Bp^{n+theta}) + * where f^{n+theta} = (1.0-theta)*f^{n} + theta*f^{n+1} with 0.5 <= theta <= 1.0 + * + * The user-specified time-biasing parameter theta used for the fields on the RHS is bound + * between 0.5 and 1.0. The algorithm is exactly energy conserving for theta = 0.5. + * Signifcant damping of high-k modes will occur as theta approaches 1.0. The algorithm is + * numerially stable for any time step. I.e., the CFL condition for light waves does not + * have to be satisifed and the time step is not limited by the plasma period. However, how + * efficiently the algorithm can use large time steps depends strongly on the nonlinear solver. + * Furthermore, the time step should always be such that particles do not travel outside the + * ghost region of the box they live in, which is an MPI-related limitation. The time step + * is always limited by the need to resolve the appropriate physics. + * + * See S. Markidis, G. Lapenta, "The energy conserving particle-in-cell method." JCP 230 (2011). + * + * See G. Chen, L. Chacon, D.C. Barnes, "An energy- and charge-conserving, implicit, + * elctrostatic particle-in-cell algorithm." JCP 230 (2011). + * + * See J.R. Angus, A. Link, A. Friedman, D. Ghosh, J. D. Johnson, "On numerical energy + * conservation for an implicit particle-in-cell method coupled with a binary Monte-Carlo + * algorithm for Coulomb collisions.", JCP 456 (2022). + * + * See J.R. Angus, W. Farmer, A. Friedman, D. Ghosh, D. Grote, D. Larson, A. Link, "An + * implicit particle code with exact energy and charge conservation for electromagnetic studies + * of dense plasmas.", JCP 491 (2023). + */ + +class ThetaImplicitEM : public ImplicitSolver +{ +public: + + ThetaImplicitEM() = default; + + ~ThetaImplicitEM() override = default; + + // Prohibit Move and Copy operations + ThetaImplicitEM(const ThetaImplicitEM&) = delete; + ThetaImplicitEM& operator=(const ThetaImplicitEM&) = delete; + ThetaImplicitEM(ThetaImplicitEM&&) = delete; + ThetaImplicitEM& operator=(ThetaImplicitEM&&) = delete; + + void Define ( WarpX* a_WarpX ) override; + + void PrintParameters () const override; + + void OneStep ( amrex::Real a_time, + amrex::Real a_dt, + int a_step ) override; + + void ComputeRHS ( WarpXSolverVec& a_RHS, + const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt, + int a_nl_iter, + bool a_from_jacobian ) override; + + [[nodiscard]] amrex::Real theta () const { return m_theta; } + +private: + + /** + * \brief Time-biasing parameter for fields used on RHS to advance system + */ + amrex::Real m_theta = 0.5; + + /** + * \brief Solver vectors to be used in the nonlinear solver to solve for the + * electric field E. The main logic for determining which variables should be + * WarpXSolverVec type is that it must have the same size and have the same + * centering of the data as the variable being solved for, which is E here. + * For example, if using a Yee grid then a container for curlB could be a + * WarpXSovlerVec, but magnetic field B should not be. + */ + WarpXSolverVec m_E, m_Eold; + + /** + * \brief B is a derived variable from E. Need to save Bold to update B during + * the iterative nonlinear solve for E. Bold is owned here, but only used by WarpX. + * It is not used directly by the nonlinear solver, nor is it the same size as the + * solver vector (size E), and so it should not be WarpXSolverVec type. + */ + amrex::Vector, 3 > > m_Bold; + + /** + * \brief Update the E and B fields owned by WarpX + */ + void UpdateWarpXFields ( const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt ); + + /** + * \brief Nonlinear solver is for the time-centered values of E. After + * the solver, need to use m_E and m_Eold to compute E^{n+1} + */ + void FinishFieldUpdate ( amrex::Real a_new_time ); + +}; + +#endif diff --git a/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp new file mode 100644 index 00000000000..026c509c3ba --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/ThetaImplicitEM.cpp @@ -0,0 +1,171 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "FieldSolver/Fields.H" +#include "ThetaImplicitEM.H" +#include "WarpX.H" + +using namespace warpx::fields; +using namespace amrex::literals; + +void ThetaImplicitEM::Define ( WarpX* const a_WarpX ) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !m_is_defined, + "ThetaImplicitEM object is already defined!"); + + // Retain a pointer back to main WarpX class + m_WarpX = a_WarpX; + + // Define E and Eold vectors + m_E.Define( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); + m_Eold.Define( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); + + // Need to define the WarpXSolverVec owned dot_mask to do dot + // product correctly for linear and nonlinear solvers + const amrex::Vector& Geom = m_WarpX->Geom(); + m_E.SetDotMask(Geom); + + // Define Bold MultiFab + const int num_levels = 1; + m_Bold.resize(num_levels); // size is number of levels + for (int lev = 0; lev < num_levels; ++lev) { + for (int n=0; n<3; n++) { + const amrex::MultiFab& Bfp = m_WarpX->getField( FieldType::Bfield_fp,lev,n); + m_Bold[lev][n] = std::make_unique( Bfp.boxArray(), + Bfp.DistributionMap(), + Bfp.nComp(), + Bfp.nGrowVect() ); + } + } + + // Parse theta-implicit solver specific parameters + const amrex::ParmParse pp("implicit_evolve"); + pp.query("theta", m_theta); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + m_theta>=0.5 && m_theta<=1.0, + "theta parameter for theta implicit time solver must be between 0.5 and 1.0"); + + // Parse nonlinear solver parameters + parseNonlinearSolverParams( pp ); + + // Define the nonlinear solver + m_nlsolver->Define(m_E, this); + m_is_defined = true; + +} + +void ThetaImplicitEM::PrintParameters () const +{ + if (!m_WarpX->Verbose()) { return; } + amrex::Print() << std::endl; + amrex::Print() << "-----------------------------------------------------------" << std::endl; + amrex::Print() << "----------- THETA IMPLICIT EM SOLVER PARAMETERS -----------" << std::endl; + amrex::Print() << "-----------------------------------------------------------" << std::endl; + amrex::Print() << "Time-bias parameter theta: " << m_theta << std::endl; + amrex::Print() << "max particle iterations: " << m_max_particle_iterations << std::endl; + amrex::Print() << "particle tolerance: " << m_particle_tolerance << std::endl; + if (m_nlsolver_type==NonlinearSolverType::Picard) { + amrex::Print() << "Nonlinear solver type: Picard" << std::endl; + } + else if (m_nlsolver_type==NonlinearSolverType::Newton) { + amrex::Print() << "Nonlinear solver type: Newton" << std::endl; + } + m_nlsolver->PrintParams(); + amrex::Print() << "-----------------------------------------------------------" << std::endl; + amrex::Print() << std::endl; +} + +void ThetaImplicitEM::OneStep ( const amrex::Real a_time, + const amrex::Real a_dt, + const int a_step ) +{ + amrex::ignore_unused(a_step); + + // Fields have E^{n} and B^{n} + // Particles have p^{n} and x^{n}. + + // Save the values at the start of the time step, + m_WarpX->SaveParticlesAtImplicitStepStart ( ); + + // Save the fields at the start of the step + m_Eold.Copy( m_WarpX->getMultiLevelField(FieldType::Efield_fp) ); + m_E.Copy(m_Eold); // initial guess for E + + const int num_levels = static_cast(m_Bold.size()); + for (int lev = 0; lev < num_levels; ++lev) { + for (int n=0; n<3; n++) { + const amrex::MultiFab& Bfp = m_WarpX->getField(FieldType::Bfield_fp,lev,n); + amrex::MultiFab& Bold = *m_Bold[lev][n]; + amrex::MultiFab::Copy(Bold, Bfp, 0, 0, 1, Bold.nGrowVect()); + } + } + + const amrex::Real theta_time = a_time + m_theta*a_dt; + + // Solve nonlinear system for E at t_{n+theta} + // Particles will be advanced to t_{n+1/2} + m_nlsolver->Solve( m_E, m_Eold, theta_time, a_dt ); + + // Update WarpX owned Efield_fp and Bfield_fp to t_{n+theta} + UpdateWarpXFields( m_E, theta_time, a_dt ); + + // Advance particles from time n+1/2 to time n+1 + m_WarpX->FinishImplicitParticleUpdate(); + + // Advance E and B fields from time n+theta to time n+1 + const amrex::Real new_time = a_time + a_dt; + FinishFieldUpdate( new_time ); + +} + +void ThetaImplicitEM::ComputeRHS ( WarpXSolverVec& a_RHS, + const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt, + int a_nl_iter, + bool a_from_jacobian ) +{ + // update WarpX-owned Efield_fp and Bfield_fp using current state of E from + // the nonlinear solver at time n+theta + UpdateWarpXFields( a_E, a_time, a_dt ); + + // Self consistently update particle positions and velocities using the + // current state of the fields E and B. Deposit current density at time n+1/2. + m_WarpX->ImplicitPreRHSOp( a_time, a_dt, a_nl_iter, a_from_jacobian ); + + // RHS = cvac^2*m_theta*dt*( curl(B^{n+theta}) - mu0*J^{n+1/2} ) + m_WarpX->ImplicitComputeRHSE(m_theta*a_dt, a_RHS); +} + +void ThetaImplicitEM::UpdateWarpXFields ( const WarpXSolverVec& a_E, + amrex::Real a_time, + amrex::Real a_dt ) +{ + amrex::ignore_unused(a_time); + + // Update Efield_fp owned by WarpX + m_WarpX->SetElectricFieldAndApplyBCs( a_E ); + + // Update Bfield_fp owned by WarpX + m_WarpX->UpdateMagneticFieldAndApplyBCs( m_Bold, m_theta*a_dt ); + +} + +void ThetaImplicitEM::FinishFieldUpdate ( amrex::Real a_new_time ) +{ + amrex::ignore_unused(a_new_time); + + // Eg^{n+1} = (1/theta)*E_g^{n+theta} + (1-1/theta)*E_g^n + // Bg^{n+1} = (1/theta)*B_g^{n+theta} + (1-1/theta)*B_g^n + + const amrex::Real c0 = 1._rt/m_theta; + const amrex::Real c1 = 1._rt - c0; + m_E.linComb( c0, m_E, c1, m_Eold ); + m_WarpX->SetElectricFieldAndApplyBCs( m_E ); + m_WarpX->FinishMagneticFieldAndApplyBCs( m_Bold, m_theta ); + +} diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp new file mode 100644 index 00000000000..7b95abade3e --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/WarpXImplicitOps.cpp @@ -0,0 +1,352 @@ +/* Copyright 2022 David Grote + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "WarpX.H" + +#include "BoundaryConditions/PML.H" +#include "Diagnostics/MultiDiagnostics.H" +#include "Diagnostics/ReducedDiags/MultiReducedDiags.H" +#include "Evolve/WarpXDtType.H" +#include "Evolve/WarpXPushType.H" +#include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" +#ifdef WARPX_USE_PSATD +# ifdef WARPX_DIM_RZ +# include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" +# else +# include "FieldSolver/SpectralSolver/SpectralSolver.H" +# endif +#endif +#include "Parallelization/GuardCellManager.H" +#include "Particles/MultiParticleContainer.H" +#include "Particles/ParticleBoundaryBuffer.H" +#include "Python/callbacks.H" +#include "Utils/TextMsg.H" +#include "Utils/WarpXAlgorithmSelection.H" +#include "Utils/WarpXUtil.H" +#include "Utils/WarpXConst.H" +#include "Utils/WarpXProfilerWrapper.H" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +void +WarpX::ImplicitPreRHSOp ( amrex::Real a_cur_time, + amrex::Real a_full_dt, + int a_nl_iter, + bool a_from_jacobian ) +{ + using namespace amrex::literals; + amrex::ignore_unused( a_full_dt, a_nl_iter, a_from_jacobian ); + + // Advance the particle positions by 1/2 dt, + // particle velocities by dt, then take average of old and new v, + // deposit currents, giving J at n+1/2 + // This uses Efield_fp and Bfield_fp, the field at n+1/2 from the previous iteration. + const bool skip_current = false; + const PushType push_type = PushType::Implicit; + PushParticlesandDeposit(a_cur_time, skip_current, push_type); + + SyncCurrentAndRho(); + +} + +void +WarpX::SetElectricFieldAndApplyBCs ( const WarpXSolverVec& a_E ) +{ + const amrex::Vector, 3 > >& Evec = a_E.getVec(); + amrex::MultiFab::Copy(*Efield_fp[0][0], *Evec[0][0], 0, 0, ncomps, Evec[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_fp[0][1], *Evec[0][1], 0, 0, ncomps, Evec[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Efield_fp[0][2], *Evec[0][2], 0, 0, ncomps, Evec[0][2]->nGrowVect()); + FillBoundaryE(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + ApplyEfieldBoundary(0, PatchType::fine); +} + +void +WarpX::UpdateMagneticFieldAndApplyBCs( const amrex::Vector, 3 > >& a_Bn, + amrex::Real a_thetadt ) +{ + amrex::MultiFab::Copy(*Bfield_fp[0][0], *a_Bn[0][0], 0, 0, ncomps, a_Bn[0][0]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_fp[0][1], *a_Bn[0][1], 0, 0, ncomps, a_Bn[0][1]->nGrowVect()); + amrex::MultiFab::Copy(*Bfield_fp[0][2], *a_Bn[0][2], 0, 0, ncomps, a_Bn[0][2]->nGrowVect()); + EvolveB(a_thetadt, DtType::Full); + ApplyMagneticFieldBCs(); +} + +void +WarpX::FinishMagneticFieldAndApplyBCs( const amrex::Vector, 3 > >& a_Bn, + amrex::Real a_theta ) +{ + FinishImplicitField(Bfield_fp, a_Bn, a_theta); + ApplyMagneticFieldBCs(); +} + +void +WarpX::ApplyMagneticFieldBCs() +{ + FillBoundaryB(guard_cells.ng_alloc_EB, WarpX::sync_nodal_points); + ApplyBfieldBoundary(0, PatchType::fine, DtType::Full); +} + +void +WarpX::SaveParticlesAtImplicitStepStart ( ) +{ + // The implicit advance routines require the particle velocity + // and position values at the beginning of the step to compute the + // time-centered position and velocity needed for the implicit stencil. + // Thus, we need to save this information. + + for (auto const& pc : *mypc) { + + for (int lev = 0; lev <= finest_level; ++lev) { +#ifdef AMREX_USE_OMP +#pragma omp parallel +#endif + { + + auto particle_comps = pc->getParticleComps(); + + for (WarpXParIter pti(*pc, lev); pti.isValid(); ++pti) { + + const auto getPosition = GetParticlePosition(pti); + + auto& attribs = pti.GetAttribs(); + amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + +#if (AMREX_SPACEDIM >= 2) + amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); +#endif + amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); + amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); + amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + + const long np = pti.numParticles(); + + amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) + { + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + +#if (AMREX_SPACEDIM >= 2) + x_n[ip] = xp; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + y_n[ip] = yp; +#endif + z_n[ip] = zp; + + ux_n[ip] = ux[ip]; + uy_n[ip] = uy[ip]; + uz_n[ip] = uz[ip]; + + }); + + } + } + + } + + } + +} + +void +WarpX::FinishImplicitParticleUpdate () +{ + using namespace amrex::literals; + + // The implicit advance routines use the time-centered position and + // momentum to advance the system in time. Thus, at the end of the + // step we need to transform the particle postion and momentum from + // time n+1/2 to time n+1. This is done here. + + for (auto const& pc : *mypc) { + + for (int lev = 0; lev <= finest_level; ++lev) { +#ifdef AMREX_USE_OMP +#pragma omp parallel +#endif + { + + auto particle_comps = pc->getParticleComps(); + + for (WarpXParIter pti(*pc, lev); pti.isValid(); ++pti) { + + const auto getPosition = GetParticlePosition(pti); + const auto setPosition = SetParticlePosition(pti); + + auto& attribs = pti.GetAttribs(); + amrex::ParticleReal* const AMREX_RESTRICT ux = attribs[PIdx::ux].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uy = attribs[PIdx::uy].dataPtr(); + amrex::ParticleReal* const AMREX_RESTRICT uz = attribs[PIdx::uz].dataPtr(); + +#if (AMREX_SPACEDIM >= 2) + amrex::ParticleReal* x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + amrex::ParticleReal* y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); +#endif + amrex::ParticleReal* z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + amrex::ParticleReal* ux_n = pti.GetAttribs(particle_comps["ux_n"]).dataPtr(); + amrex::ParticleReal* uy_n = pti.GetAttribs(particle_comps["uy_n"]).dataPtr(); + amrex::ParticleReal* uz_n = pti.GetAttribs(particle_comps["uz_n"]).dataPtr(); + + const long np = pti.numParticles(); + + amrex::ParallelFor( np, [=] AMREX_GPU_DEVICE (long ip) + { + amrex::ParticleReal xp, yp, zp; + getPosition(ip, xp, yp, zp); + +#if (AMREX_SPACEDIM >= 2) + xp = 2._rt*xp - x_n[ip]; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + yp = 2._rt*yp - y_n[ip]; +#endif + zp = 2._rt*zp - z_n[ip]; + + ux[ip] = 2._rt*ux[ip] - ux_n[ip]; + uy[ip] = 2._rt*uy[ip] - uy_n[ip]; + uz[ip] = 2._rt*uz[ip] - uz_n[ip]; + + setPosition(ip, xp, yp, zp); + }); + + } + } + + } + + } + +} + +void +WarpX::FinishImplicitField( amrex::Vector, 3 > >& Field_fp, + const amrex::Vector, 3 > >& Field_n, + amrex::Real theta ) +{ + using namespace amrex::literals; + + // The implicit field advance routines use the fields at time n+theta + // with 0.5 <= theta <= 1.0. Thus, at the end of the step we need to + // transform the fields from time n+theta to time n+1. This is done here. + + for (int lev = 0; lev <= finest_level; ++lev) { + +#ifdef AMREX_USE_OMP +#pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) +#endif + for ( amrex::MFIter mfi(*Field_fp[lev][0], amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi ) + { + + amrex::Array4 const& Fx = Field_fp[lev][0]->array(mfi); + amrex::Array4 const& Fy = Field_fp[lev][1]->array(mfi); + amrex::Array4 const& Fz = Field_fp[lev][2]->array(mfi); + + amrex::Array4 const& Fx_n = Field_n[lev][0]->array(mfi); + amrex::Array4 const& Fy_n = Field_n[lev][1]->array(mfi); + amrex::Array4 const& Fz_n = Field_n[lev][2]->array(mfi); + + amrex::Box const& tbx = mfi.tilebox(Field_n[lev][0]->ixType().toIntVect()); + amrex::Box const& tby = mfi.tilebox(Field_n[lev][1]->ixType().toIntVect()); + amrex::Box const& tbz = mfi.tilebox(Field_n[lev][2]->ixType().toIntVect()); + + const amrex::Real c0 = 1._rt/theta; + const amrex::Real c1 = 1._rt - c0; + + amrex::ParallelFor( + tbx, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fx(i,j,k,n) = c0*Fx(i,j,k,n) + c1*Fx_n(i,j,k,n); + }, + tby, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fy(i,j,k,n) = c0*Fy(i,j,k,n) + c1*Fy_n(i,j,k,n); + }, + tbz, ncomps, [=] AMREX_GPU_DEVICE (int i, int j, int k, int n) + { + Fz(i,j,k,n) = c0*Fz(i,j,k,n) + c1*Fz_n(i,j,k,n); + }); + } + } +} + +void +WarpX::ImplicitComputeRHSE (amrex::Real a_dt, WarpXSolverVec& a_Erhs_vec) +{ + for (int lev = 0; lev <= finest_level; ++lev) + { + ImplicitComputeRHSE(lev, a_dt, a_Erhs_vec); + } +} + +void +WarpX::ImplicitComputeRHSE (int lev, amrex::Real a_dt, WarpXSolverVec& a_Erhs_vec) +{ + WARPX_PROFILE("WarpX::ImplicitComputeRHSE()"); + ImplicitComputeRHSE(lev, PatchType::fine, a_dt, a_Erhs_vec); + if (lev > 0) + { + ImplicitComputeRHSE(lev, PatchType::coarse, a_dt, a_Erhs_vec); + } +} + +void +WarpX::ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real a_dt, WarpXSolverVec& a_Erhs_vec) +{ + // set RHS to zero value + a_Erhs_vec.getVec()[lev][0]->setVal(0.0); + a_Erhs_vec.getVec()[lev][1]->setVal(0.0); + a_Erhs_vec.getVec()[lev][2]->setVal(0.0); + + // Compute Efield_rhs in regular cells by calling EvolveE. Because + // a_Erhs_vec is set to zero above, calling EvolveE below results in + // a_Erhs_vec storing only the RHS of the update equation. I.e., + // c^2*dt*(curl(B^{n+theta} - mu0*J^{n+1/2}) + if (patch_type == PatchType::fine) { + m_fdtd_solver_fp[lev]->EvolveE( a_Erhs_vec.getVec()[lev], Bfield_fp[lev], + current_fp[lev], m_edge_lengths[lev], + m_face_areas[lev], ECTRhofield[lev], + F_fp[lev], lev, a_dt ); + } else { + m_fdtd_solver_cp[lev]->EvolveE( a_Erhs_vec.getVec()[lev], Bfield_cp[lev], + current_cp[lev], m_edge_lengths[lev], + m_face_areas[lev], ECTRhofield[lev], + F_cp[lev], lev, a_dt ); + } + + // Compute Efield_rhs in PML cells by calling EvolveEPML + if (do_pml && pml[lev]->ok()) { + amrex::Abort("PML not yet implemented with implicit solvers."); + } + +} diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H new file mode 100644 index 00000000000..89a0b82b700 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.H @@ -0,0 +1,233 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WarpXSolverVec_H_ +#define WarpXSolverVec_H_ + +#include "Utils/TextMsg.H" + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +/** + * \brief This is a wrapper class around a Vector of array of pointers to MultiFabs that + * contains basic math operators and functionality needed to interact with nonlinear + * solvers in WarpX and linear solvers in AMReX, such as GMRES. + * + * The size of the Vector is the number of amr levels. Hardcoded for 1 right now. The + * size of the array is the number of MultiFabs. It is hardcoded for 3 right now as it + * is only used for the electric field in the implicit electromagnetic time solvers. In + * the future, the array size can be made a template parameter so that this class can + * be used for other solver vectors, such as electrostatic (array size 1) or Darwin (array size 4). + */ + +class WarpXSolverVec +{ +public: + + WarpXSolverVec() = default; + + WarpXSolverVec(const WarpXSolverVec&) = delete; + + ~WarpXSolverVec() = default; + + using value_type = amrex::Real; + using RT = value_type; + + [[nodiscard]] inline bool IsDefined () const { return m_is_defined; } + + inline + void Define (const WarpXSolverVec& a_vec) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + a_vec.IsDefined(), + "WarpXSolverVec::Define(a_vec) called with undefined a_vec"); + Define( a_vec.getVec() ); + } + + inline + void Define ( const amrex::Vector, 3 > >& a_solver_vec ) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !IsDefined(), + "WarpXSolverVec::Define() called on undefined WarpXSolverVec"); + m_field_vec.resize(m_num_amr_levels); + const int lev = 0; + for (int n=0; n<3; n++) { + const amrex::MultiFab& mf_model = *a_solver_vec[lev][n]; + m_field_vec[lev][n] = std::make_unique( mf_model.boxArray(), mf_model.DistributionMap(), + mf_model.nComp(), amrex::IntVect::TheZeroVector() ); + } + m_is_defined = true; + } + + void SetDotMask( const amrex::Vector& a_Geom ); + [[nodiscard]] RT dotProduct( const WarpXSolverVec& a_X ) const; + + inline + void Copy ( const amrex::Vector, 3 > >& a_solver_vec ) + { + AMREX_ASSERT_WITH_MESSAGE( + IsDefined(), + "WarpXSolverVec::Copy() called on undefined WarpXSolverVec"); + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n = 0; n < 3; ++n) { + amrex::MultiFab::Copy(*m_field_vec[lev][n], *a_solver_vec[lev][n], 0, 0, m_ncomp, amrex::IntVect::TheZeroVector() ); + } + } + } + + inline + void Copy ( const WarpXSolverVec& a_vec ) + { + AMREX_ASSERT_WITH_MESSAGE( + a_vec.IsDefined(), + "WarpXSolverVec::Copy(a_vec) called with undefined a_vec"); + if (!IsDefined()) { Define(a_vec); } + const amrex::Vector, 3 > >& field_vec = a_vec.getVec(); + Copy(field_vec); + } + + // Prohibit Copy assignment operator + WarpXSolverVec& operator= ( const WarpXSolverVec& a_vec ) = delete; + + // Move assignment operator + WarpXSolverVec(WarpXSolverVec&&) noexcept = default; + WarpXSolverVec& operator= ( WarpXSolverVec&& a_vec ) noexcept + { + if (this != &a_vec) { + m_field_vec = std::move(a_vec.m_field_vec); + m_is_defined = true; + } + return *this; + } + + inline + void operator+= ( const WarpXSolverVec& a_vec ) + { + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n=0; n<3; n++) { + m_field_vec[lev][n]->plus(*(a_vec.getVec()[lev][n]), 0, 1, 0); + } + } + } + + inline + void operator-= (const WarpXSolverVec& a_vec) + { + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n=0; n<3; n++) { + m_field_vec[lev][n]->minus(*(a_vec.getVec()[lev][n]), 0, 1, 0); + } + } + } + + /** + * \brief Y = a*X + b*Y + */ + inline + void linComb (const RT a, const WarpXSolverVec& X, const RT b, const WarpXSolverVec& Y) + { + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n=0; n<3; n++) { + amrex::MultiFab::LinComb(*m_field_vec[lev][n], a, *X.getVec()[lev][n], 0, + b, *Y.getVec()[lev][n], 0, + 0, 1, 0); + } + } + } + + /** + * \brief Increment Y by a*X (Y += a*X) + */ + void increment (const WarpXSolverVec& X, const RT a) + { + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n=0; n<3; n++) { + amrex::MultiFab::Saxpy( *m_field_vec[lev][n], a, *X.getVec()[lev][n], + 0, 0, 1, amrex::IntVect::TheZeroVector() ); + } + } + } + + /** + * \brief Scale Y by a (Y *= a) + */ + inline + void scale (RT a_a) + { + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n=0; n<3; n++) { + m_field_vec[lev][n]->mult(a_a, 0, 1); + } + } + } + + inline + void zero () { setVal(0.0); } + + inline + void setVal ( const RT a_val ) + { + AMREX_ASSERT_WITH_MESSAGE( + IsDefined(), + "WarpXSolverVec::ones() called on undefined WarpXSolverVec"); + for (int lev = 0; lev < m_num_amr_levels; ++lev) { + for (int n=0; n<3; n++) { + m_field_vec[lev][n]->setVal(a_val); + } + } + } + + [[nodiscard]] inline RT norm2 () const + { + auto const norm = dotProduct(*this); + return std::sqrt(norm); + } + + [[nodiscard]] const amrex::Vector, 3 > >& getVec() const {return m_field_vec;} + amrex::Vector, 3 > >& getVec() {return m_field_vec;} + + // clearDotMask() must be called by the highest class that owns WarpXSolverVec() + // after it is done being used ( typically in the destructor ) to avoid the + // following error message after the simulation finishes: + // malloc_consolidate(): unaligned fastbin chunk detected + static void clearDotMask() { m_dotMask.clear(); } + +private: + + bool m_is_defined = false; + amrex::Vector, 3 > > m_field_vec; + + static constexpr int m_ncomp = 1; + static constexpr int m_num_amr_levels = 1; + + inline static bool m_dot_mask_defined = false; + inline static amrex::Vector,3>> m_dotMask; + +}; + +#endif diff --git a/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp new file mode 100644 index 00000000000..b181f038fb5 --- /dev/null +++ b/Source/FieldSolver/ImplicitSolvers/WarpXSolverVec.cpp @@ -0,0 +1,51 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" + +void WarpXSolverVec::SetDotMask( const amrex::Vector& a_Geom ) +{ + if (m_dot_mask_defined) { return; } + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + IsDefined(), + "WarpXSolverVec::SetDotMask() called from undefined instance "); + + m_dotMask.resize(m_num_amr_levels); + for ( int n = 0; n < 3; n++) { + const amrex::BoxArray& grids = m_field_vec[0][n]->boxArray(); + const amrex::MultiFab tmp( grids, m_field_vec[0][n]->DistributionMap(), + 1, 0, amrex::MFInfo().SetAlloc(false) ); + const amrex::Periodicity& period = a_Geom[0].periodicity(); + m_dotMask[0][n] = tmp.OwnerMask(period); + } + m_dot_mask_defined = true; + + // If the function below is not called, then the following + // error message occurs after the simulation finishes: + // malloc_consolidate(): unaligned fastbin chunk detected + amrex::ExecOnFinalize(WarpXSolverVec::clearDotMask); +} + +[[nodiscard]] amrex::Real WarpXSolverVec::dotProduct ( const WarpXSolverVec& a_X ) const +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + m_dot_mask_defined, + "WarpXSolverVec::dotProduct called with m_dotMask not yet defined"); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + a_X.IsDefined(), + "WarpXSolverVec::dotProduct(a_X) called with undefined a_X"); + amrex::Real result = 0.0; + const int lev = 0; + const bool local = true; + for (int n = 0; n < 3; ++n) { + auto rtmp = amrex::MultiFab::Dot( *m_dotMask[lev][n], + *m_field_vec[lev][n], 0, + *a_X.getVec()[lev][n], 0, 1, 0, local); + result += rtmp; + } + amrex::ParallelAllReduce::Sum(result, amrex::ParallelContext::CommunicatorSub()); + return result; +} diff --git a/Source/FieldSolver/Make.package b/Source/FieldSolver/Make.package index c1535ae2f22..a8af4c2de97 100644 --- a/Source/FieldSolver/Make.package +++ b/Source/FieldSolver/Make.package @@ -2,10 +2,11 @@ CEXE_sources += WarpXPushFieldsEM.cpp CEXE_sources += WarpXPushFieldsHybridPIC.cpp CEXE_sources += ElectrostaticSolver.cpp CEXE_sources += WarpX_QED_Field_Pushers.cpp -ifeq ($(USE_PSATD),TRUE) +ifeq ($(USE_FFT),TRUE) include $(WARPX_HOME)/Source/FieldSolver/SpectralSolver/Make.package endif include $(WARPX_HOME)/Source/FieldSolver/FiniteDifferenceSolver/Make.package include $(WARPX_HOME)/Source/FieldSolver/MagnetostaticSolver/Make.package +include $(WARPX_HOME)/Source/FieldSolver/ImplicitSolvers/Make.package VPATH_LOCATIONS += $(WARPX_HOME)/Source/FieldSolver diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H index 87dfe6f2d9a..b7ee256f414 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.H @@ -5,6 +5,8 @@ #include "FieldSolver/SpectralSolver/SpectralKSpace.H" #include "SpectralBaseAlgorithm.H" +#include + #include #include #include @@ -14,7 +16,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT /* \brief Class that updates the field in spectral space and stores the coefficients * of the corresponding update equation, according to the comoving spectral scheme. @@ -32,7 +34,7 @@ class PsatdAlgorithmComoving : public SpectralBaseAlgorithm int norder_x, int norder_y, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_comoving, amrex::Real dt, bool update_with_rho); @@ -86,5 +88,5 @@ class PsatdAlgorithmComoving : public SpectralBaseAlgorithm amrex::Real m_dt; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_PSATD_ALGORITHM_COMOVING_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp index 640cf1b82af..f809c3f2c20 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp @@ -5,6 +5,8 @@ #include "Utils/WarpXConst.H" #include "Utils/WarpX_Complex.H" +#include + #include #include #include @@ -19,15 +21,17 @@ #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT using namespace amrex; PsatdAlgorithmComoving::PsatdAlgorithmComoving (const SpectralKSpace& spectral_kspace, const DistributionMapping& dm, const SpectralFieldIndex& spectral_index, - const int norder_x, const int norder_y, - const int norder_z, const short grid_type, + const int norder_x, + const int norder_y, + const int norder_z, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_comoving, const amrex::Real dt, const bool update_with_rho) @@ -35,12 +39,12 @@ PsatdAlgorithmComoving::PsatdAlgorithmComoving (const SpectralKSpace& spectral_k : SpectralBaseAlgorithm{spectral_kspace, dm, spectral_index, norder_x, norder_y, norder_z, grid_type}, // Initialize the infinite-order k vectors (the argument n_order = -1 selects // the infinite order option, the argument grid_type=GridType::Staggered is then irrelevant) - kx_vec{spectral_kspace.getModifiedKComponent(dm, 0, -1, GridType::Staggered)}, + kx_vec{spectral_kspace.getModifiedKComponent(dm, 0, -1, ablastr::utils::enums::GridType::Staggered)}, #if defined(WARPX_DIM_3D) ky_vec{spectral_kspace.getModifiedKComponent(dm, 1, -1, GridType::Staggered)}, kz_vec{spectral_kspace.getModifiedKComponent(dm, 2, -1, GridType::Staggered)}, #else - kz_vec{spectral_kspace.getModifiedKComponent(dm, 1, -1, GridType::Staggered)}, + kz_vec{spectral_kspace.getModifiedKComponent(dm, 1, -1, ablastr::utils::enums::GridType::Staggered)}, #endif m_v_comoving{v_comoving}, m_dt{dt} @@ -505,4 +509,4 @@ PsatdAlgorithmComoving::VayDeposition (SpectralFieldData& /*field_data*/) "Vay deposition not implemented for comoving PSATD"); } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H index 8fde50d8172..23c38f33a85 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.H @@ -11,6 +11,8 @@ #include "FieldSolver/SpectralSolver/SpectralKSpace.H" #include "SpectralBaseAlgorithm.H" +#include + #include #include #include @@ -20,7 +22,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT /* * \brief Class that updates the fields in spectral space according to the first-order PSATD equations. */ @@ -50,7 +52,7 @@ class PsatdAlgorithmFirstOrder : public SpectralBaseAlgorithm int norder_x, int norder_y, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, amrex::Real dt, bool div_cleaning, int J_in_time, @@ -94,5 +96,5 @@ class PsatdAlgorithmFirstOrder : public SpectralBaseAlgorithm int m_J_in_time; int m_rho_in_time; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_PSATD_ALGORITHM_FIRST_ORDER_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp index 6d4d8613940..14db3c9af48 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmFirstOrder.cpp @@ -24,22 +24,23 @@ #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT using namespace amrex::literals; -PsatdAlgorithmFirstOrder::PsatdAlgorithmFirstOrder( +PsatdAlgorithmFirstOrder::PsatdAlgorithmFirstOrder ( const SpectralKSpace& spectral_kspace, const amrex::DistributionMapping& dm, const SpectralFieldIndex& spectral_index, const int norder_x, const int norder_y, const int norder_z, - const short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Real dt, const bool div_cleaning, const int J_in_time, - const int rho_in_time) + const int rho_in_time +) // Initializer list : SpectralBaseAlgorithm(spectral_kspace, dm, spectral_index, norder_x, norder_y, norder_z, grid_type), m_dt(dt), @@ -371,4 +372,4 @@ PsatdAlgorithmFirstOrder::VayDeposition (SpectralFieldData& field_data) "Vay deposition not implemented for first-order PSATD equations"); } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.H index ad0aca1ecf5..0861eba3abf 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.H @@ -9,6 +9,9 @@ #include "SpectralBaseAlgorithmRZ.H" +#include + + /* \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. */ @@ -20,7 +23,7 @@ class PsatdAlgorithmGalileanRZ : public SpectralBaseAlgorithmRZ amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, int n_rz_azimuthal_modes, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, amrex::Real dt_step, bool update_with_rho); diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.cpp index c24648fbf07..63127969149 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmGalileanRZ.cpp @@ -20,7 +20,7 @@ PsatdAlgorithmGalileanRZ::PsatdAlgorithmGalileanRZ (SpectralKSpaceRZ const & spe amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, int const n_rz_azimuthal_modes, int const norder_z, - short const grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, amrex::Real const dt, bool const update_with_rho): diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H index 94f00d6d4eb..12e3203db39 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.H @@ -11,6 +11,8 @@ #include "FieldSolver/SpectralSolver/SpectralKSpace.H" #include "SpectralBaseAlgorithm.H" +#include + #include #include #include @@ -20,7 +22,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT /* \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. */ @@ -52,7 +54,7 @@ class PsatdAlgorithmJConstantInTime : public SpectralBaseAlgorithm int norder_x, int norder_y, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, amrex::Real dt, bool update_with_rho, @@ -139,5 +141,5 @@ class PsatdAlgorithmJConstantInTime : public SpectralBaseAlgorithm bool m_divb_cleaning; bool m_is_galilean; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_PSATD_ALGORITHM_J_CONSTANT_IN_TIME_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp index 167d7565bc0..991bf981924 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp @@ -25,7 +25,7 @@ #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT using namespace amrex; @@ -36,7 +36,7 @@ PsatdAlgorithmJConstantInTime::PsatdAlgorithmJConstantInTime( const int norder_x, const int norder_y, const int norder_z, - const short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, const amrex::Real dt, const bool update_with_rho, @@ -861,4 +861,4 @@ PsatdAlgorithmJConstantInTime::VayDeposition (SpectralFieldData& field_data) } } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H index aa3bcc08e77..c26fc4a81a4 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.H @@ -11,6 +11,8 @@ #include "FieldSolver/SpectralSolver/SpectralKSpace.H" #include "SpectralBaseAlgorithm.H" +#include + #include #include #include @@ -20,7 +22,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT /* * \brief Class that updates the fields in spectral space according to the multi-J algorithm * and stores the coefficients of the corresponding update equations. J is assumed to be @@ -53,11 +55,12 @@ class PsatdAlgorithmJLinearInTime : public SpectralBaseAlgorithm int norder_x, int norder_y, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, amrex::Real dt, bool time_averaging, bool dive_cleaning, - bool divb_cleaning); + bool divb_cleaning + ); /** * \brief Updates the E and B fields in spectral space, according to the multi-J PSATD equations @@ -125,5 +128,5 @@ class PsatdAlgorithmJLinearInTime : public SpectralBaseAlgorithm bool m_dive_cleaning; bool m_divb_cleaning; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_PSATD_ALGORITHM_J_LINEAR_IN_TIME_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp index 92e35c8c03b..2626dd0376e 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJLinearInTime.cpp @@ -24,7 +24,7 @@ #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT using namespace amrex::literals; @@ -35,7 +35,7 @@ PsatdAlgorithmJLinearInTime::PsatdAlgorithmJLinearInTime( const int norder_x, const int norder_y, const int norder_z, - const short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Real dt, const bool time_averaging, const bool dive_cleaning, @@ -444,4 +444,4 @@ PsatdAlgorithmJLinearInTime::VayDeposition (SpectralFieldData& field_data) "Vay deposition not implemented for multi-J PSATD algorithm"); } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H index 1c25ca91903..f01b8cad379 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.H @@ -12,14 +12,15 @@ #include "FieldSolver/SpectralSolver/SpectralFieldData_fwd.H" #include "FieldSolver/SpectralSolver/SpectralKSpace_fwd.H" -#include +#include +#include #include #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT /* * \brief Class that updates the field in spectral space @@ -51,7 +52,7 @@ class PsatdAlgorithmPml : public SpectralBaseAlgorithm int norder_x, int norder_y, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, amrex::Real dt, bool dive_cleaning, @@ -115,5 +116,5 @@ class PsatdAlgorithmPml : public SpectralBaseAlgorithm bool m_is_galilean; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_PSATD_ALGORITHM_PML_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp index 792d5126ef7..5907485a526 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPml.cpp @@ -26,7 +26,7 @@ #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT using namespace amrex; @@ -37,7 +37,7 @@ PsatdAlgorithmPml::PsatdAlgorithmPml( int norder_x, int norder_y, int norder_z, - short grid_type, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, Real dt, bool dive_cleaning, @@ -466,4 +466,4 @@ void PsatdAlgorithmPml::VayDeposition (SpectralFieldData& /*field_data*/) WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented for PML PSATD"); } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.H index 00264956900..9f2a21db595 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.H @@ -9,6 +9,9 @@ #include "SpectralBaseAlgorithmRZ.H" +#include + + /* \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. */ @@ -19,8 +22,10 @@ class PsatdAlgorithmPmlRZ : public SpectralBaseAlgorithmRZ PsatdAlgorithmPmlRZ (SpectralKSpaceRZ const & spectral_kspace, amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, - int n_rz_azimuthal_modes, int norder_z, - short grid_type, amrex::Real dt_step); + int n_rz_azimuthal_modes, + int norder_z, + ablastr::utils::enums::GridType grid_type, + amrex::Real dt_step); // Redefine functions from base class void pushSpectralFields (SpectralFieldDataRZ & f) final; diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.cpp index 66aa463503f..50522f3c9b2 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmPmlRZ.cpp @@ -19,8 +19,10 @@ using namespace amrex::literals; PsatdAlgorithmPmlRZ::PsatdAlgorithmPmlRZ (SpectralKSpaceRZ const & spectral_kspace, amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, - int const n_rz_azimuthal_modes, int const norder_z, - short const grid_type, amrex::Real const dt): + int const n_rz_azimuthal_modes, + int const norder_z, + ablastr::utils::enums::GridType grid_type, + amrex::Real const dt): // Initialize members of base class and member variables SpectralBaseAlgorithmRZ{spectral_kspace, dm, spectral_index, norder_z, grid_type}, m_dt{dt} diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H index 10feafec38b..c2efbd79935 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.H @@ -9,6 +9,9 @@ #include "SpectralBaseAlgorithmRZ.H" +#include + + /* \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. */ @@ -20,7 +23,8 @@ class PsatdAlgorithmRZ : public SpectralBaseAlgorithmRZ amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, int n_rz_azimuthal_modes, int norder_z, - short grid_type, amrex::Real dt_step, + ablastr::utils::enums::GridType grid_type, + amrex::Real dt_step, bool update_with_rho, bool time_averaging, int J_in_time, diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp index f621d3c7af9..efff5c30a41 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmRZ.cpp @@ -19,8 +19,10 @@ using namespace amrex::literals; PsatdAlgorithmRZ::PsatdAlgorithmRZ (SpectralKSpaceRZ const & spectral_kspace, amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, - int const n_rz_azimuthal_modes, int const norder_z, - short const grid_type, amrex::Real const dt, + int const n_rz_azimuthal_modes, + int const norder_z, + ablastr::utils::enums::GridType grid_type, + amrex::Real const dt, bool const update_with_rho, const bool time_averaging, const int J_in_time, diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H index 4af24cfa559..c72e7db250d 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.H @@ -13,6 +13,8 @@ #include "FieldSolver/SpectralSolver/SpectralFieldData_fwd.H" #include "FieldSolver/SpectralSolver/SpectralFieldData.H" +#include + #include #include #include @@ -23,7 +25,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT /* \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. @@ -88,8 +90,10 @@ class SpectralBaseAlgorithm SpectralBaseAlgorithm(const SpectralKSpace& spectral_kspace, const amrex::DistributionMapping& dm, const SpectralFieldIndex& spectral_index, - int norder_x, int norder_y, - int norder_z, short grid_type); + int norder_x, + int norder_y, + int norder_z, + ablastr::utils::enums::GridType grid_type); SpectralFieldIndex m_spectral_index; @@ -101,5 +105,5 @@ class SpectralBaseAlgorithm KVectorComponent modified_kz_vec; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_SPECTRAL_BASE_ALGORITHM_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp index d635a5debe3..b3f18dd6912 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithm.cpp @@ -27,11 +27,15 @@ using namespace amrex; /** * \brief Constructor */ -SpectralBaseAlgorithm::SpectralBaseAlgorithm(const SpectralKSpace& spectral_kspace, +SpectralBaseAlgorithm::SpectralBaseAlgorithm ( + const SpectralKSpace& spectral_kspace, const amrex::DistributionMapping& dm, const SpectralFieldIndex& spectral_index, - const int norder_x, const int norder_y, - const int norder_z, const short grid_type): + const int norder_x, + const int norder_y, + const int norder_z, + ablastr::utils::enums::GridType grid_type +): m_spectral_index(spectral_index), // Compute and assign the modified k vectors modified_kx_vec(spectral_kspace.getModifiedKComponent(dm,0,norder_x,grid_type)), diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H index 95c68f6d61b..8e03a2a2559 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/SpectralBaseAlgorithmRZ.H @@ -10,6 +10,9 @@ #include "FieldSolver/SpectralSolver/SpectralKSpaceRZ.H" #include "FieldSolver/SpectralSolver/SpectralFieldDataRZ.H" +#include + + /* \brief Class that updates the field in spectral space * and stores the coefficients of the corresponding update equation. * @@ -84,7 +87,8 @@ class SpectralBaseAlgorithmRZ SpectralBaseAlgorithmRZ(SpectralKSpaceRZ const & spectral_kspace, amrex::DistributionMapping const & dm, const SpectralFieldIndex& spectral_index, - int const norder_z, short const grid_type) + int const norder_z, + ablastr::utils::enums::GridType grid_type) // Compute and assign the modified k vectors : m_spectral_index(spectral_index), modified_kz_vec(spectral_kspace.getModifiedKComponent(dm, 1, norder_z, grid_type)) diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp b/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp index 35d37df6038..20c97f4b5d4 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldData.cpp @@ -29,7 +29,7 @@ #include #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT using namespace amrex; @@ -475,4 +475,4 @@ SpectralFieldData::BackwardTransform (const int lev, } } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp index b26248fb2d0..d295d61b43a 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralFieldDataRZ.cpp @@ -199,8 +199,8 @@ SpectralFieldDataRZ::SpectralFieldDataRZ (const int lev, #endif // Create the Hankel transformer for each box. - std::array xmax = WarpX::UpperCorner(mfi.tilebox(), lev, 0._rt); - multi_spectral_hankel_transformer[mfi] = SpectralHankelTransformer(grid_size[0], n_rz_azimuthal_modes, xmax[0]); + amrex::XDim3 const xyzmax = WarpX::UpperCorner(mfi.tilebox(), lev, 0._rt); + multi_spectral_hankel_transformer[mfi] = SpectralHankelTransformer(grid_size[0], n_rz_azimuthal_modes, xyzmax.x); } } diff --git a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp index 9f4cac71736..7fd419b54f8 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralHankelTransform/HankelTransform.cpp @@ -36,7 +36,8 @@ HankelTransform::HankelTransform (int const hankel_order, // SYCL note: we need to double check AMReX device ID conventions and // BLAS++ device ID conventions are the same int const device_id = amrex::Gpu::Device::deviceId(); - m_queue = std::make_unique( device_id, 0 ); + blas::Queue::stream_t stream_id = amrex::Gpu::gpuStream(); + m_queue = std::make_unique( device_id, stream_id ); #endif amrex::Vector alphas; diff --git a/Source/FieldSolver/SpectralSolver/SpectralKSpace.H b/Source/FieldSolver/SpectralSolver/SpectralKSpace.H index 2a4dd3d1580..16f93d8292a 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralKSpace.H +++ b/Source/FieldSolver/SpectralSolver/SpectralKSpace.H @@ -12,6 +12,8 @@ #include "Utils/WarpX_Complex.H" +#include + #include #include #include @@ -59,9 +61,11 @@ class SpectralKSpace const amrex::BoxArray& realspace_ba, int i_dim, bool only_positive_k ) const; - KVectorComponent getModifiedKComponent( - const amrex::DistributionMapping& dm, int i_dim, - int n_order, short grid_type ) const; + KVectorComponent getModifiedKComponent ( + const amrex::DistributionMapping& dm, + int i_dim, + int n_order, + ablastr::utils::enums::GridType grid_type) const; SpectralShiftFactor getSpectralShiftFactor( const amrex::DistributionMapping& dm, int i_dim, diff --git a/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp b/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp index 91fe2953668..94bd384f265 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralKSpace.cpp @@ -188,10 +188,10 @@ SpectralKSpace::getSpectralShiftFactor( const DistributionMapping& dm, * \param grid_type type of grid (collocated or not) */ KVectorComponent -SpectralKSpace::getModifiedKComponent( const DistributionMapping& dm, +SpectralKSpace::getModifiedKComponent (const DistributionMapping& dm, const int i_dim, const int n_order, - const short grid_type ) const + ablastr::utils::enums::GridType grid_type) const { // Initialize an empty DeviceVector in each box KVectorComponent modified_k_comp(spectralspace_ba, dm); diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolver.H b/Source/FieldSolver/SpectralSolver/SpectralSolver.H index 9c74cdbea5a..26581900c02 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolver.H +++ b/Source/FieldSolver/SpectralSolver/SpectralSolver.H @@ -12,6 +12,8 @@ #include "SpectralAlgorithms/SpectralBaseAlgorithm.H" #include "SpectralFieldData.H" +#include + #include #include #include @@ -21,7 +23,7 @@ #include #include -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT /** * \brief Top-level class for the electromagnetic spectral solver * @@ -72,8 +74,10 @@ class SpectralSolver SpectralSolver (int lev, const amrex::BoxArray& realspace_ba, const amrex::DistributionMapping& dm, - int norder_x, int norder_y, - int norder_z, short grid_type, + int norder_x, + int norder_y, + int norder_z, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, const amrex::Vector& v_comoving, amrex::RealVect dx, @@ -205,5 +209,5 @@ class SpectralSolver // to an instance of a sub-class defining a specific algorithm std::unique_ptr algorithm; }; -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT #endif // WARPX_SPECTRAL_SOLVER_H_ diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp index 80fb2c39545..c6192187064 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralSolver.cpp @@ -14,18 +14,23 @@ #include "SpectralKSpace.H" #include "SpectralSolver.H" #include "Utils/TextMsg.H" +#include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXProfilerWrapper.H" +#include + #include -#if WARPX_USE_PSATD +#if WARPX_USE_FFT -SpectralSolver::SpectralSolver( +SpectralSolver::SpectralSolver ( const int lev, const amrex::BoxArray& realspace_ba, const amrex::DistributionMapping& dm, - const int norder_x, const int norder_y, - const int norder_z, const short grid_type, + const int norder_x, + const int norder_y, + const int norder_z, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, const amrex::Vector& v_comoving, const amrex::RealVect dx, const amrex::Real dt, @@ -144,4 +149,4 @@ SpectralSolver::pushSpectralFields(){ algorithm->pushSpectralFields( field_data ); } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H index 30bfe5cf16c..2e94fe95da2 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H +++ b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.H @@ -12,6 +12,9 @@ #include "SpectralAlgorithms/SpectralBaseAlgorithmRZ.H" #include "SpectralFieldDataRZ.H" +#include + + /* \brief Top-level class for the electromagnetic spectral solver * * Stores the field in spectral space, and has member functions @@ -31,7 +34,8 @@ class SpectralSolverRZ amrex::BoxArray const & realspace_ba, amrex::DistributionMapping const & dm, int n_rz_azimuthal_modes, - int norder_z, short grid_type, + int norder_z, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, amrex::RealVect dx, amrex::Real dt, bool with_pml, diff --git a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp index 9b0d2f21f94..3529247ef56 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralSolverRZ.cpp @@ -29,7 +29,8 @@ SpectralSolverRZ::SpectralSolverRZ (const int lev, amrex::BoxArray const & realspace_ba, amrex::DistributionMapping const & dm, int const n_rz_azimuthal_modes, - int const norder_z, short const grid_type, + int const norder_z, + ablastr::utils::enums::GridType grid_type, const amrex::Vector& v_galilean, amrex::RealVect const dx, amrex::Real const dt, bool const with_pml, diff --git a/Source/FieldSolver/WarpXPushFieldsEM.cpp b/Source/FieldSolver/WarpXPushFieldsEM.cpp index 0d29cdca74a..bc75e08be00 100644 --- a/Source/FieldSolver/WarpXPushFieldsEM.cpp +++ b/Source/FieldSolver/WarpXPushFieldsEM.cpp @@ -11,7 +11,7 @@ #include "BoundaryConditions/PML.H" #include "Evolve/WarpXDtType.H" #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" -#if defined(WARPX_USE_PSATD) +#if defined(WARPX_USE_FFT) # include "FieldSolver/SpectralSolver/SpectralFieldData.H" # ifdef WARPX_DIM_RZ # include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" @@ -54,7 +54,7 @@ using namespace amrex; -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT namespace { void ForwardTransformVect ( @@ -645,12 +645,12 @@ WarpX::PSATDScaleAverageFields (const amrex::Real scale_factor) } } } -#endif // WARPX_USE_PSATD +#endif // WARPX_USE_FFT void WarpX::PushPSATD () { -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT WARPX_ABORT_WITH_MESSAGE( "PushFieldsEM: PSATD solver selected but not built"); #else @@ -1245,11 +1245,11 @@ WarpX::ApplyInverseVolumeScalingToCurrentDensity (MultiFab* Jx, MultiFab* Jy, Mu // Lower corner of tile box physical domain // Note that this is done before the tilebox.grow so that // these do not include the guard cells. - const std::array& xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); - const Real rmin = xyzmin[0]; - const Real rminr = xyzmin[0] + (tbr.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); - const Real rmint = xyzmin[0] + (tbt.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); - const Real rminz = xyzmin[0] + (tbz.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); + const Real rmin = xyzmin.x; + const Real rminr = xyzmin.x + (tbr.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); + const Real rmint = xyzmin.x + (tbt.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); + const Real rminz = xyzmin.x + (tbz.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); const Dim3 lo = lbound(tilebox); const int irmin = lo.x; @@ -1416,10 +1416,10 @@ WarpX::ApplyInverseVolumeScalingToChargeDensity (MultiFab* Rho, int lev) // Lower corner of tile box physical domain // Note that this is done before the tilebox.grow so that // these do not include the guard cells. - const std::array& xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(tilebox, lev, 0._rt); const Dim3 lo = lbound(tilebox); - const Real rmin = xyzmin[0]; - const Real rminr = xyzmin[0] + (tb.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); + const Real rmin = xyzmin.x; + const Real rminr = xyzmin.x + (tb.type(0) == NODE ? 0._rt : 0.5_rt*dx[0]); const int irmin = lo.x; const int ishift = (rminr > rmin ? 1 : 0); diff --git a/Source/Filter/Filter.cpp b/Source/Filter/Filter.cpp index 5952ffde59c..6243ce4ebbf 100644 --- a/Source/Filter/Filter.cpp +++ b/Source/Filter/Filter.cpp @@ -9,6 +9,7 @@ #include "Utils/TextMsg.H" #include "Utils/WarpXProfilerWrapper.H" +#include "WarpX.H" #include #include diff --git a/Source/Initialization/CMakeLists.txt b/Source/Initialization/CMakeLists.txt index 34807338c4d..bfb134d6660 100644 --- a/Source/Initialization/CMakeLists.txt +++ b/Source/Initialization/CMakeLists.txt @@ -12,6 +12,7 @@ foreach(D IN LISTS WarpX_DIMS) VelocityProperties.cpp ReconnectionPerturbation.cpp WarpXAMReXInit.cpp + WarpXInit.cpp WarpXInitData.cpp ) endforeach() diff --git a/Source/Initialization/ExternalField.cpp b/Source/Initialization/ExternalField.cpp index e7a272e2e95..69120c0ae3c 100644 --- a/Source/Initialization/ExternalField.cpp +++ b/Source/Initialization/ExternalField.cpp @@ -172,7 +172,6 @@ ExternalFieldParams::ExternalFieldParams(const amrex::ParmParse& pp_warpx) // // External fields from file // - if (E_ext_grid_type == ExternalFieldType::read_from_file || B_ext_grid_type == ExternalFieldType::read_from_file){ const std::string read_fields_from_path="./"; diff --git a/Source/Initialization/Make.package b/Source/Initialization/Make.package index bf4f78a2168..e643e61dc2c 100644 --- a/Source/Initialization/Make.package +++ b/Source/Initialization/Make.package @@ -8,6 +8,7 @@ CEXE_sources += TemperatureProperties.cpp CEXE_sources += VelocityProperties.cpp CEXE_sources += ReconnectionPerturbation.cpp CEXE_sources += WarpXAMReXInit.cpp +CEXE_sources += WarpXInit.cpp CEXE_sources += WarpXInitData.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Initialization diff --git a/Source/Initialization/VelocityProperties.cpp b/Source/Initialization/VelocityProperties.cpp index 14b48c79524..b6986970e31 100644 --- a/Source/Initialization/VelocityProperties.cpp +++ b/Source/Initialization/VelocityProperties.cpp @@ -18,26 +18,25 @@ VelocityProperties::VelocityProperties (const amrex::ParmParse& pp, std::string std::string vel_dir_s = "x"; utils::parser::query(pp, source_name, "bulk_vel_dir", vel_dir_s); - if(vel_dir_s[0] == '-'){ - m_sign_dir = -1; - } - else { - m_sign_dir = 1; + + if(vel_dir_s.empty()){ + WARPX_ABORT_WITH_MESSAGE("'.bulk_vel_dir input ' can't be empty."); } - if ((vel_dir_s == "x" || vel_dir_s[1] == 'x') || - (vel_dir_s == "X" || vel_dir_s[1] == 'X')){ + m_sign_dir = (vel_dir_s[0] == '-') ? -1 : 1; + + const auto dir = std::tolower(vel_dir_s.back()); + + if (dir == 'x'){ m_dir = 0; } - else if ((vel_dir_s == "y" || vel_dir_s[1] == 'y') || - (vel_dir_s == "Y" || vel_dir_s[1] == 'Y')){ + else if (dir == 'y'){ m_dir = 1; } - else if ((vel_dir_s == "z" || vel_dir_s[1] == 'z') || - (vel_dir_s == "Z" || vel_dir_s[1] == 'Z')) { + else if (dir == 'z'){ m_dir = 2; } - else { + else{ WARPX_ABORT_WITH_MESSAGE( "Cannot interpret .bulk_vel_dir input '" + vel_dir_s + "'. Please enter +/- x, y, or z with no whitespace between the sign and"+ diff --git a/Source/Initialization/WarpXAMReXInit.cpp b/Source/Initialization/WarpXAMReXInit.cpp index 4036a49235c..900dbb53e06 100644 --- a/Source/Initialization/WarpXAMReXInit.cpp +++ b/Source/Initialization/WarpXAMReXInit.cpp @@ -7,9 +7,12 @@ #include "Initialization/WarpXAMReXInit.H" +#include "Utils/TextMsg.H" + #include #include #include +#include #include @@ -46,6 +49,22 @@ namespace { pp_amr.add("blocking_factor", 1); } + //See https://github.com/AMReX-Codes/amrex/pull/3763 +#ifdef AMREX_USE_GPU + bool warpx_do_device_synchronize = true; +#else + bool warpx_do_device_synchronize = false; +#endif + pp_warpx.query("do_device_synchronize", warpx_do_device_synchronize); + bool do_device_synchronize = warpx_do_device_synchronize; + amrex::ParmParse pp_tiny_profiler("tiny_profiler"); + if (pp_tiny_profiler.queryAdd("device_synchronize_around_region", do_device_synchronize) ) + { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + do_device_synchronize == warpx_do_device_synchronize, + "tiny_profiler.device_synchronize_around_region overrides warpx.do_device_synchronize."); + } + // Here we override the default tiling option for particles, which is always // "false" in AMReX, to "false" if compiling for GPU execution and "true" // if compiling for CPU. diff --git a/Source/Initialization/WarpXInit.H b/Source/Initialization/WarpXInit.H new file mode 100644 index 00000000000..ce179e2e997 --- /dev/null +++ b/Source/Initialization/WarpXInit.H @@ -0,0 +1,30 @@ +/* Copyright 2024 Luca Fedeli + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef WARPX_INIT_H_ +#define WARPX_INIT_H_ + +namespace warpx::initialization +{ + /** Initializes, in the following order: + * - the MPI library through the mpi_init helper function in ablastr + * - the AMReX library + * - the FFT library through the anyfft::setup() function in ablastr + * + * @param[in] argc number of arguments from main() + * @param[in] argv argument strings from main() + */ + void initialize_external_libraries(int argc, char* argv[]); + + /** Initializes, in the following order: + * - the FFT library through the anyfft::cleanup() function in ablastr + * - the AMReX library + * - the MPI library through the mpi_finalize helper function in ablastr + */ + void finalize_external_libraries(); +} + +#endif //WARPX_INIT_H_ diff --git a/Source/Initialization/WarpXInit.cpp b/Source/Initialization/WarpXInit.cpp new file mode 100644 index 00000000000..7e00760bf30 --- /dev/null +++ b/Source/Initialization/WarpXInit.cpp @@ -0,0 +1,29 @@ +/* Copyright 2024 Luca Fedeli + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ + +#include "WarpXInit.H" + +#include "Initialization/WarpXAMReXInit.H" + +#include + +#include +#include + +void warpx::initialization::initialize_external_libraries(int argc, char* argv[]) +{ + ablastr::parallelization::mpi_init(argc, argv); + warpx::initialization::amrex_init(argc, argv); + ablastr::math::anyfft::setup(); +} + +void warpx::initialization::finalize_external_libraries() +{ + ablastr::math::anyfft::cleanup(); + amrex::Finalize(); + ablastr::parallelization::mpi_finalize(); +} diff --git a/Source/Initialization/WarpXInitData.cpp b/Source/Initialization/WarpXInitData.cpp index 6b0cba707fe..f88f64798f8 100644 --- a/Source/Initialization/WarpXInitData.cpp +++ b/Source/Initialization/WarpXInitData.cpp @@ -11,11 +11,12 @@ #include "WarpX.H" #include "BoundaryConditions/PML.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif #include "Diagnostics/MultiDiagnostics.H" #include "Diagnostics/ReducedDiags/MultiReducedDiags.H" +#include "FieldSolver/Fields.H" #include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" #include "Filter/BilinearFilter.H" @@ -158,13 +159,13 @@ WarpX::PrintMainPICparameters () amrex::Print() << "-------------------------------------------------------------------------------\n"; // print warpx build information - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { amrex::Print() << "Precision: | SINGLE" << "\n"; } else { amrex::Print() << "Precision: | DOUBLE" << "\n"; } - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { amrex::Print() << "Particle precision: | SINGLE" << "\n"; } else { @@ -282,7 +283,7 @@ WarpX::PrintMainPICparameters () else if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC){ amrex::Print() << "Maxwell Solver: | Hybrid-PIC (Ohm's law) \n"; } - #ifdef WARPX_USE_PSATD + #ifdef WARPX_USE_FFT // Print PSATD solver's configuration if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD){ amrex::Print() << "Maxwell Solver: | PSATD \n"; @@ -297,19 +298,19 @@ WarpX::PrintMainPICparameters () " | - v_comoving = (" << m_v_comoving[0] << "," << m_v_comoving[1] << "," << m_v_comoving[2] << ")\n"; } - if (WarpX::update_with_rho==1) { + if (WarpX::update_with_rho) { amrex::Print() << " | - update with rho is ON \n"; } - if (current_correction==1) { + if (current_correction) { amrex::Print() << " | - current correction is ON \n"; } - if (WarpX::do_dive_cleaning==1) { + if (WarpX::do_dive_cleaning) { amrex::Print() << " | - div(E) cleaning is ON \n"; } - if (WarpX::do_divb_cleaning==1) { + if (WarpX::do_divb_cleaning) { amrex::Print() << " | - div(B) cleaning is ON \n"; } - if (do_multi_J == 1){ + if (do_multi_J){ amrex::Print() << " | - multi-J deposition is ON \n"; amrex::Print() << " | - do_multi_J_n_depositions = " << WarpX::do_multi_J_n_depositions << "\n"; @@ -326,15 +327,15 @@ WarpX::PrintMainPICparameters () amrex::Print() << " | - rho_in_time = constant \n"; } } - if (fft_do_time_averaging == 1){ + if (fft_do_time_averaging){ amrex::Print()<<" | - time-averaged is ON \n"; } - #endif // WARPX_USE_PSATD + #endif // WARPX_USE_FFT if (grid_type == GridType::Collocated){ amrex::Print() << " | - collocated grid \n"; } - #ifdef WARPX_USE_PSATD + #ifdef WARPX_USE_FFT if ( (grid_type == GridType::Staggered) && (field_gathering_algo == GatheringAlgo::EnergyConserving) ){ amrex::Print()<<" | - staggered grid " << "\n"; } @@ -387,7 +388,7 @@ WarpX::PrintMainPICparameters () amrex::Print() << "Guard cells | - ng_alloc_EB = " << guard_cells.ng_alloc_EB << "\n"; amrex::Print() << " (allocated for E/B) | \n"; - #endif // WARPX_USE_PSATD + #endif // WARPX_USE_FFT amrex::Print() << "-------------------------------------------------------------------------------" << "\n"; //Print main boosted frame algorithm's parameters if (WarpX::gamma_boost!=1){ @@ -477,12 +478,9 @@ WarpX::InitData () BuildBufferMasks(); - if (WarpX::em_solver_medium==1) { + if (WarpX::em_solver_medium == MediumForEM::Macroscopic) { const int lev_zero = 0; m_macroscopic_properties->InitData( - boxArray(lev_zero), - DistributionMap(lev_zero), - getngEB(), Geom(lev_zero), getField(warpx::fields::FieldType::Efield_fp, lev_zero,0).ixType().toIntVect(), getField(warpx::fields::FieldType::Efield_fp, lev_zero,1).ixType().toIntVect(), @@ -504,6 +502,9 @@ WarpX::InitData () CheckGuardCells(); PrintMainPICparameters(); + if (m_implicit_solver) { + m_implicit_solver->PrintParameters(); + } WriteUsedInputsFile(); if (restart_chkfile.empty()) @@ -516,12 +517,6 @@ WarpX::InitData () if (electrostatic_solver_id == ElectrostaticSolverAlgo::LabFrameElectroMagnetostatic) { ComputeMagnetostaticField(); } - - // Set up an invariant condition through the rest of - // execution, that any code besides the field solver that - // looks at field values will see the composite of the field - // solution and any external field - AddExternalFields(); } if (restart_chkfile.empty() || write_diagnostics_on_restart) { @@ -542,15 +537,27 @@ WarpX::InitData () } void -WarpX::AddExternalFields () { - for (int lev = 0; lev <= finest_level; ++lev) { - // FIXME: RZ multimode has more than one component for all these - if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { +WarpX::AddExternalFields (int const lev) { + // FIXME: RZ multimode has more than one component for all these + if (m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::default_zero) { + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::constant) { + Efield_fp[lev][0]->plus(m_p_ext_field_params->E_external_grid[0], guard_cells.ng_alloc_EB.min()); + Efield_fp[lev][1]->plus(m_p_ext_field_params->E_external_grid[1], guard_cells.ng_alloc_EB.min()); + Efield_fp[lev][2]->plus(m_p_ext_field_params->E_external_grid[2], guard_cells.ng_alloc_EB.min()); + } + else { amrex::MultiFab::Add(*Efield_fp[lev][0], *Efield_fp_external[lev][0], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Efield_fp[lev][1], *Efield_fp_external[lev][1], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Efield_fp[lev][2], *Efield_fp_external[lev][2], 0, 0, 1, guard_cells.ng_alloc_EB); } - if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { + } + if (m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::default_zero) { + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::constant) { + Bfield_fp[lev][0]->plus(m_p_ext_field_params->B_external_grid[0], guard_cells.ng_alloc_EB.min()); + Bfield_fp[lev][1]->plus(m_p_ext_field_params->B_external_grid[1], guard_cells.ng_alloc_EB.min()); + Bfield_fp[lev][2]->plus(m_p_ext_field_params->B_external_grid[2], guard_cells.ng_alloc_EB.min()); + } + else { amrex::MultiFab::Add(*Bfield_fp[lev][0], *Bfield_fp_external[lev][0], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Bfield_fp[lev][1], *Bfield_fp_external[lev][1], 0, 0, 1, guard_cells.ng_alloc_EB); amrex::MultiFab::Add(*Bfield_fp[lev][2], *Bfield_fp_external[lev][2], 0, 0, 1, guard_cells.ng_alloc_EB); @@ -571,10 +578,33 @@ WarpX::InitFromScratch () AmrCore::InitFromScratch(time); // This will call MakeNewLevelFromScratch + if (m_implicit_solver) { + + m_implicit_solver->Define(this); + m_implicit_solver->GetParticleSolverParams( max_particle_its_in_implicit_scheme, + particle_tol_in_implicit_scheme ); + + // Add space to save the positions and velocities at the start of the time steps + for (auto const& pc : *mypc) { +#if (AMREX_SPACEDIM >= 2) + pc->AddRealComp("x_n"); +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + pc->AddRealComp("y_n"); +#endif + pc->AddRealComp("z_n"); + pc->AddRealComp("ux_n"); + pc->AddRealComp("uy_n"); + pc->AddRealComp("uz_n"); + } + + } + mypc->AllocData(); mypc->InitData(); InitPML(); + } void @@ -593,7 +623,7 @@ WarpX::InitPML () if (finest_level > 0) { do_pml = 1; } if (do_pml) { -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) do_pml_Lo[0][0] = 0; // no PML at r=0, in cylindrical geometry pml_rz[0] = std::make_unique(0, boxArray(0), DistributionMap(0), &Geom(0), pml_ncell, do_pml_in_domain); #else @@ -768,7 +798,7 @@ WarpX::PostRestart () { mypc->PostRestart(); for (int lev = 0; lev <= maxLevel(); ++lev) { - LoadExternalFieldsFromFile(lev); + LoadExternalFields(lev); } } @@ -790,7 +820,6 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::default_zero; if ( is_B_ext_const && (lev <= maxlevel_extEMfield_init) ) { - Bfield_fp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); if (fft_do_time_averaging) { Bfield_avg_fp[lev][i]->setVal(m_p_ext_field_params->B_external_grid[i]); } @@ -811,7 +840,6 @@ WarpX::InitLevelData (int lev, Real /*time*/) m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::default_zero; if ( is_E_ext_const && (lev <= maxlevel_extEMfield_init) ) { - Efield_fp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); if (fft_do_time_averaging) { Efield_avg_fp[lev][i]->setVal(m_p_ext_field_params->E_external_grid[i]); } @@ -836,13 +864,12 @@ WarpX::InitLevelData (int lev, Real /*time*/) // Externally imposed fields are only initialized until the user-defined maxlevel_extEMfield_init. // The default maxlevel_extEMfield_init value is the total number of levels in the simulation if ((m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) - && (lev <= maxlevel_extEMfield_init)) { + && (lev > 0) && (lev <= maxlevel_extEMfield_init)) { - // Initialize Bfield_fp with external function InitializeExternalFieldsOnGridUsingParser( - Bfield_fp[lev][0].get(), - Bfield_fp[lev][1].get(), - Bfield_fp[lev][2].get(), + Bfield_aux[lev][0].get(), + Bfield_aux[lev][1].get(), + Bfield_aux[lev][2].get(), m_p_ext_field_params->Bxfield_parser->compile<3>(), m_p_ext_field_params->Byfield_parser->compile<3>(), m_p_ext_field_params->Bzfield_parser->compile<3>(), @@ -851,31 +878,17 @@ WarpX::InitLevelData (int lev, Real /*time*/) 'B', lev, PatchType::fine); - if (lev > 0) { - InitializeExternalFieldsOnGridUsingParser( - Bfield_aux[lev][0].get(), - Bfield_aux[lev][1].get(), - Bfield_aux[lev][2].get(), - m_p_ext_field_params->Bxfield_parser->compile<3>(), - m_p_ext_field_params->Byfield_parser->compile<3>(), - m_p_ext_field_params->Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'B', - lev, PatchType::fine); - - InitializeExternalFieldsOnGridUsingParser( - Bfield_cp[lev][0].get(), - Bfield_cp[lev][1].get(), - Bfield_cp[lev][2].get(), - m_p_ext_field_params->Bxfield_parser->compile<3>(), - m_p_ext_field_params->Byfield_parser->compile<3>(), - m_p_ext_field_params->Bzfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'B', - lev, PatchType::coarse); - } + InitializeExternalFieldsOnGridUsingParser( + Bfield_cp[lev][0].get(), + Bfield_cp[lev][1].get(), + Bfield_cp[lev][2].get(), + m_p_ext_field_params->Bxfield_parser->compile<3>(), + m_p_ext_field_params->Byfield_parser->compile<3>(), + m_p_ext_field_params->Bzfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'B', + lev, PatchType::coarse); } ParmParse pp_warpx("warpx"); int IncludeBfieldPerturbation = 0; @@ -924,19 +937,6 @@ WarpX::InitLevelData (int lev, Real /*time*/) if ((m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::parse_ext_grid_function) && (lev <= maxlevel_extEMfield_init)) { - // Initialize Efield_fp with external function - InitializeExternalFieldsOnGridUsingParser( - Efield_fp[lev][0].get(), - Efield_fp[lev][1].get(), - Efield_fp[lev][2].get(), - m_p_ext_field_params->Exfield_parser->compile<3>(), - m_p_ext_field_params->Eyfield_parser->compile<3>(), - m_p_ext_field_params->Ezfield_parser->compile<3>(), - m_edge_lengths[lev], - m_face_areas[lev], - 'E', - lev, PatchType::fine); - #ifdef AMREX_USE_EB // We initialize ECTRhofield consistently with the Efield if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) { @@ -981,7 +981,10 @@ WarpX::InitLevelData (int lev, Real /*time*/) } } - LoadExternalFieldsFromFile(lev); + // load external grid fields into E/Bfield_fp_external multifabs + LoadExternalFields(lev); + // add the external fields to the fine patch fields as initial conditions for the fields + AddExternalFields(lev); if (costs[lev]) { const auto iarr = costs[lev]->IndexArray(); @@ -1372,9 +1375,37 @@ void WarpX::CheckKnownIssues() } void -WarpX::LoadExternalFieldsFromFile (int const lev) +WarpX::LoadExternalFields (int const lev) { - if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { + // External fields from file are currently not compatible with the moving window + // In order to support the moving window, the MultiFab containing the external + // fields should be updated every time the window moves. + if ( (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) || + (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) || + (mypc->m_B_ext_particle_s == "read_from_file") || + (mypc->m_E_ext_particle_s == "read_from_file") ) { + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + WarpX::do_moving_window == 0, + "External fields from file are not compatible with the moving window." ); + } + + // External grid fields + if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { + // Initialize Bfield_fp_external with external function + InitializeExternalFieldsOnGridUsingParser( + Bfield_fp_external[lev][0].get(), + Bfield_fp_external[lev][1].get(), + Bfield_fp_external[lev][2].get(), + m_p_ext_field_params->Bxfield_parser->compile<3>(), + m_p_ext_field_params->Byfield_parser->compile<3>(), + m_p_ext_field_params->Bzfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'B', + lev, PatchType::fine); + } + else if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); @@ -1387,7 +1418,22 @@ WarpX::LoadExternalFieldsFromFile (int const lev) ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Bfield_fp_external[lev][2].get(), "B", "z"); #endif } - if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { + + if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::parse_ext_grid_function) { + // Initialize Efield_fp_external with external function + InitializeExternalFieldsOnGridUsingParser( + Efield_fp_external[lev][0].get(), + Efield_fp_external[lev][1].get(), + Efield_fp_external[lev][2].get(), + m_p_ext_field_params->Exfield_parser->compile<3>(), + m_p_ext_field_params->Eyfield_parser->compile<3>(), + m_p_ext_field_params->Ezfield_parser->compile<3>(), + m_edge_lengths[lev], + m_face_areas[lev], + 'E', + lev, PatchType::fine); + } + else if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { #if defined(WARPX_DIM_RZ) WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, "External field reading is not implemented for more than one RZ mode (see #3829)"); @@ -1398,6 +1444,41 @@ WarpX::LoadExternalFieldsFromFile (int const lev) ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][0].get(), "E", "x"); ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][1].get(), "E", "y"); ReadExternalFieldFromFile(m_p_ext_field_params->external_fields_path, Efield_fp_external[lev][2].get(), "E", "z"); +#endif + } + + // External particle fields + + if (mypc->m_B_ext_particle_s == "read_from_file") { + std::string external_fields_path; + const amrex::ParmParse pp_particles("particles"); + pp_particles.get("read_fields_from_path", external_fields_path ); +#if defined(WARPX_DIM_RZ) + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, + "External field reading is not implemented for more than one RZ mode (see #3829)"); + ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][0].get(), "B", "r"); + ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][1].get(), "B", "t"); + ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][2].get(), "B", "z"); +#else + ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][0].get(), "B", "x"); + ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][1].get(), "B", "y"); + ReadExternalFieldFromFile(external_fields_path, B_external_particle_field[lev][2].get(), "B", "z"); +#endif + } + if (mypc->m_E_ext_particle_s == "read_from_file") { + std::string external_fields_path; + const amrex::ParmParse pp_particles("particles"); + pp_particles.get("read_fields_from_path", external_fields_path ); +#if defined(WARPX_DIM_RZ) + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(n_rz_azimuthal_modes == 1, + "External field reading is not implemented for more than one RZ mode (see #3829)"); + ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][0].get(), "E", "r"); + ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][1].get(), "E", "t"); + ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][2].get(), "E", "z"); +#else + ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][0].get(), "E", "x"); + ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][1].get(), "E", "y"); + ReadExternalFieldFromFile(external_fields_path, E_external_particle_field[lev][2].get(), "E", "z"); #endif } } diff --git a/Source/Make.WarpX b/Source/Make.WarpX index 9051c3598e6..18405ca548d 100644 --- a/Source/Make.WarpX +++ b/Source/Make.WarpX @@ -76,6 +76,7 @@ include $(WARPX_HOME)/Source/BoundaryConditions/Make.package include $(WARPX_HOME)/Source/Diagnostics/Make.package include $(WARPX_HOME)/Source/EmbeddedBoundary/Make.package include $(WARPX_HOME)/Source/FieldSolver/Make.package +include $(WARPX_HOME)/Source/NonlinearSolvers/Make.package include $(WARPX_HOME)/Source/Filter/Make.package include $(WARPX_HOME)/Source/Fluids/Make.package include $(WARPX_HOME)/Source/Initialization/Make.package @@ -90,7 +91,7 @@ include $(AMREX_HOME)/Src/Base/Make.package include $(AMREX_HOME)/Src/Particle/Make.package include $(AMREX_HOME)/Src/Boundary/Make.package include $(AMREX_HOME)/Src/AmrCore/Make.package -include $(AMREX_HOME)/Src/LinearSolvers/MLMG/Make.package +include $(AMREX_HOME)/Src/LinearSolvers/Make.package ifeq ($(USE_SENSEI_INSITU),TRUE) include $(AMREX_HOME)/Src/Amr/Make.package @@ -171,9 +172,9 @@ ifeq ($(USE_OPENPMD), TRUE) endif -ifeq ($(USE_PSATD),TRUE) +ifeq ($(USE_FFT),TRUE) USERSuffix := $(USERSuffix).PSATD - DEFINES += -DWARPX_USE_PSATD -DABLASTR_USE_FFT + DEFINES += -DWARPX_USE_FFT -DABLASTR_USE_FFT ifeq ($(USE_CUDA),TRUE) # Use cuFFT libraries += -lcufft @@ -327,7 +328,7 @@ $(srcTempDir)/Utils/export.H: @echo "#endif" >> $@ cleanconfig:: - $(SILENT) $(RM) -r $(srcTempDir)/Utils + $(SILENT) $(RM) -r $(srcTempDir)/Utils $(srcTempDir)/ablastr include $(AMREX_HOME)/Tools/GNUMake/Make.rules diff --git a/Source/NonlinearSolvers/CMakeLists.txt b/Source/NonlinearSolvers/CMakeLists.txt new file mode 100644 index 00000000000..938cdaad547 --- /dev/null +++ b/Source/NonlinearSolvers/CMakeLists.txt @@ -0,0 +1,6 @@ +foreach(D IN LISTS WarpX_DIMS) + warpx_set_suffix_dims(SD ${D}) + target_sources(lib_${SD} + PRIVATE + ) +endforeach() diff --git a/Source/NonlinearSolvers/JacobianFunctionMF.H b/Source/NonlinearSolvers/JacobianFunctionMF.H new file mode 100644 index 00000000000..823523df23c --- /dev/null +++ b/Source/NonlinearSolvers/JacobianFunctionMF.H @@ -0,0 +1,234 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef JacobianFunctionMF_H_ +#define JacobianFunctionMF_H_ + +/** + * \brief This is a linear function class for computing the action of a + * Jacobian on a vector using a matrix-free finite-difference method. + * This class has all of the required functions to be used as the + * linear operator template parameter in AMReX_GMRES. + */ + +template +class JacobianFunctionMF +{ + public: + + using RT = typename T::value_type; + + JacobianFunctionMF() = default; + ~JacobianFunctionMF() = default; + + // Default move and copy operations + JacobianFunctionMF(const JacobianFunctionMF&) = default; + JacobianFunctionMF& operator=(const JacobianFunctionMF&) = default; + JacobianFunctionMF(JacobianFunctionMF&&) noexcept = default; + JacobianFunctionMF& operator=(JacobianFunctionMF&&) noexcept = default; + + void apply ( T& a_dF, const T& a_dU ); + + inline + void precond ( T& a_U, const T& a_X ) + { + if (m_usePreCond) { a_U.zero(); } + else { a_U.Copy(a_X); } + } + + inline + void updatePreCondMat ( const T& a_X ) + { + amrex::ignore_unused(a_X); + } + + inline + void create ( T& a_Z, const T& a_U ) + { + a_Z.define(a_U); + } + + T makeVecLHS () const; + T makeVecRHS () const; + + inline + void assign( T& a_Z, const T& a_U ) { + a_Z.Copy(a_U); + } + + inline + void increment( T& a_Z, const T& a_U, RT a_scale ) + { + a_Z.increment(a_U,a_scale); + } + + inline + void scale ( T& a_U, RT a_scale ) + { + a_U.scale(a_scale); + } + + inline + void linComb ( T& a_U, RT a, const T& X, RT b, const T& Y ) + { + a_U.linComb( a, X, b, Y ); + } + + inline + void setToZero ( T& a_U ) + { + a_U.zero(); + } + + inline + void setVal ( T& a_U, RT a_val ) + { + a_U.setVal(a_val); + } + + inline + RT dotProduct( const T& a_X, const T& a_Y ) + { + return( a_X.dotProduct(a_Y) ); + } + + inline + RT norm2( const T& a_U ) + { + return ( a_U.norm2() ); + } + + [[nodiscard]] inline + bool isDefined() const { return m_is_defined; } + + inline + void setBaseSolution ( const T& a_U ) + { + m_Y0.Copy(a_U); + m_normY0 = norm2(m_Y0); + } + + inline + void setBaseRHS ( const T& a_R ) + { + m_R0.Copy(a_R); + } + + inline + void setJFNKEps ( RT a_eps ) + { + m_epsJFNK = a_eps; + } + + inline + void setIsLinear ( bool a_isLinear ) + { + m_is_linear = a_isLinear; + } + + inline + void curTime ( RT a_time ) + { + m_cur_time = a_time; + } + + inline + void curTimeStep ( RT a_dt ) + { + m_dt = a_dt; + } + + void define( const T&, Ops* ); + + private: + + bool m_is_defined = false; + bool m_is_linear = false; + bool m_usePreCond = false; + RT m_epsJFNK = RT(1.0e-6); + RT m_normY0; + RT m_cur_time, m_dt; + std::string m_pc_type; + + T m_Z, m_Y0, m_R0, m_R; + Ops* m_ops; + +}; + +template +void JacobianFunctionMF::define ( const T& a_U, + Ops* a_ops ) +{ + m_Z.Define(a_U); + m_Y0.Define(a_U); + m_R0.Define(a_U); + m_R.Define(a_U); + + m_ops = a_ops; + + m_is_defined = true; +} + +template +auto JacobianFunctionMF::makeVecRHS () const -> T +{ + T Vec; + Vec.Define(m_R); + return Vec; +} + +template +auto JacobianFunctionMF::makeVecLHS () const -> T +{ + T Vec; + Vec.Define(m_R); + return Vec; +} + +template +void JacobianFunctionMF::apply (T& a_dF, const T& a_dU) +{ + BL_PROFILE("JacobianFunctionMF::apply()"); + using namespace amrex::literals; + RT const normY = norm2(a_dU); // always 1 when called from GMRES + + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + isDefined(), + "JacobianFunction::apply() called on undefined JacobianFunction"); + + if (normY < 1.0e-15) { a_dF.zero(); } + else { + + RT eps; + if (m_is_linear) { + eps = 1.0_rt; + } else { + /* eps = error_rel * sqrt(1 + ||Y0||) / ||dU|| + * M. Pernice and H. F. Walker, "NITSOL: A Newton Iterative Solver for + * Nonlinear Systems", SIAM J. Sci. Stat. Comput., 1998, vol 19, + * pp. 302--318. */ + if (m_normY0==0.0) { eps = m_epsJFNK * norm2(m_R0) / normY; } + else { + // m_epsJFNK * sqrt(1.0 + m_normY0) / normY + // above commonly used form not recommend for poorly scaled Y0 + eps = m_epsJFNK * m_normY0 / normY; + } + } + const RT eps_inv = 1.0_rt/eps; + + m_Z.linComb( 1.0, m_Y0, eps, a_dU ); // Z = Y0 + eps*dU + m_ops->ComputeRHS(m_R, m_Z, m_cur_time, m_dt, -1, true ); + + // F(Y) = Y - b - R(Y) ==> dF = dF/dY*dU = [1 - dR/dY]*dU + // = dU - (R(Z)-R(Y0))/eps + a_dF.linComb( 1.0, a_dU, eps_inv, m_R0 ); + a_dF.increment(m_R,-eps_inv); + + } + +} + +#endif diff --git a/Source/NonlinearSolvers/Make.package b/Source/NonlinearSolvers/Make.package new file mode 100644 index 00000000000..4f2dcfa9617 --- /dev/null +++ b/Source/NonlinearSolvers/Make.package @@ -0,0 +1 @@ +VPATH_LOCATIONS += $(WARPX_HOME)/Source/NonlinearSolvers diff --git a/Source/NonlinearSolvers/NewtonSolver.H b/Source/NonlinearSolvers/NewtonSolver.H new file mode 100644 index 00000000000..93ad432208a --- /dev/null +++ b/Source/NonlinearSolvers/NewtonSolver.H @@ -0,0 +1,345 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef NEWTON_SOLVER_H_ +#define NEWTON_SOLVER_H_ + +#include "NonlinearSolver.H" +#include "JacobianFunctionMF.H" + +#include +#include +#include "Utils/TextMsg.H" + +#include + +/** + * \brief Newton method to solve nonlinear equation of form: + * F(U) = U - b - R(U) = 0. U is the solution vector, b is a constant, + * and R(U) is some nonlinear function of U, which is computed in the + * ComputeRHS() Ops function. + */ + +template +class NewtonSolver : public NonlinearSolver +{ +public: + + NewtonSolver() = default; + + ~NewtonSolver() override = default; + + // Prohibit Move and Copy operations + NewtonSolver(const NewtonSolver&) = delete; + NewtonSolver& operator=(const NewtonSolver&) = delete; + NewtonSolver(NewtonSolver&&) noexcept = delete; + NewtonSolver& operator=(NewtonSolver&&) noexcept = delete; + + void Define ( const Vec& a_U, + Ops* a_ops ) override; + + void Solve ( Vec& a_U, + const Vec& a_b, + amrex::Real a_time, + amrex::Real a_dt ) const override; + + void GetSolverParams ( amrex::Real& a_rtol, + amrex::Real& a_atol, + int& a_maxits ) override + { + a_rtol = m_rtol; + a_atol = m_atol; + a_maxits = m_maxits; + } + + inline void CurTime ( amrex::Real a_time ) const + { + m_cur_time = a_time; + m_linear_function->curTime( a_time ); + } + + inline void CurTimeStep ( amrex::Real a_dt ) const + { + m_dt = a_dt; + m_linear_function->curTimeStep( a_dt ); + } + + void PrintParams () const override + { + amrex::Print() << "Newton verbose: " << (this->m_verbose?"true":"false") << std::endl; + amrex::Print() << "Newton max iterations: " << m_maxits << std::endl; + amrex::Print() << "Newton relative tolerance: " << m_rtol << std::endl; + amrex::Print() << "Newton absolute tolerance: " << m_atol << std::endl; + amrex::Print() << "Newton require convergence: " << (m_require_convergence?"true":"false") << std::endl; + amrex::Print() << "GMRES verbose: " << m_gmres_verbose_int << std::endl; + amrex::Print() << "GMRES restart length: " << m_gmres_restart_length << std::endl; + amrex::Print() << "GMRES max iterations: " << m_gmres_maxits << std::endl; + amrex::Print() << "GMRES relative tolerance: " << m_gmres_rtol << std::endl; + amrex::Print() << "GMRES absolute tolerance: " << m_gmres_atol << std::endl; + } + +private: + + /** + * \brief Intermediate Vec containers used by the solver. + */ + mutable Vec m_dU, m_F, m_R; + + /** + * \brief Pointer to Ops class. + */ + Ops* m_ops = nullptr; + + /** + * \brief Flag to determine whether convergence is required. + */ + bool m_require_convergence = true; + + /** + * \brief Relative tolerance for the Newton solver. + */ + amrex::Real m_rtol = 1.0e-6; + + /** + * \brief Absolute tolerance for the Newton solver. + */ + amrex::Real m_atol = 0.; + + /** + * \brief Maximum iterations for the Newton solver. + */ + int m_maxits = 100; + + /** + * \brief Relative tolerance for GMRES. + */ + amrex::Real m_gmres_rtol = 1.0e-4; + + /** + * \brief Absolute tolerance for GMRES. + */ + amrex::Real m_gmres_atol = 0.; + + /** + * \brief Maximum iterations for GMRES. + */ + int m_gmres_maxits = 1000; + + /** + * \brief Verbosity flag for GMRES. + */ + int m_gmres_verbose_int = 2; + + /** + * \brief Restart iteration for GMRES. + */ + int m_gmres_restart_length = 30; + + mutable amrex::Real m_cur_time, m_dt; + mutable bool m_update_pc = false; + mutable bool m_update_pc_init = false; + + /** + * \brief The linear function used by GMRES to compute A*v. + * In the contect of JFNK, A = dF/dU (i.e., system Jacobian) + */ + std::unique_ptr> m_linear_function; + + /** + * \brief The linear solver (GMRES) object. + */ + std::unique_ptr>> m_linear_solver; + + void ParseParameters (); + + /** + * \brief Compute the nonlinear residual: F(U) = U - b - R(U). + */ + void EvalResidual ( Vec& a_F, + const Vec& a_U, + const Vec& a_b, + amrex::Real a_time, + amrex::Real a_dt, + int a_iter ) const; + +}; + +template +void NewtonSolver::Define ( const Vec& a_U, + Ops* a_ops ) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !this->m_is_defined, + "Newton nonlinear solver object is already defined!"); + + ParseParameters(); + + m_dU.Define(a_U); + m_F.Define(a_U); // residual function F(U) = U - b - R(U) = 0 + m_R.Define(a_U); // right hand side function R(U) + + m_ops = a_ops; + + m_linear_function = std::make_unique>(); + m_linear_function->define(m_F, m_ops); + + m_linear_solver = std::make_unique>>(); + m_linear_solver->define(*m_linear_function); + m_linear_solver->setVerbose( m_gmres_verbose_int ); + m_linear_solver->setRestartLength( m_gmres_restart_length ); + m_linear_solver->setMaxIters( m_gmres_maxits ); + + this->m_is_defined = true; + +} + +template +void NewtonSolver::ParseParameters () +{ + const amrex::ParmParse pp_newton("newton"); + pp_newton.query("verbose", this->m_verbose); + pp_newton.query("absolute_tolerance", m_atol); + pp_newton.query("relative_tolerance", m_rtol); + pp_newton.query("max_iterations", m_maxits); + pp_newton.query("require_convergence", m_require_convergence); + + const amrex::ParmParse pp_gmres("gmres"); + pp_gmres.query("verbose_int", m_gmres_verbose_int); + pp_gmres.query("restart_length", m_gmres_restart_length); + pp_gmres.query("absolute_tolerance", m_gmres_atol); + pp_gmres.query("relative_tolerance", m_gmres_rtol); + pp_gmres.query("max_iterations", m_gmres_maxits); +} + +template +void NewtonSolver::Solve ( Vec& a_U, + const Vec& a_b, + amrex::Real a_time, + amrex::Real a_dt ) const +{ + BL_PROFILE("NewtonSolver::Solve()"); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + this->m_is_defined, + "NewtonSolver::Solve() called on undefined object"); + using namespace amrex::literals; + + // + // Newton routine to solve nonlinear equation of form: + // F(U) = U - b - R(U) = 0 + // + + CurTime(a_time); + CurTimeStep(a_dt); + + amrex::Real norm_abs = 0.; + amrex::Real norm0 = 1._rt; + amrex::Real norm_rel = 0.; + + int iter; + for (iter = 0; iter < m_maxits;) { + + // Compute residual: F(U) = U - b - R(U) + EvalResidual(m_F, a_U, a_b, a_time, a_dt, iter); + + // Compute norm of the residual + norm_abs = m_F.norm2(); + if (iter == 0) { + if (norm_abs > 0.) { norm0 = norm_abs; } + else { norm0 = 1._rt; } + } + norm_rel = norm_abs/norm0; + + // Check for convergence criteria + if (this->m_verbose || iter == m_maxits) { + amrex::Print() << "Newton: iteration = " << std::setw(3) << iter << ", norm = " + << std::scientific << std::setprecision(5) << norm_abs << " (abs.), " + << std::scientific << std::setprecision(5) << norm_rel << " (rel.)" << "\n"; + } + + if (norm_abs < m_rtol) { + amrex::Print() << "Newton: exiting at iteration = " << std::setw(3) << iter + << ". Satisfied absolute tolerance " << m_atol << std::endl; + break; + } + + if (norm_rel < m_rtol) { + amrex::Print() << "Newton: exiting at iteration = " << std::setw(3) << iter + << ". Satisfied relative tolerance " << m_rtol << std::endl; + break; + } + + if (norm_abs > 100._rt*norm0) { + amrex::Print() << "Newton: exiting at iteration = " << std::setw(3) << iter + << ". SOLVER DIVERGED! relative tolerance = " << m_rtol << std::endl; + std::stringstream convergenceMsg; + convergenceMsg << "Newton: exiting at iteration " << std::setw(3) << iter << + ". SOLVER DIVERGED! absolute norm = " << norm_abs << + " has increased by 100X from that after first iteration."; + WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); + } + + // Solve linear system for Newton step [Jac]*dU = F + m_dU.zero(); + m_linear_solver->solve( m_dU, m_F, m_gmres_rtol, m_gmres_atol ); + + // Update solution + a_U -= m_dU; + + iter++; + if (iter >= m_maxits) { + amrex::Print() << "Newton: exiting at iter = " << std::setw(3) << iter + << ". Maximum iteration reached: iter = " << m_maxits << std::endl; + break; + } + + } + + if (m_rtol > 0. && iter == m_maxits) { + std::stringstream convergenceMsg; + convergenceMsg << "Newton solver failed to converge after " << iter << + " iterations. Relative norm is " << norm_rel << + " and the relative tolerance is " << m_rtol << + ". Absolute norm is " << norm_abs << + " and the absolute tolerance is " << m_atol; + if (this->m_verbose) { amrex::Print() << convergenceMsg.str() << std::endl; } + if (m_require_convergence) { + WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); + } else { + ablastr::warn_manager::WMRecordWarning("NewtonSolver", convergenceMsg.str()); + } + } + +} + +template +void NewtonSolver::EvalResidual ( Vec& a_F, + const Vec& a_U, + const Vec& a_b, + amrex::Real a_time, + amrex::Real a_dt, + int a_iter ) const +{ + + m_ops->ComputeRHS( m_R, a_U, a_time, a_dt, a_iter, false ); + + // set base U and R(U) for matrix-free Jacobian action calculation + m_linear_function->setBaseSolution(a_U); + m_linear_function->setBaseRHS(m_R); + + // update preconditioner + if (m_update_pc || m_update_pc_init) { + m_linear_function->updatePreCondMat(a_U); + } + m_update_pc_init = false; + + // Compute residual: F(U) = U - b - R(U) + a_F.Copy(a_U); + a_F -= m_R; + a_F -= a_b; + +} + +#endif diff --git a/Source/NonlinearSolvers/NonlinearSolver.H b/Source/NonlinearSolvers/NonlinearSolver.H new file mode 100644 index 00000000000..5587826474c --- /dev/null +++ b/Source/NonlinearSolvers/NonlinearSolver.H @@ -0,0 +1,87 @@ +#ifndef WARPX_NONLINEAR_SOLVER_H_ +#define WARPX_NONLINEAR_SOLVER_H_ + +#include +#include + +#include + +#include +#include +#include + +/** + * \brief Top-level class for the nonlinear solver + * + * This class is templated on a vector class Vec, and an operator class Ops. + * + * The Ops class must have the following function: + * ComputeRHS( R_vec, U_vec, time, dt, nl_iter, from_jacobian ), + * where U_vec and R_vec are of type Vec. + * + * The Vec class must have basic math operators, such as Copy, +=, -=, + * increment(), linComb(), scale(), etc.. See WarpXSolverVec.H for an example. + */ + +template +class NonlinearSolver +{ +public: + + NonlinearSolver() = default; + + virtual ~NonlinearSolver() = default; + + // Prohibit Move and Copy operations + NonlinearSolver(const NonlinearSolver&) = delete; + NonlinearSolver& operator=(const NonlinearSolver&) = delete; + NonlinearSolver(NonlinearSolver&&) noexcept = delete; + NonlinearSolver& operator=(NonlinearSolver&&) noexcept = delete; + + /** + * \brief Read user-provided parameters that control the nonlinear solver. + * Allocate intermediate data containers needed by the solver. For Newton, + * setup the linear solver for computing the Newton step. + */ + virtual void Define ( const Vec&, + Ops* ) = 0; + + /** + * \brief Check if the nonlinear solver has been defined. + */ + [[nodiscard]] bool IsDefined () const { return m_is_defined; } + + /** + * \brief Solve the specified nonlinear equation for U. + * Picard: U = b + R(U). + * Newton: F(U) = U - b - R(U) = 0. + */ + virtual void Solve ( Vec&, + const Vec&, + amrex::Real, + amrex::Real ) const = 0; + + /** + * \brief Print parameters used by the nonlinear solver. + */ + virtual void PrintParams () const = 0; + + /** + * \brief Return the convergence parameters used by the nonlinear solver. + */ + virtual void GetSolverParams (amrex::Real&, amrex::Real&, int&) = 0; + + /** + * \brief Allow caller to dynamically change the verbosity flag. For + * example, one may want to only print solver information every 100 steps. + */ + void Verbose ( bool a_verbose ) { m_verbose = a_verbose; } + +protected: + + bool m_is_defined = false; + mutable bool m_verbose = true; + +}; + +#endif diff --git a/Source/NonlinearSolvers/NonlinearSolverLibrary.H b/Source/NonlinearSolvers/NonlinearSolverLibrary.H new file mode 100644 index 00000000000..5803d6ca753 --- /dev/null +++ b/Source/NonlinearSolvers/NonlinearSolverLibrary.H @@ -0,0 +1,15 @@ +#ifndef NONLINEAR_SOLVER_LIBRARY_H_ +#define NONLINEAR_SOLVER_LIBRARY_H_ + +#include "PicardSolver.H" // IWYU pragma: export +#include "NewtonSolver.H" // IWYU pragma: export + +/** + * \brief struct to select the nonlinear solver for implicit schemes + */ +enum NonlinearSolverType { + Picard = 0, + Newton = 1 +}; + +#endif diff --git a/Source/NonlinearSolvers/PicardSolver.H b/Source/NonlinearSolvers/PicardSolver.H new file mode 100644 index 00000000000..f05b9a106e6 --- /dev/null +++ b/Source/NonlinearSolvers/PicardSolver.H @@ -0,0 +1,217 @@ +/* Copyright 2024 Justin Angus + * + * This file is part of WarpX. + * + * License: BSD-3-Clause-LBNL + */ +#ifndef PICARD_SOLVER_H_ +#define PICARD_SOLVER_H_ + +#include "NonlinearSolver.H" + +#include +#include "Utils/TextMsg.H" + +#include + +/** + * \brief Picard fixed-point iteration method to solve nonlinear + * equation of form: U = b + R(U). U is the solution vector. b + * is a constant. R(U) is some nonlinear function of U, which + * is computed in the Ops function ComputeRHS(). + */ + +template +class PicardSolver : public NonlinearSolver +{ +public: + + PicardSolver() = default; + + ~PicardSolver() override = default; + + // Prohibit Move and Copy operations + PicardSolver(const PicardSolver&) = delete; + PicardSolver& operator=(const PicardSolver&) = delete; + PicardSolver(PicardSolver&&) noexcept = delete; + PicardSolver& operator=(PicardSolver&&) noexcept = delete; + + void Define ( const Vec& a_U, + Ops* a_ops ) override; + + void Solve ( Vec& a_U, + const Vec& a_b, + amrex::Real a_time, + amrex::Real a_dt ) const override; + + void GetSolverParams ( amrex::Real& a_rtol, + amrex::Real& a_atol, + int& a_maxits ) override + { + a_rtol = m_rtol; + a_atol = m_atol; + a_maxits = m_maxits; + } + + void PrintParams () const override + { + amrex::Print() << "Picard max iterations: " << m_maxits << std::endl; + amrex::Print() << "Picard relative tolerance: " << m_rtol << std::endl; + amrex::Print() << "Picard absolute tolerance: " << m_atol << std::endl; + amrex::Print() << "Picard require convergence: " << (m_require_convergence?"true":"false") << std::endl; + } + +private: + + /** + * \brief Intermediate Vec containers used by the solver. + */ + mutable Vec m_Usave, m_R; + + /** + * \brief Pointer to Ops class. + */ + Ops* m_ops = nullptr; + + /** + * \brief Flag to determine whether convergence is required. + */ + bool m_require_convergence = true; + + /** + * \brief Relative tolerance for the Picard nonlinear solver + */ + amrex::Real m_rtol = 1.0e-6; + + /** + * \brief Absolute tolerance for the Picard nonlinear solver + */ + amrex::Real m_atol = 0.; + + /** + * \brief Maximum iterations for the Picard nonlinear solver + */ + int m_maxits = 100; + + void ParseParameters( ); + +}; + +template +void PicardSolver::Define ( const Vec& a_U, + Ops* a_ops ) +{ + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + !this->m_is_defined, + "Picard nonlinear solver object is already defined!"); + + ParseParameters(); + + m_Usave.Define(a_U); + m_R.Define(a_U); + + m_ops = a_ops; + + this->m_is_defined = true; + +} + +template +void PicardSolver::ParseParameters () +{ + const amrex::ParmParse pp_picard("picard"); + pp_picard.query("verbose", this->m_verbose); + pp_picard.query("absolute_tolerance", m_atol); + pp_picard.query("relative_tolerance", m_rtol); + pp_picard.query("max_iterations", m_maxits); + pp_picard.query("require_convergence", m_require_convergence); + +} + +template +void PicardSolver::Solve ( Vec& a_U, + const Vec& a_b, + amrex::Real a_time, + amrex::Real a_dt ) const +{ + BL_PROFILE("PicardSolver::Solve()"); + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + this->m_is_defined, + "PicardSolver::Solve() called on undefined object"); + using namespace amrex::literals; + + // + // Picard fixed-point iteration method to solve nonlinear + // equation of form: U = b + R(U) + // + + amrex::Real norm_abs = 0.; + amrex::Real norm0 = 1._rt; + amrex::Real norm_rel = 0.; + + int iter; + for (iter = 0; iter < m_maxits;) { + + // Save previous state for norm calculation + m_Usave.Copy(a_U); + + // Update the solver state (a_U = a_b + m_R) + m_ops->ComputeRHS( m_R, a_U, a_time, a_dt, iter, false ); + a_U.Copy(a_b); + a_U += m_R; + + // Compute the step norm and update iter + m_Usave -= a_U; + norm_abs = m_Usave.norm2(); + if (iter == 0) { + if (norm_abs > 0.) { norm0 = norm_abs; } + else { norm0 = 1._rt; } + } + norm_rel = norm_abs/norm0; + iter++; + + // Check for convergence criteria + if (this->m_verbose || iter == m_maxits) { + amrex::Print() << "Picard: iter = " << std::setw(3) << iter << ", norm = " + << std::scientific << std::setprecision(5) << norm_abs << " (abs.), " + << std::scientific << std::setprecision(5) << norm_rel << " (rel.)" << "\n"; + } + + if (norm_abs < m_atol) { + amrex::Print() << "Picard: exiting at iter = " << std::setw(3) << iter + << ". Satisfied absolute tolerance " << m_atol << std::endl; + break; + } + + if (norm_rel < m_rtol) { + amrex::Print() << "Picard: exiting at iter = " << std::setw(3) << iter + << ". Satisfied relative tolerance " << m_rtol << std::endl; + break; + } + + if (iter >= m_maxits) { + amrex::Print() << "Picard: exiting at iter = " << std::setw(3) << iter + << ". Maximum iteration reached: iter = " << m_maxits << std::endl; + break; + } + + } + + if (m_rtol > 0. && iter == m_maxits) { + std::stringstream convergenceMsg; + convergenceMsg << "Picard solver failed to converge after " << iter << + " iterations. Relative norm is " << norm_rel << + " and the relative tolerance is " << m_rtol << + ". Absolute norm is " << norm_abs << + " and the absolute tolerance is " << m_atol; + if (this->m_verbose) { amrex::Print() << convergenceMsg.str() << std::endl; } + if (m_require_convergence) { + WARPX_ABORT_WITH_MESSAGE(convergenceMsg.str()); + } else { + ablastr::warn_manager::WMRecordWarning("PicardSolver", convergenceMsg.str()); + } + } + +} + +#endif diff --git a/Source/Parallelization/GuardCellManager.H b/Source/Parallelization/GuardCellManager.H index 341db01bef6..c70bd6d3a35 100644 --- a/Source/Parallelization/GuardCellManager.H +++ b/Source/Parallelization/GuardCellManager.H @@ -7,6 +7,8 @@ #ifndef WARPX_GUARDCELLMANAGER_H_ #define WARPX_GUARDCELLMANAGER_H_ +#include + #include #include #include @@ -52,10 +54,10 @@ public: */ void Init( amrex::Real dt, - amrex::RealVect dx, + const amrex::Real * dx, bool do_subcycling, bool do_fdtd_nci_corr, - short grid_type, + ablastr::utils::enums::GridType grid_type, bool do_moving_window, int moving_window_dir, int nox, diff --git a/Source/Parallelization/GuardCellManager.cpp b/Source/Parallelization/GuardCellManager.cpp index 28157e09d8c..4c2f978484d 100644 --- a/Source/Parallelization/GuardCellManager.cpp +++ b/Source/Parallelization/GuardCellManager.cpp @@ -33,10 +33,10 @@ using namespace amrex; void guardCellManager::Init ( const amrex::Real dt, - const amrex::RealVect dx, + const amrex::Real *dx, const bool do_subcycling, const bool do_fdtd_nci_corr, - const short grid_type, + ablastr::utils::enums::GridType grid_type, const bool do_moving_window, const int moving_window_dir, const int nox, @@ -59,9 +59,9 @@ guardCellManager::Init ( // When using subcycling, the particles on the fine level perform two pushes // before being redistributed ; therefore, we need one extra guard cell // (the particles may move by 2*c*dt) - int ngx_tmp = (max_level > 0 && do_subcycling == 1) ? nox+1 : nox; - int ngy_tmp = (max_level > 0 && do_subcycling == 1) ? nox+1 : nox; - int ngz_tmp = (max_level > 0 && do_subcycling == 1) ? nox+1 : nox; + int ngx_tmp = (max_level > 0 && do_subcycling) ? nox+1 : nox; + int ngy_tmp = (max_level > 0 && do_subcycling) ? nox+1 : nox; + int ngz_tmp = (max_level > 0 && do_subcycling) ? nox+1 : nox; const bool galilean = (v_galilean[0] != 0. || v_galilean[1] != 0. || v_galilean[2] != 0.); const bool comoving = (v_comoving[0] != 0. || v_comoving[1] != 0. || v_comoving[2] != 0.); @@ -200,6 +200,7 @@ guardCellManager::Init ( // currents in the latter case). This does not seem to be necessary in x and y, // where it still seems fine to set half the number of guard cells of the nodal case. + using namespace ablastr::utils::enums; int ngFFt_x = (grid_type == GridType::Collocated) ? nox_fft : nox_fft / 2; int ngFFt_y = (grid_type == GridType::Collocated) ? noy_fft : noy_fft / 2; int ngFFt_z = (grid_type == GridType::Collocated || galilean) ? noz_fft : noz_fft / 2; diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index f3bff69546b..6c44df061fd 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -9,7 +9,7 @@ #include "WarpX.H" #include "BoundaryConditions/PML.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif #include "Filter/BilinearFilter.H" @@ -18,6 +18,7 @@ #include "Utils/WarpXProfilerWrapper.H" #include "WarpXComm_K.H" #include "WarpXSumGuardCells.H" +#include "Particles/MultiParticleContainer.H" #include #include @@ -59,12 +60,27 @@ WarpX::UpdateAuxilaryData () } else { UpdateAuxilaryDataStagToNodal(); } + + // When loading particle fields from file: add the external fields: + for (int lev = 0; lev <= finest_level; ++lev) { + if (mypc->m_E_ext_particle_s == "read_from_file") { + amrex::MultiFab::Add(*Efield_aux[lev][0], *E_external_particle_field[lev][0], 0, 0, E_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Efield_aux[lev][1], *E_external_particle_field[lev][1], 0, 0, E_external_particle_field[lev][1]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Efield_aux[lev][2], *E_external_particle_field[lev][2], 0, 0, E_external_particle_field[lev][2]->nComp(), guard_cells.ng_FieldGather); + } + if (mypc->m_B_ext_particle_s == "read_from_file") { + amrex::MultiFab::Add(*Bfield_aux[lev][0], *B_external_particle_field[lev][0], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux[lev][1], *B_external_particle_field[lev][1], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); + amrex::MultiFab::Add(*Bfield_aux[lev][2], *B_external_particle_field[lev][2], 0, 0, B_external_particle_field[lev][0]->nComp(), guard_cells.ng_FieldGather); + } + } + } void WarpX::UpdateAuxilaryDataStagToNodal () { -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( false, "WarpX::UpdateAuxilaryDataStagToNodal: PSATD solver requires " @@ -155,136 +171,191 @@ WarpX::UpdateAuxilaryDataStagToNodal () // Bfield { - Array,3> Btmp; - if (Bfield_cax[lev][0]) { - for (int i = 0; i < 3; ++i) { - Btmp[i] = std::make_unique( - *Bfield_cax[lev][i], amrex::make_alias, 0, 1); + if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { + Array,3> Btmp; + if (Bfield_cax[lev][0]) { + for (int i = 0; i < 3; ++i) { + Btmp[i] = std::make_unique( + *Bfield_cax[lev][i], amrex::make_alias, 0, 1); + } + } else { + const IntVect ngtmp = Bfield_aux[lev-1][0]->nGrowVect(); + for (int i = 0; i < 3; ++i) { + Btmp[i] = std::make_unique(cnba, dm, 1, ngtmp); + } } - } else { - const IntVect ngtmp = Bfield_aux[lev-1][0]->nGrowVect(); + Btmp[0]->setVal(0.0); + Btmp[1]->setVal(0.0); + Btmp[2]->setVal(0.0); + // ParallelCopy from coarse level for (int i = 0; i < 3; ++i) { - Btmp[i] = std::make_unique(cnba, dm, 1, ngtmp); + const IntVect ng = Btmp[i]->nGrowVect(); + // Guard cells may not be up to date beyond ng_FieldGather + const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; + // Copy Bfield_aux to Btmp, using up to ng_src (=ng_FieldGather) guard cells from + // Bfield_aux and filling up to ng (=nGrow) guard cells in Btmp + ablastr::utils::communication::ParallelCopy(*Btmp[i], *Bfield_aux[lev - 1][i], 0, 0, 1, + ng_src, ng, WarpX::do_single_precision_comms, cperiod); } - } - Btmp[0]->setVal(0.0); - Btmp[1]->setVal(0.0); - Btmp[2]->setVal(0.0); - // ParallelCopy from coarse level - for (int i = 0; i < 3; ++i) { - const IntVect ng = Btmp[i]->nGrowVect(); - // Guard cells may not be up to date beyond ng_FieldGather - const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; - // Copy Bfield_aux to Btmp, using up to ng_src (=ng_FieldGather) guard cells from - // Bfield_aux and filling up to ng (=nGrow) guard cells in Btmp - ablastr::utils::communication::ParallelCopy(*Btmp[i], *Bfield_aux[lev - 1][i], 0, 0, 1, - ng_src, ng, WarpX::do_single_precision_comms, cperiod); - } - const amrex::IntVect& refinement_ratio = refRatio(lev-1); + const amrex::IntVect& refinement_ratio = refRatio(lev-1); - const amrex::IntVect& Bx_fp_stag = Bfield_fp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& By_fp_stag = Bfield_fp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Bz_fp_stag = Bfield_fp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Bx_fp_stag = Bfield_fp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& By_fp_stag = Bfield_fp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Bz_fp_stag = Bfield_fp[lev][2]->ixType().toIntVect(); - const amrex::IntVect& Bx_cp_stag = Bfield_cp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& By_cp_stag = Bfield_cp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Bz_cp_stag = Bfield_cp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Bx_cp_stag = Bfield_cp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& By_cp_stag = Bfield_cp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Bz_cp_stag = Bfield_cp[lev][2]->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*Bfield_aux[lev][0], TilingIfNotGPU()); mfi.isValid(); ++mfi) - { - Array4 const& bx_aux = Bfield_aux[lev][0]->array(mfi); - Array4 const& by_aux = Bfield_aux[lev][1]->array(mfi); - Array4 const& bz_aux = Bfield_aux[lev][2]->array(mfi); - Array4 const& bx_fp = Bfield_fp[lev][0]->const_array(mfi); - Array4 const& by_fp = Bfield_fp[lev][1]->const_array(mfi); - Array4 const& bz_fp = Bfield_fp[lev][2]->const_array(mfi); - Array4 const& bx_cp = Bfield_cp[lev][0]->const_array(mfi); - Array4 const& by_cp = Bfield_cp[lev][1]->const_array(mfi); - Array4 const& bz_cp = Bfield_cp[lev][2]->const_array(mfi); - Array4 const& bx_c = Btmp[0]->const_array(mfi); - Array4 const& by_c = Btmp[1]->const_array(mfi); - Array4 const& bz_c = Btmp[2]->const_array(mfi); - - const Box& bx = mfi.growntilebox(); - amrex::ParallelFor(bx, - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + for (MFIter mfi(*Bfield_aux[lev][0], TilingIfNotGPU()); mfi.isValid(); ++mfi) { - warpx_interp(j, k, l, bx_aux, bx_fp, bx_cp, bx_c, Bx_fp_stag, Bx_cp_stag, refinement_ratio); - warpx_interp(j, k, l, by_aux, by_fp, by_cp, by_c, By_fp_stag, By_cp_stag, refinement_ratio); - warpx_interp(j, k, l, bz_aux, bz_fp, bz_cp, bz_c, Bz_fp_stag, Bz_cp_stag, refinement_ratio); - }); + Array4 const& bx_aux = Bfield_aux[lev][0]->array(mfi); + Array4 const& by_aux = Bfield_aux[lev][1]->array(mfi); + Array4 const& bz_aux = Bfield_aux[lev][2]->array(mfi); + Array4 const& bx_fp = Bfield_fp[lev][0]->const_array(mfi); + Array4 const& by_fp = Bfield_fp[lev][1]->const_array(mfi); + Array4 const& bz_fp = Bfield_fp[lev][2]->const_array(mfi); + Array4 const& bx_cp = Bfield_cp[lev][0]->const_array(mfi); + Array4 const& by_cp = Bfield_cp[lev][1]->const_array(mfi); + Array4 const& bz_cp = Bfield_cp[lev][2]->const_array(mfi); + Array4 const& bx_c = Btmp[0]->const_array(mfi); + Array4 const& by_c = Btmp[1]->const_array(mfi); + Array4 const& bz_c = Btmp[2]->const_array(mfi); + + const Box& bx = mfi.growntilebox(); + amrex::ParallelFor(bx, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, bx_aux, bx_fp, bx_cp, bx_c, Bx_fp_stag, Bx_cp_stag, refinement_ratio); + warpx_interp(j, k, l, by_aux, by_fp, by_cp, by_c, By_fp_stag, By_cp_stag, refinement_ratio); + warpx_interp(j, k, l, bz_aux, bz_fp, bz_cp, bz_c, Bz_fp_stag, Bz_cp_stag, refinement_ratio); + }); + } + } + else { // electrostatic + const amrex::IntVect& Bx_fp_stag = Bfield_fp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& By_fp_stag = Bfield_fp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Bz_fp_stag = Bfield_fp[lev][2]->ixType().toIntVect(); +#ifdef AMREX_USE_OMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*Bfield_aux[lev][0], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + Array4 const& bx_aux = Bfield_aux[lev][0]->array(mfi); + Array4 const& by_aux = Bfield_aux[lev][1]->array(mfi); + Array4 const& bz_aux = Bfield_aux[lev][2]->array(mfi); + Array4 const& bx_fp = Bfield_fp[lev][0]->const_array(mfi); + Array4 const& by_fp = Bfield_fp[lev][1]->const_array(mfi); + Array4 const& bz_fp = Bfield_fp[lev][2]->const_array(mfi); + + const Box& bx = mfi.growntilebox(); + amrex::ParallelFor(bx, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, bx_aux, bx_fp, Bx_fp_stag); + warpx_interp(j, k, l, by_aux, by_fp, By_fp_stag); + warpx_interp(j, k, l, bz_aux, bz_fp, Bz_fp_stag); + }); + } } } - // Efield { - Array,3> Etmp; - if (Efield_cax[lev][0]) { - for (int i = 0; i < 3; ++i) { - Etmp[i] = std::make_unique( - *Efield_cax[lev][i], amrex::make_alias, 0, 1); + if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { + Array,3> Etmp; + if (Efield_cax[lev][0]) { + for (int i = 0; i < 3; ++i) { + Etmp[i] = std::make_unique( + *Efield_cax[lev][i], amrex::make_alias, 0, 1); + } + } else { + const IntVect ngtmp = Efield_aux[lev-1][0]->nGrowVect(); + for (int i = 0; i < 3; ++i) { + Etmp[i] = std::make_unique( + cnba, dm, 1, ngtmp); + } } - } else { - const IntVect ngtmp = Efield_aux[lev-1][0]->nGrowVect(); + Etmp[0]->setVal(0.0); + Etmp[1]->setVal(0.0); + Etmp[2]->setVal(0.0); + // ParallelCopy from coarse level for (int i = 0; i < 3; ++i) { - Etmp[i] = std::make_unique( - cnba, dm, 1, ngtmp); + const IntVect ng = Etmp[i]->nGrowVect(); + // Guard cells may not be up to date beyond ng_FieldGather + const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; + // Copy Efield_aux to Etmp, using up to ng_src (=ng_FieldGather) guard cells from + // Efield_aux and filling up to ng (=nGrow) guard cells in Etmp + ablastr::utils::communication::ParallelCopy(*Etmp[i], *Efield_aux[lev - 1][i], 0, 0, 1, + ng_src, ng, WarpX::do_single_precision_comms, cperiod); } - } - Etmp[0]->setVal(0.0); - Etmp[1]->setVal(0.0); - Etmp[2]->setVal(0.0); - // ParallelCopy from coarse level - for (int i = 0; i < 3; ++i) { - const IntVect ng = Etmp[i]->nGrowVect(); - // Guard cells may not be up to date beyond ng_FieldGather - const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; - // Copy Efield_aux to Etmp, using up to ng_src (=ng_FieldGather) guard cells from - // Efield_aux and filling up to ng (=nGrow) guard cells in Etmp - ablastr::utils::communication::ParallelCopy(*Etmp[i], *Efield_aux[lev - 1][i], 0, 0, 1, - ng_src, ng, WarpX::do_single_precision_comms, cperiod); - } - const amrex::IntVect& refinement_ratio = refRatio(lev-1); + const amrex::IntVect& refinement_ratio = refRatio(lev-1); - const amrex::IntVect& Ex_fp_stag = Efield_fp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& Ey_fp_stag = Efield_fp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Ez_fp_stag = Efield_fp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Ex_fp_stag = Efield_fp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& Ey_fp_stag = Efield_fp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Ez_fp_stag = Efield_fp[lev][2]->ixType().toIntVect(); - const amrex::IntVect& Ex_cp_stag = Efield_cp[lev][0]->ixType().toIntVect(); - const amrex::IntVect& Ey_cp_stag = Efield_cp[lev][1]->ixType().toIntVect(); - const amrex::IntVect& Ez_cp_stag = Efield_cp[lev][2]->ixType().toIntVect(); + const amrex::IntVect& Ex_cp_stag = Efield_cp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& Ey_cp_stag = Efield_cp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Ez_cp_stag = Efield_cp[lev][2]->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*Efield_aux[lev][0]); mfi.isValid(); ++mfi) - { - Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); - Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); - Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); - Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); - Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); - Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); - Array4 const& ex_cp = Efield_cp[lev][0]->const_array(mfi); - Array4 const& ey_cp = Efield_cp[lev][1]->const_array(mfi); - Array4 const& ez_cp = Efield_cp[lev][2]->const_array(mfi); - Array4 const& ex_c = Etmp[0]->const_array(mfi); - Array4 const& ey_c = Etmp[1]->const_array(mfi); - Array4 const& ez_c = Etmp[2]->const_array(mfi); - - const Box& bx = mfi.fabbox(); - amrex::ParallelFor(bx, - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + for (MFIter mfi(*Efield_aux[lev][0], TilingIfNotGPU()); mfi.isValid(); ++mfi) { - warpx_interp(j, k, l, ex_aux, ex_fp, ex_cp, ex_c, Ex_fp_stag, Ex_cp_stag, refinement_ratio); - warpx_interp(j, k, l, ey_aux, ey_fp, ey_cp, ey_c, Ey_fp_stag, Ey_cp_stag, refinement_ratio); - warpx_interp(j, k, l, ez_aux, ez_fp, ez_cp, ez_c, Ez_fp_stag, Ez_cp_stag, refinement_ratio); - }); + Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); + Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); + Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); + Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); + Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); + Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); + Array4 const& ex_cp = Efield_cp[lev][0]->const_array(mfi); + Array4 const& ey_cp = Efield_cp[lev][1]->const_array(mfi); + Array4 const& ez_cp = Efield_cp[lev][2]->const_array(mfi); + Array4 const& ex_c = Etmp[0]->const_array(mfi); + Array4 const& ey_c = Etmp[1]->const_array(mfi); + Array4 const& ez_c = Etmp[2]->const_array(mfi); + + const Box& bx = mfi.growntilebox(); + amrex::ParallelFor(bx, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, ex_aux, ex_fp, ex_cp, ex_c, Ex_fp_stag, Ex_cp_stag, refinement_ratio); + warpx_interp(j, k, l, ey_aux, ey_fp, ey_cp, ey_c, Ey_fp_stag, Ey_cp_stag, refinement_ratio); + warpx_interp(j, k, l, ez_aux, ez_fp, ez_cp, ez_c, Ez_fp_stag, Ez_cp_stag, refinement_ratio); + }); + } + } + else { // electrostatic + const amrex::IntVect& Ex_fp_stag = Efield_fp[lev][0]->ixType().toIntVect(); + const amrex::IntVect& Ey_fp_stag = Efield_fp[lev][1]->ixType().toIntVect(); + const amrex::IntVect& Ez_fp_stag = Efield_fp[lev][2]->ixType().toIntVect(); +#ifdef AMREX_USE_OMP +#pragma omp parallel if (Gpu::notInLaunchRegion()) +#endif + for (MFIter mfi(*Efield_aux[lev][0], TilingIfNotGPU()); mfi.isValid(); ++mfi) + { + Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); + Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); + Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); + Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); + Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); + Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); + + const Box& bx = mfi.growntilebox(); + amrex::ParallelFor(bx, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, ex_aux, ex_fp, Ex_fp_stag); + warpx_interp(j, k, l, ey_aux, ey_fp, Ey_fp_stag); + warpx_interp(j, k, l, ez_aux, ez_fp, Ez_fp_stag); + }); + } } } } @@ -293,6 +364,30 @@ WarpX::UpdateAuxilaryDataStagToNodal () void WarpX::UpdateAuxilaryDataSameType () { + // Update aux field, including guard cells, up to ng_FieldGather + const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; + + // Level 0: Copy from fine to aux + // Note: in some configurations, Efield_aux/Bfield_aux and Efield_fp/Bfield_fp are simply aliases to the + // same MultiFab object. MultiFab::Copy operation automatically detects this and does nothing in this case. + if (WarpX::fft_do_time_averaging) + { + MultiFab::Copy(*Efield_aux[0][0], *Efield_avg_fp[0][0], 0, 0, Efield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][1], *Efield_avg_fp[0][1], 0, 0, Efield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][2], *Efield_avg_fp[0][2], 0, 0, Efield_aux[0][2]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][0], *Bfield_avg_fp[0][0], 0, 0, Bfield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][1], *Bfield_avg_fp[0][1], 0, 0, Bfield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][2], *Bfield_avg_fp[0][2], 0, 0, Bfield_aux[0][2]->nComp(), ng_src); + } + else + { + MultiFab::Copy(*Efield_aux[0][0], *Efield_fp[0][0], 0, 0, Efield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][1], *Efield_fp[0][1], 0, 0, Efield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Efield_aux[0][2], *Efield_fp[0][2], 0, 0, Efield_aux[0][2]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][0], *Bfield_fp[0][0], 0, 0, Bfield_aux[0][0]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][1], *Bfield_fp[0][1], 0, 0, Bfield_aux[0][1]->nComp(), ng_src); + MultiFab::Copy(*Bfield_aux[0][2], *Bfield_fp[0][2], 0, 0, Bfield_aux[0][2]->nComp(), ng_src); + } for (int lev = 1; lev <= finest_level; ++lev) { const amrex::Periodicity& crse_period = Geom(lev-1).periodicity(); @@ -301,145 +396,158 @@ WarpX::UpdateAuxilaryDataSameType () // B field { - MultiFab dBx(Bfield_cp[lev][0]->boxArray(), dm, Bfield_cp[lev][0]->nComp(), ng); - MultiFab dBy(Bfield_cp[lev][1]->boxArray(), dm, Bfield_cp[lev][1]->nComp(), ng); - MultiFab dBz(Bfield_cp[lev][2]->boxArray(), dm, Bfield_cp[lev][2]->nComp(), ng); - dBx.setVal(0.0); - dBy.setVal(0.0); - dBz.setVal(0.0); - - // Guard cells may not be up to date beyond ng_FieldGather - const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; - // Copy Bfield_aux to the dB MultiFabs, using up to ng_src (=ng_FieldGather) guard - // cells from Bfield_aux and filling up to ng (=nGrow) guard cells in the dB MultiFabs - - ablastr::utils::communication::ParallelCopy(dBx, *Bfield_aux[lev - 1][0], 0, 0, - Bfield_aux[lev - 1][0]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, - crse_period); - ablastr::utils::communication::ParallelCopy(dBy, *Bfield_aux[lev - 1][1], 0, 0, - Bfield_aux[lev - 1][1]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, - crse_period); - ablastr::utils::communication::ParallelCopy(dBz, *Bfield_aux[lev - 1][2], 0, 0, - Bfield_aux[lev - 1][2]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, - crse_period); - - if (Bfield_cax[lev][0]) + if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { - MultiFab::Copy(*Bfield_cax[lev][0], dBx, 0, 0, Bfield_cax[lev][0]->nComp(), ng); - MultiFab::Copy(*Bfield_cax[lev][1], dBy, 0, 0, Bfield_cax[lev][1]->nComp(), ng); - MultiFab::Copy(*Bfield_cax[lev][2], dBz, 0, 0, Bfield_cax[lev][2]->nComp(), ng); - } - MultiFab::Subtract(dBx, *Bfield_cp[lev][0], 0, 0, Bfield_cp[lev][0]->nComp(), ng); - MultiFab::Subtract(dBy, *Bfield_cp[lev][1], 0, 0, Bfield_cp[lev][1]->nComp(), ng); - MultiFab::Subtract(dBz, *Bfield_cp[lev][2], 0, 0, Bfield_cp[lev][2]->nComp(), ng); + MultiFab dBx(Bfield_cp[lev][0]->boxArray(), dm, Bfield_cp[lev][0]->nComp(), ng); + MultiFab dBy(Bfield_cp[lev][1]->boxArray(), dm, Bfield_cp[lev][1]->nComp(), ng); + MultiFab dBz(Bfield_cp[lev][2]->boxArray(), dm, Bfield_cp[lev][2]->nComp(), ng); + dBx.setVal(0.0); + dBy.setVal(0.0); + dBz.setVal(0.0); + + // Copy Bfield_aux to the dB MultiFabs, using up to ng_src (=ng_FieldGather) guard + // cells from Bfield_aux and filling up to ng (=nGrow) guard cells in the dB MultiFabs + + ablastr::utils::communication::ParallelCopy(dBx, *Bfield_aux[lev - 1][0], 0, 0, + Bfield_aux[lev - 1][0]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, + crse_period); + ablastr::utils::communication::ParallelCopy(dBy, *Bfield_aux[lev - 1][1], 0, 0, + Bfield_aux[lev - 1][1]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, + crse_period); + ablastr::utils::communication::ParallelCopy(dBz, *Bfield_aux[lev - 1][2], 0, 0, + Bfield_aux[lev - 1][2]->nComp(), ng_src, ng, WarpX::do_single_precision_comms, + crse_period); + + if (Bfield_cax[lev][0]) + { + MultiFab::Copy(*Bfield_cax[lev][0], dBx, 0, 0, Bfield_cax[lev][0]->nComp(), ng); + MultiFab::Copy(*Bfield_cax[lev][1], dBy, 0, 0, Bfield_cax[lev][1]->nComp(), ng); + MultiFab::Copy(*Bfield_cax[lev][2], dBz, 0, 0, Bfield_cax[lev][2]->nComp(), ng); + } + MultiFab::Subtract(dBx, *Bfield_cp[lev][0], 0, 0, Bfield_cp[lev][0]->nComp(), ng); + MultiFab::Subtract(dBy, *Bfield_cp[lev][1], 0, 0, Bfield_cp[lev][1]->nComp(), ng); + MultiFab::Subtract(dBz, *Bfield_cp[lev][2], 0, 0, Bfield_cp[lev][2]->nComp(), ng); - const amrex::IntVect& refinement_ratio = refRatio(lev-1); + const amrex::IntVect& refinement_ratio = refRatio(lev-1); - const amrex::IntVect& Bx_stag = Bfield_aux[lev-1][0]->ixType().toIntVect(); - const amrex::IntVect& By_stag = Bfield_aux[lev-1][1]->ixType().toIntVect(); - const amrex::IntVect& Bz_stag = Bfield_aux[lev-1][2]->ixType().toIntVect(); + const amrex::IntVect& Bx_stag = Bfield_aux[lev-1][0]->ixType().toIntVect(); + const amrex::IntVect& By_stag = Bfield_aux[lev-1][1]->ixType().toIntVect(); + const amrex::IntVect& Bz_stag = Bfield_aux[lev-1][2]->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*Bfield_aux[lev][0]); mfi.isValid(); ++mfi) - { - Array4 const& bx_aux = Bfield_aux[lev][0]->array(mfi); - Array4 const& by_aux = Bfield_aux[lev][1]->array(mfi); - Array4 const& bz_aux = Bfield_aux[lev][2]->array(mfi); - Array4 const& bx_fp = Bfield_fp[lev][0]->const_array(mfi); - Array4 const& by_fp = Bfield_fp[lev][1]->const_array(mfi); - Array4 const& bz_fp = Bfield_fp[lev][2]->const_array(mfi); - Array4 const& bx_c = dBx.const_array(mfi); - Array4 const& by_c = dBy.const_array(mfi); - Array4 const& bz_c = dBz.const_array(mfi); - - amrex::ParallelFor(Box(bx_aux), Box(by_aux), Box(bz_aux), - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept - { - warpx_interp(j, k, l, bx_aux, bx_fp, bx_c, Bx_stag, refinement_ratio); - }, - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept - { - warpx_interp(j, k, l, by_aux, by_fp, by_c, By_stag, refinement_ratio); - }, - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + for (MFIter mfi(*Bfield_aux[lev][0]); mfi.isValid(); ++mfi) { - warpx_interp(j, k, l, bz_aux, bz_fp, bz_c, Bz_stag, refinement_ratio); - }); + Array4 const& bx_aux = Bfield_aux[lev][0]->array(mfi); + Array4 const& by_aux = Bfield_aux[lev][1]->array(mfi); + Array4 const& bz_aux = Bfield_aux[lev][2]->array(mfi); + Array4 const& bx_fp = Bfield_fp[lev][0]->const_array(mfi); + Array4 const& by_fp = Bfield_fp[lev][1]->const_array(mfi); + Array4 const& bz_fp = Bfield_fp[lev][2]->const_array(mfi); + Array4 const& bx_c = dBx.const_array(mfi); + Array4 const& by_c = dBy.const_array(mfi); + Array4 const& bz_c = dBz.const_array(mfi); + + amrex::ParallelFor(Box(bx_aux), Box(by_aux), Box(bz_aux), + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, bx_aux, bx_fp, bx_c, Bx_stag, refinement_ratio); + }, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, by_aux, by_fp, by_c, By_stag, refinement_ratio); + }, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, bz_aux, bz_fp, bz_c, Bz_stag, refinement_ratio); + }); + } + } + else // electrostatic + { + MultiFab::Copy(*Bfield_aux[lev][0], *Bfield_fp[lev][0], 0, 0, Bfield_aux[lev][0]->nComp(), Bfield_aux[lev][0]->nGrowVect()); + MultiFab::Copy(*Bfield_aux[lev][1], *Bfield_fp[lev][1], 0, 0, Bfield_aux[lev][1]->nComp(), Bfield_aux[lev][1]->nGrowVect()); + MultiFab::Copy(*Bfield_aux[lev][2], *Bfield_fp[lev][2], 0, 0, Bfield_aux[lev][2]->nComp(), Bfield_aux[lev][2]->nGrowVect()); } } - // E field { - MultiFab dEx(Efield_cp[lev][0]->boxArray(), dm, Efield_cp[lev][0]->nComp(), ng); - MultiFab dEy(Efield_cp[lev][1]->boxArray(), dm, Efield_cp[lev][1]->nComp(), ng); - MultiFab dEz(Efield_cp[lev][2]->boxArray(), dm, Efield_cp[lev][2]->nComp(), ng); - dEx.setVal(0.0); - dEy.setVal(0.0); - dEz.setVal(0.0); - - // Guard cells may not be up to date beyond ng_FieldGather - const amrex::IntVect& ng_src = guard_cells.ng_FieldGather; - // Copy Efield_aux to the dE MultiFabs, using up to ng_src (=ng_FieldGather) guard - // cells from Efield_aux and filling up to ng (=nGrow) guard cells in the dE MultiFabs - ablastr::utils::communication::ParallelCopy(dEx, *Efield_aux[lev - 1][0], 0, 0, - Efield_aux[lev - 1][0]->nComp(), ng_src, ng, - WarpX::do_single_precision_comms, - crse_period); - ablastr::utils::communication::ParallelCopy(dEy, *Efield_aux[lev - 1][1], 0, 0, - Efield_aux[lev - 1][1]->nComp(), ng_src, ng, - WarpX::do_single_precision_comms, - crse_period); - ablastr::utils::communication::ParallelCopy(dEz, *Efield_aux[lev - 1][2], 0, 0, - Efield_aux[lev - 1][2]->nComp(), ng_src, ng, - WarpX::do_single_precision_comms, - crse_period); - - if (Efield_cax[lev][0]) + if (electromagnetic_solver_id != ElectromagneticSolverAlgo::None) { - MultiFab::Copy(*Efield_cax[lev][0], dEx, 0, 0, Efield_cax[lev][0]->nComp(), ng); - MultiFab::Copy(*Efield_cax[lev][1], dEy, 0, 0, Efield_cax[lev][1]->nComp(), ng); - MultiFab::Copy(*Efield_cax[lev][2], dEz, 0, 0, Efield_cax[lev][2]->nComp(), ng); - } - MultiFab::Subtract(dEx, *Efield_cp[lev][0], 0, 0, Efield_cp[lev][0]->nComp(), ng); - MultiFab::Subtract(dEy, *Efield_cp[lev][1], 0, 0, Efield_cp[lev][1]->nComp(), ng); - MultiFab::Subtract(dEz, *Efield_cp[lev][2], 0, 0, Efield_cp[lev][2]->nComp(), ng); + MultiFab dEx(Efield_cp[lev][0]->boxArray(), dm, Efield_cp[lev][0]->nComp(), ng); + MultiFab dEy(Efield_cp[lev][1]->boxArray(), dm, Efield_cp[lev][1]->nComp(), ng); + MultiFab dEz(Efield_cp[lev][2]->boxArray(), dm, Efield_cp[lev][2]->nComp(), ng); + dEx.setVal(0.0); + dEy.setVal(0.0); + dEz.setVal(0.0); + + // Copy Efield_aux to the dE MultiFabs, using up to ng_src (=ng_FieldGather) guard + // cells from Efield_aux and filling up to ng (=nGrow) guard cells in the dE MultiFabs + ablastr::utils::communication::ParallelCopy(dEx, *Efield_aux[lev - 1][0], 0, 0, + Efield_aux[lev - 1][0]->nComp(), ng_src, ng, + WarpX::do_single_precision_comms, + crse_period); + ablastr::utils::communication::ParallelCopy(dEy, *Efield_aux[lev - 1][1], 0, 0, + Efield_aux[lev - 1][1]->nComp(), ng_src, ng, + WarpX::do_single_precision_comms, + crse_period); + ablastr::utils::communication::ParallelCopy(dEz, *Efield_aux[lev - 1][2], 0, 0, + Efield_aux[lev - 1][2]->nComp(), ng_src, ng, + WarpX::do_single_precision_comms, + crse_period); + + if (Efield_cax[lev][0]) + { + MultiFab::Copy(*Efield_cax[lev][0], dEx, 0, 0, Efield_cax[lev][0]->nComp(), ng); + MultiFab::Copy(*Efield_cax[lev][1], dEy, 0, 0, Efield_cax[lev][1]->nComp(), ng); + MultiFab::Copy(*Efield_cax[lev][2], dEz, 0, 0, Efield_cax[lev][2]->nComp(), ng); + } + MultiFab::Subtract(dEx, *Efield_cp[lev][0], 0, 0, Efield_cp[lev][0]->nComp(), ng); + MultiFab::Subtract(dEy, *Efield_cp[lev][1], 0, 0, Efield_cp[lev][1]->nComp(), ng); + MultiFab::Subtract(dEz, *Efield_cp[lev][2], 0, 0, Efield_cp[lev][2]->nComp(), ng); - const amrex::IntVect& refinement_ratio = refRatio(lev-1); + const amrex::IntVect& refinement_ratio = refRatio(lev-1); - const amrex::IntVect& Ex_stag = Efield_aux[lev-1][0]->ixType().toIntVect(); - const amrex::IntVect& Ey_stag = Efield_aux[lev-1][1]->ixType().toIntVect(); - const amrex::IntVect& Ez_stag = Efield_aux[lev-1][2]->ixType().toIntVect(); + const amrex::IntVect& Ex_stag = Efield_aux[lev-1][0]->ixType().toIntVect(); + const amrex::IntVect& Ey_stag = Efield_aux[lev-1][1]->ixType().toIntVect(); + const amrex::IntVect& Ez_stag = Efield_aux[lev-1][2]->ixType().toIntVect(); #ifdef AMREX_USE_OMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(*Efield_aux[lev][0]); mfi.isValid(); ++mfi) - { - Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); - Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); - Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); - Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); - Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); - Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); - Array4 const& ex_c = dEx.const_array(mfi); - Array4 const& ey_c = dEy.const_array(mfi); - Array4 const& ez_c = dEz.const_array(mfi); - - amrex::ParallelFor(Box(ex_aux), Box(ey_aux), Box(ez_aux), - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept - { - warpx_interp(j, k, l, ex_aux, ex_fp, ex_c, Ex_stag, refinement_ratio); - }, - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + for (MFIter mfi(*Efield_aux[lev][0]); mfi.isValid(); ++mfi) { - warpx_interp(j, k, l, ey_aux, ey_fp, ey_c, Ey_stag, refinement_ratio); - }, - [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept - { - warpx_interp(j, k, l, ez_aux, ez_fp, ez_c, Ez_stag, refinement_ratio); - }); + Array4 const& ex_aux = Efield_aux[lev][0]->array(mfi); + Array4 const& ey_aux = Efield_aux[lev][1]->array(mfi); + Array4 const& ez_aux = Efield_aux[lev][2]->array(mfi); + Array4 const& ex_fp = Efield_fp[lev][0]->const_array(mfi); + Array4 const& ey_fp = Efield_fp[lev][1]->const_array(mfi); + Array4 const& ez_fp = Efield_fp[lev][2]->const_array(mfi); + Array4 const& ex_c = dEx.const_array(mfi); + Array4 const& ey_c = dEy.const_array(mfi); + Array4 const& ez_c = dEz.const_array(mfi); + + amrex::ParallelFor(Box(ex_aux), Box(ey_aux), Box(ez_aux), + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, ex_aux, ex_fp, ex_c, Ex_stag, refinement_ratio); + }, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, ey_aux, ey_fp, ey_c, Ey_stag, refinement_ratio); + }, + [=] AMREX_GPU_DEVICE (int j, int k, int l) noexcept + { + warpx_interp(j, k, l, ez_aux, ez_fp, ez_c, Ez_stag, refinement_ratio); + }); + } + } + else // electrostatic + { + MultiFab::Copy(*Efield_aux[lev][0], *Efield_fp[lev][0], 0, 0, Efield_aux[lev][0]->nComp(), Efield_aux[lev][0]->nGrowVect()); + MultiFab::Copy(*Efield_aux[lev][1], *Efield_fp[lev][1], 0, 0, Efield_aux[lev][1]->nComp(), Efield_aux[lev][1]->nGrowVect()); + MultiFab::Copy(*Efield_aux[lev][2], *Efield_fp[lev][2], 0, 0, Efield_aux[lev][2]->nComp(), Efield_aux[lev][2]->nGrowVect()); } } } @@ -584,7 +692,7 @@ WarpX::FillBoundaryE (const int lev, const PatchType patch_type, const amrex::In pml[lev]->FillBoundaryE(patch_type, nodal_sync); } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { pml_rz[lev]->FillBoundaryE(patch_type, nodal_sync); @@ -641,7 +749,7 @@ WarpX::FillBoundaryB (const int lev, const PatchType patch_type, const amrex::In pml[lev]->FillBoundaryB(patch_type, nodal_sync); } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev]) { pml_rz[lev]->FillBoundaryB(patch_type, nodal_sync); diff --git a/Source/Parallelization/WarpXComm_K.H b/Source/Parallelization/WarpXComm_K.H index a2b8fe38ed4..c3362087ad9 100644 --- a/Source/Parallelization/WarpXComm_K.H +++ b/Source/Parallelization/WarpXComm_K.H @@ -12,7 +12,7 @@ /** * \brief Interpolation function called within WarpX::UpdateAuxilaryDataSameType - * to interpolate data from the coarse and fine grids to the fine aux grid, + * with electromagnetic solver to interpolate data from the coarse and fine grids to the fine aux grid, * assuming that all grids have the same staggering (either collocated or staggered). * * \param[in] j index along x of the output array @@ -285,6 +285,94 @@ void warpx_interp (int j, int k, int l, // Final result arr_aux(j,k,l) = tmp + (fine - coarse); } +/** + * \brief Interpolation function called within WarpX::UpdateAuxilaryDataStagToNodal + * to interpolate data from the coarse and fine grids to the fine aux grid, + * with momentum-conserving field gathering, hence between grids with different staggering, + * and assuming that the aux grid is collocated. + * + * \param[in] j index along x of the output array + * \param[in] k index along y (in 3D) or z (in 2D) of the output array + * \param[in] l index along z (in 3D, l=0 in 2D) of the output array + * \param[in,out] arr_aux output array where interpolated values are stored + * \param[in] arr_fine input fine-patch array storing the values to interpolate + * \param[in] arr_fine_stag IndexType of the fine-patch arrays + */ +AMREX_GPU_DEVICE AMREX_FORCE_INLINE +void warpx_interp (int j, int k, int l, + amrex::Array4 const& arr_aux, + amrex::Array4 const& arr_fine, + const amrex::IntVect& arr_fine_stag) +{ + using namespace amrex; + + // Pad input arrays with zeros beyond ghost cells + // for out-of-bound accesses due to large-stencil operations + const auto arr_fine_zeropad = [arr_fine] (const int jj, const int kk, const int ll) noexcept + { + return arr_fine.contains(jj,kk,ll) ? arr_fine(jj,kk,ll) : 0.0_rt; + }; + + // NOTE Indices (j,k,l) in the following refer to: + // - (z,-,-) in 1D + // - (x,z,-) in 2D + // - (r,z,-) in RZ + // - (x,y,z) in 3D + + // Staggering of fine array (0: cell-centered; 1: nodal) + const int sj_fp = arr_fine_stag[0]; +#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const int sk_fp = arr_fine_stag[1]; +#elif defined(WARPX_DIM_3D) + const int sk_fp = arr_fine_stag[1]; + const int sl_fp = arr_fine_stag[2]; +#endif + + // Number of points used for interpolation from coarse grid to fine grid + int nj; + int nk; + int nl; + + amrex::Real fine = 0.0_rt; + + // 3) Interpolation from fine staggered to fine nodal + + nj = (sj_fp == 0) ? 2 : 1; +#if defined(WARPX_DIM_1D_Z) + nk = 1; + nl = 1; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + nk = (sk_fp == 0) ? 2 : 1; + nl = 1; +#else + nk = (sk_fp == 0) ? 2 : 1; + nl = (sl_fp == 0) ? 2 : 1; +#endif + + const int jm = (sj_fp == 0) ? j-1 : j; +#if defined(WARPX_DIM_1D_Z) + const int km = k; + const int lm = l; +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) + const int km = (sk_fp == 0) ? k-1 : k; + const int lm = l; +#else + const int km = (sk_fp == 0) ? k-1 : k; + const int lm = (sl_fp == 0) ? l-1 : l; +#endif + + for (int jj = 0; jj < nj; jj++) { + for (int kk = 0; kk < nk; kk++) { + for (int ll = 0; ll < nl; ll++) { + fine += arr_fine_zeropad(jm+jj,km+kk,lm+ll); + } + } + } + fine = fine/static_cast(nj*nk*nl); + + // Final result + arr_aux(j,k,l) = fine; +} /** * \brief Arbitrary-order interpolation function used to center a given MultiFab between two grids diff --git a/Source/Parallelization/WarpXRegrid.cpp b/Source/Parallelization/WarpXRegrid.cpp index b2a7004a1af..58df62974bd 100644 --- a/Source/Parallelization/WarpXRegrid.cpp +++ b/Source/Parallelization/WarpXRegrid.cpp @@ -166,22 +166,19 @@ WarpX::LoadBalance () #endif } - -template void -RemakeMultiFab (std::unique_ptr& mf, const DistributionMapping& dm, - const bool redistribute, const int lev) -{ - if (mf == nullptr) { return; } - const IntVect& ng = mf->nGrowVect(); - std::unique_ptr pmf; - WarpX::AllocInitMultiFab(pmf, mf->boxArray(), dm, mf->nComp(), ng, lev, mf->tags()[0]); - if (redistribute) { pmf->Redistribute(*mf, 0, 0, mf->nComp(), ng); } - mf = std::move(pmf); -} - void WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const DistributionMapping& dm) { + + const auto RemakeMultiFab = [&](auto& mf, const bool redistribute){ + if (mf == nullptr) { return; } + const IntVect& ng = mf->nGrowVect(); + auto pmf = std::remove_reference_t{}; + AllocInitMultiFab(pmf, mf->boxArray(), dm, mf->nComp(), ng, lev, mf->tags()[0]); + if (redistribute) { pmf->Redistribute(*mf, 0, 0, mf->nComp(), ng); } + mf = std::move(pmf); + }; + if (ba == boxArray(lev)) { if (ParallelDescriptor::NProcs() == 1) { return; } @@ -189,60 +186,66 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi // Fine patch for (int idim=0; idim < 3; ++idim) { - RemakeMultiFab(Bfield_fp[lev][idim], dm, true ,lev); - RemakeMultiFab(Efield_fp[lev][idim], dm, true ,lev); + RemakeMultiFab(Bfield_fp[lev][idim], true); + RemakeMultiFab(Efield_fp[lev][idim], true); if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { - RemakeMultiFab(Bfield_fp_external[lev][idim], dm, true ,lev); + RemakeMultiFab(Bfield_fp_external[lev][idim], true); } if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { - RemakeMultiFab(Efield_fp_external[lev][idim], dm, true ,lev); + RemakeMultiFab(Efield_fp_external[lev][idim], true); + } + if (mypc->m_B_ext_particle_s == "read_from_file") { + RemakeMultiFab(B_external_particle_field[lev][idim], true); + } + if (mypc->m_E_ext_particle_s == "read_from_file") { + RemakeMultiFab(E_external_particle_field[lev][idim], true); } - RemakeMultiFab(current_fp[lev][idim], dm, false ,lev); - RemakeMultiFab(current_store[lev][idim], dm, false ,lev); + RemakeMultiFab(current_fp[lev][idim], false); + RemakeMultiFab(current_store[lev][idim], false); if (current_deposition_algo == CurrentDepositionAlgo::Vay) { - RemakeMultiFab(current_fp_vay[lev][idim], dm, false ,lev); + RemakeMultiFab(current_fp_vay[lev][idim], false); } if (do_current_centering) { - RemakeMultiFab(current_fp_nodal[lev][idim], dm, false ,lev); + RemakeMultiFab(current_fp_nodal[lev][idim], false); } if (fft_do_time_averaging) { - RemakeMultiFab(Efield_avg_fp[lev][idim], dm, true ,lev); - RemakeMultiFab(Bfield_avg_fp[lev][idim], dm, true ,lev); + RemakeMultiFab(Efield_avg_fp[lev][idim], true); + RemakeMultiFab(Bfield_avg_fp[lev][idim], true); } if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { - RemakeMultiFab(m_hybrid_pic_model->current_fp_temp[lev][idim], dm, true, lev); - RemakeMultiFab(m_hybrid_pic_model->current_fp_ampere[lev][idim], dm, false, lev); - RemakeMultiFab(m_hybrid_pic_model->current_fp_external[lev][idim], dm, true, lev); + RemakeMultiFab(m_hybrid_pic_model->current_fp_temp[lev][idim], true); + RemakeMultiFab(m_hybrid_pic_model->current_fp_ampere[lev][idim], false); + RemakeMultiFab(m_hybrid_pic_model->current_fp_external[lev][idim],true); } #ifdef AMREX_USE_EB if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) { - RemakeMultiFab(m_edge_lengths[lev][idim], dm, false ,lev); - RemakeMultiFab(m_face_areas[lev][idim], dm, false ,lev); + RemakeMultiFab(m_edge_lengths[lev][idim], false); + RemakeMultiFab(m_face_areas[lev][idim], false); if(WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT){ - RemakeMultiFab(Venl[lev][idim], dm, false ,lev); - RemakeMultiFab(m_flag_info_face[lev][idim], dm, false ,lev); - RemakeMultiFab(m_flag_ext_face[lev][idim], dm, false ,lev); - RemakeMultiFab(m_area_mod[lev][idim], dm, false ,lev); - RemakeMultiFab(ECTRhofield[lev][idim], dm, false ,lev); + RemakeMultiFab(Venl[lev][idim], false); + RemakeMultiFab(m_flag_info_face[lev][idim], false); + RemakeMultiFab(m_flag_ext_face[lev][idim], false); + RemakeMultiFab(m_area_mod[lev][idim], false); + RemakeMultiFab(ECTRhofield[lev][idim], false); m_borrowing[lev][idim] = std::make_unique>(amrex::convert(ba, Bfield_fp[lev][idim]->ixType().toIntVect()), dm); } } #endif } - RemakeMultiFab(F_fp[lev], dm, true ,lev); - RemakeMultiFab(rho_fp[lev], dm, false ,lev); + RemakeMultiFab(F_fp[lev], true); + RemakeMultiFab(rho_fp[lev], false); // phi_fp should be redistributed since we use the solution from // the last step as the initial guess for the next solve - RemakeMultiFab(phi_fp[lev], dm, true ,lev); + RemakeMultiFab(phi_fp[lev], true); if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { - RemakeMultiFab(m_hybrid_pic_model->rho_fp_temp[lev], dm, true, lev); - RemakeMultiFab(m_hybrid_pic_model->electron_pressure_fp[lev], dm, false, lev); + RemakeMultiFab(m_hybrid_pic_model->rho_fp_temp[lev], true); + RemakeMultiFab(m_hybrid_pic_model->electron_pressure_fp[lev], false); } #ifdef AMREX_USE_EB - RemakeMultiFab(m_distance_to_eb[lev], dm, false ,lev); + RemakeMultiFab(m_distance_to_eb[lev], false); int max_guard = guard_cells.ng_FieldSolver.max(); m_field_factory[lev] = amrex::makeEBFabFactory(Geom(lev), ba, dm, @@ -254,7 +257,7 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi m_field_factory[lev] = std::make_unique(); #endif -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { if (spectral_solver_fp[lev] != nullptr) { // Get the cell-centered box @@ -298,8 +301,8 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi } else { for (int idim=0; idim < 3; ++idim) { - RemakeMultiFab(Bfield_aux[lev][idim], dm, false ,lev); - RemakeMultiFab(Efield_aux[lev][idim], dm, false ,lev); + RemakeMultiFab(Bfield_aux[lev][idim], false); + RemakeMultiFab(Efield_aux[lev][idim], false); } } @@ -307,18 +310,18 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi if (lev > 0) { for (int idim=0; idim < 3; ++idim) { - RemakeMultiFab(Bfield_cp[lev][idim], dm, true ,lev); - RemakeMultiFab(Efield_cp[lev][idim], dm, true ,lev); - RemakeMultiFab(current_cp[lev][idim], dm, false ,lev); + RemakeMultiFab(Bfield_cp[lev][idim], true); + RemakeMultiFab(Efield_cp[lev][idim], true); + RemakeMultiFab(current_cp[lev][idim], false); if (fft_do_time_averaging) { - RemakeMultiFab(Efield_avg_cp[lev][idim], dm, true ,lev); - RemakeMultiFab(Bfield_avg_cp[lev][idim], dm, true ,lev); + RemakeMultiFab(Efield_avg_cp[lev][idim], true); + RemakeMultiFab(Bfield_avg_cp[lev][idim], true); } } - RemakeMultiFab(F_cp[lev], dm, true ,lev); - RemakeMultiFab(rho_cp[lev], dm, false ,lev); + RemakeMultiFab(F_cp[lev], true); + RemakeMultiFab(rho_cp[lev], false); -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { if (spectral_solver_cp[lev] != nullptr) { BoxArray cba = ba; @@ -356,14 +359,14 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi if (lev > 0 && (n_field_gather_buffer > 0 || n_current_deposition_buffer > 0)) { for (int idim=0; idim < 3; ++idim) { - RemakeMultiFab(Bfield_cax[lev][idim], dm, false ,lev); - RemakeMultiFab(Efield_cax[lev][idim], dm, false ,lev); - RemakeMultiFab(current_buf[lev][idim], dm, false ,lev); + RemakeMultiFab(Bfield_cax[lev][idim], false); + RemakeMultiFab(Efield_cax[lev][idim], false); + RemakeMultiFab(current_buf[lev][idim], false); } - RemakeMultiFab(charge_buf[lev], dm, false ,lev); + RemakeMultiFab(charge_buf[lev], false); // we can avoid redistributing these since we immediately re-build the values via BuildBufferMasks() - RemakeMultiFab(current_buffer_masks[lev], dm, false ,lev); - RemakeMultiFab(gather_buffer_masks[lev], dm, false ,lev); + RemakeMultiFab(current_buffer_masks[lev], false); + RemakeMultiFab(gather_buffer_masks[lev], false); if (current_buffer_masks[lev] || gather_buffer_masks[lev]) { BuildBufferMasks(); diff --git a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp index 1eb89ff4c0f..80ce13744fd 100644 --- a/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp +++ b/Source/Particles/Collision/BackgroundMCC/BackgroundMCCCollision.cpp @@ -234,9 +234,12 @@ BackgroundMCCCollision::doCollisions (amrex::Real cur_time, amrex::Real dt, Mult // dt has to be small enough that a linear expansion of the collision // probability is sufficiently accurately, otherwise the MCC results // will be very heavily affected by small changes in the timestep - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(coll_n < 0.1_prt, - "dt is too large to ensure accurate MCC results" - ); + if (coll_n > 0.1_prt) { + ablastr::warn_manager::WMRecordWarning("BackgroundMCC Collisions", + "dt is too large to ensure accurate MCC results , coll_n: " + + std::to_string(coll_n) + " is > 0.1 and collision probability is = " + + std::to_string(m_total_collision_prob) + "\n"); + } if (ionization_flag) { // calculate maximum collision frequency for ionization @@ -246,9 +249,12 @@ BackgroundMCCCollision::doCollisions (amrex::Real cur_time, amrex::Real dt, Mult auto coll_n_ioniz = m_nu_max_ioniz * dt; m_total_collision_prob_ioniz = 1.0_prt - std::exp(-coll_n_ioniz); - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(coll_n_ioniz < 0.1_prt, - "dt is too large to ensure accurate MCC results" - ); + if (coll_n_ioniz > 0.1_prt) { + ablastr::warn_manager::WMRecordWarning("BackgroundMCC Collisions", + "dt is too large to ensure accurate MCC ionization , coll_n_ionization: " + + std::to_string(coll_n_ioniz) + " is > 0.1 and ionization probability is = " + + std::to_string(m_total_collision_prob_ioniz) + "\n"); + } // if an ionization process is included the secondary species mass // is taken as the background mass diff --git a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H index b3d39a9c581..6f8f61e66b1 100644 --- a/Source/Particles/Collision/BinaryCollision/BinaryCollision.H +++ b/Source/Particles/Collision/BinaryCollision/BinaryCollision.H @@ -8,6 +8,7 @@ #define WARPX_PARTICLES_COLLISION_BINARYCOLLISION_H_ #include "Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H" +#include "Particles/Collision/BinaryCollision/Coulomb/ComputeTemperature.H" #include "Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H" #include "Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H" #include "Particles/Collision/BinaryCollision/ParticleCreationFunc.H" @@ -108,7 +109,7 @@ public: } m_have_product_species = !m_product_species.empty(); - if ((std::is_same::value) && (m_have_product_species)) { + if ((std::is_same_v) && (m_have_product_species)) { WARPX_ABORT_WITH_MESSAGE( "Binary collision " + collision_name + " does not produce species. Thus, `product_species` should not be specified in the input script." ); } @@ -360,6 +361,18 @@ public: End of calculations only required when creating product particles */ + // create vectors to store density and temperature on cell level + amrex::Gpu::DeviceVector n1_vec; + amrex::Gpu::DeviceVector T1_vec; + if (binary_collision_functor.m_computeSpeciesDensities) { + n1_vec.resize(n_cells); + } + if (binary_collision_functor.m_computeSpeciesTemperatures) { + T1_vec.resize(n_cells); + } + amrex::ParticleReal* AMREX_RESTRICT n1_in_each_cell = n1_vec.dataPtr(); + amrex::ParticleReal* AMREX_RESTRICT T1_in_each_cell = T1_vec.dataPtr(); + // Loop over cells amrex::ParallelForRNG( n_cells, [=] AMREX_GPU_DEVICE (int i_cell, amrex::RandomEngine const& engine) noexcept @@ -368,14 +381,36 @@ public: // given by the `indices_1[cell_start_1:cell_stop_1]` index_type const cell_start_1 = cell_offsets_1[i_cell]; index_type const cell_stop_1 = cell_offsets_1[i_cell+1]; - index_type const cell_half_1 = (cell_start_1+cell_stop_1)/2; // Do not collide if there is only one particle in the cell if ( cell_stop_1 - cell_start_1 <= 1 ) { return; } + // compute local density [1/m^3] + if (binary_collision_functor.m_computeSpeciesDensities) { + amrex::ParticleReal wtot1 = 0.0; + amrex::ParticleReal * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; + for (index_type i1=cell_start_1; i1 n1_vec, n2_vec; + amrex::Gpu::DeviceVector T1_vec, T2_vec; + if (binary_collision_functor.m_computeSpeciesDensities) { + n1_vec.resize(n_cells); + n2_vec.resize(n_cells); + } + if (binary_collision_functor.m_computeSpeciesTemperatures) { + T1_vec.resize(n_cells); + T2_vec.resize(n_cells); + } + amrex::ParticleReal* AMREX_RESTRICT n1_in_each_cell = n1_vec.dataPtr(); + amrex::ParticleReal* AMREX_RESTRICT n2_in_each_cell = n2_vec.dataPtr(); + amrex::ParticleReal* AMREX_RESTRICT T1_in_each_cell = T1_vec.dataPtr(); + amrex::ParticleReal* AMREX_RESTRICT T2_in_each_cell = T2_vec.dataPtr(); // Loop over cells amrex::ParallelForRNG( n_cells, @@ -581,6 +643,43 @@ public: // ux_1[ indices_1[i] ], where i is between // cell_start_1 (inclusive) and cell_start_2 (exclusive) + // compute local densities [1/m^3] + if (binary_collision_functor.m_computeSpeciesDensities) { + amrex::ParticleReal w1tot = 0.0; + amrex::ParticleReal w2tot = 0.0; + amrex::ParticleReal * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; + amrex::ParticleReal * const AMREX_RESTRICT w2 = soa_2.m_rdata[PIdx::w]; + for (index_type i1=cell_start_1; i1 AMREX_GPU_HOST_DEVICE T_R ComputeTemperature ( T_index const Is, T_index const Ie, T_index const * AMREX_RESTRICT I, + T_R const * AMREX_RESTRICT w, T_R const * AMREX_RESTRICT ux, T_R const * AMREX_RESTRICT uy, T_R const * AMREX_RESTRICT uz, T_R const m ) { @@ -26,6 +27,7 @@ T_R ComputeTemperature ( T_R vx = T_R(0.0); T_R vy = T_R(0.0); T_R vz = T_R(0.0); T_R vs = T_R(0.0); T_R gm = T_R(0.0); T_R us = T_R(0.0); + T_R wtot = T_R(0.0); for (int i = Is; i < static_cast(Ie); ++i) { @@ -33,14 +35,15 @@ T_R ComputeTemperature ( uy[ I[i] ] * uy[ I[i] ] + uz[ I[i] ] * uz[ I[i] ] ); gm = std::sqrt( T_R(1.0) + us*inv_c2 ); - vx += ux[ I[i] ] / gm; - vy += uy[ I[i] ] / gm; - vz += uz[ I[i] ] / gm; - vs += us / gm / gm; + wtot += w[ I[i] ]; + vx += w[ I[i] ] * ux[ I[i] ] / gm; + vy += w[ I[i] ] * uy[ I[i] ] / gm; + vz += w[ I[i] ] * uz[ I[i] ] / gm; + vs += w[ I[i] ] * us / gm / gm; } - vx = vx / N; vy = vy / N; - vz = vz / N; vs = vs / N; + vx = vx / wtot; vy = vy / wtot; + vz = vz / wtot; vs = vs / wtot; return m/T_R(3.0)*(vs-(vx*vx+vy*vy+vz*vz)); } diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H index e407ae1603b..d418b7dc92d 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/ElasticCollisionPerez.H @@ -7,7 +7,6 @@ #ifndef WARPX_PARTICLES_COLLISION_ELASTIC_COLLISION_PEREZ_H_ #define WARPX_PARTICLES_COLLISION_ELASTIC_COLLISION_PEREZ_H_ -#include "ComputeTemperature.H" #include "UpdateMomentumPerezElastic.H" #include "Particles/WarpXParticleContainer.H" #include "Utils/WarpXConst.H" @@ -25,18 +24,15 @@ * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). * @param[in] I1,I2 the index arrays. They determine all elements that will be used. * @param[in,out] soa_1,soa_2 the struct of array for species 1/2 + * @param[in] n1,n2 density of species 1/2 + * @param[in] T1,T2 temperature [Joules] of species 1/2 + * only used if L <= 0 * @param[in] q1,q2 charge of species 1/2 * @param[in] m1,m2 mass of species 1/2 - * @param[in] T1 temperature (Joule) of species 1 - * and will be used if greater than zero, - * otherwise will be computed. - * @param[in] T2 temperature (Joule) of species 2, @see T1 * @param[in] dt is the time step length between two collision calls. * @param[in] L is the Coulomb log and will be used if greater than zero, * otherwise will be computed. - * @param[in] dV is the volume of the corresponding cell. * @param[in] engine the random number generator state & factory - * @param[in] isSameSpecies whether this is an intra-species collision process * @param[in] coll_idx is the collision index offset. */ @@ -48,17 +44,18 @@ void ElasticCollisionPerez ( T_index const* AMREX_RESTRICT I1, T_index const* AMREX_RESTRICT I2, SoaData_type soa_1, SoaData_type soa_2, + T_PR const n1, T_PR const n2, + T_PR const T1, T_PR const T2, T_PR const q1, T_PR const q2, T_PR const m1, T_PR const m2, - T_PR const T1, T_PR const T2, - T_R const dt, T_PR const L, T_R const dV, + T_R const dt, T_PR const L, T_R const dV, amrex::RandomEngine const& engine, bool const isSameSpecies, T_index coll_idx) { const T_index NI1 = I1e - I1s; const T_index NI2 = I2e - I2s; const T_index max_N = amrex::max(NI1,NI2); - const T_index min_N = amrex::max(NI1,NI2); + const T_index min_N = amrex::min(NI1,NI2); T_PR * const AMREX_RESTRICT w1 = soa_1.m_rdata[PIdx::w]; T_PR * const AMREX_RESTRICT u1x = soa_1.m_rdata[PIdx::ux]; @@ -70,57 +67,19 @@ void ElasticCollisionPerez ( T_PR * const AMREX_RESTRICT u2y = soa_2.m_rdata[PIdx::uy]; T_PR * const AMREX_RESTRICT u2z = soa_2.m_rdata[PIdx::uz]; - // get local T1t and T2t - T_PR T1t; T_PR T2t; - if ( T1 <= T_PR(0.0) && L <= T_PR(0.0) ) - { - T1t = ComputeTemperature(I1s,I1e,I1,u1x,u1y,u1z,m1); + // compute Debye length lmdD (if not using a fixed L = Coulomb log) + T_PR lmdD = T_PR(-1.0); + if ( L <= T_PR(0.0) ) { + lmdD = T_PR(1.0)/std::sqrt( n1*q1*q1/(T1*PhysConst::ep0) + + n2*q2*q2/(T2*PhysConst::ep0) ); } - else { T1t = T1; } - if ( T2 <= T_PR(0.0) && L <= T_PR(0.0) ) - { - T2t = ComputeTemperature(I2s,I2e,I2,u2x,u2y,u2z,m2); - } - else { T2t = T2; } - - // local density - T_PR n1 = T_PR(0.0); - T_PR n2 = T_PR(0.0); - T_PR n12 = T_PR(0.0); - for (T_index i1=I1s; i1( 1.0/std::cbrt(4.0*MathConst::pi/3.0*maxn) ); + + // bmax (screening length) cannot be smaller than atomic spacing + const T_PR bmax = amrex::max(lmdD, rmin); #if (defined WARPX_DIM_RZ) T_PR * const AMREX_RESTRICT theta1 = soa_1.m_rdata[PIdx::theta]; @@ -151,12 +110,23 @@ void ElasticCollisionPerez ( u1y[I1[i1]] = u1xbuf*std::sin(theta) + u1y[I1[i1]]*std::cos(theta); #endif + // Compute the effective density n12 used to compute the normalized + // scattering path s12 in UpdateMomentumPerezElastic(). + // s12 is defined such that the expected value of the change in particle + // velocity is equal to that from the full NxN pairing method, as described + // here https://arxiv.org/submit/5758216/view. This method is a direct extension + // of the original method by Takizuka and Abe JCP 25 (1977) to weighted particles. + T_PR n12; + const T_PR wpmax = amrex::max(w1[ I1[i1] ],w2[ I2[i2] ]); + if (isSameSpecies) { n12 = wpmax*static_cast(min_N+max_N-1)/dV; } + else { n12 = wpmax*static_cast(min_N)/dV; } + UpdateMomentumPerezElastic( u1x[ I1[i1] ], u1y[ I1[i1] ], u1z[ I1[i1] ], u2x[ I2[i2] ], u2y[ I2[i2] ], u2z[ I2[i2] ], n1, n2, n12, q1, m1, w1[ I1[i1] ], q2, m2, w2[ I2[i2] ], - dt, L, lmdD, + dt, L, bmax, engine); #if (defined WARPX_DIM_RZ) diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H index 26782c2e42a..3322c19ed2d 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/PairWiseCoulombCollisionFunc.H @@ -60,6 +60,7 @@ public: m_CoulombLog = CoulombLog; m_exe.m_CoulombLog = m_CoulombLog; + if (m_CoulombLog<0.0) { m_exe.m_computeSpeciesTemperatures = true; } m_exe.m_isSameSpecies = m_isSameSpecies; } @@ -72,6 +73,8 @@ public: * @param[in] I1e,I2e is the stop index for I1,I2 (exclusive). * @param[in] I1,I2 index arrays. They determine all elements that will be used. * @param[in,out] soa_1,soa_2 contain the struct of array data of the two species. + * @param[in] n1,n2 are local densities. + * @param[in] T1,T2 are local temperatures. * @param[in] q1,q2 are charges. * @param[in] m1,m2 are masses. * @param[in] dt is the time step length between two collision calls. @@ -87,6 +90,8 @@ public: index_type const* AMREX_RESTRICT I2, const SoaData_type& soa_1, const SoaData_type& soa_2, GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, + amrex::ParticleReal const n1, amrex::ParticleReal const n2, + amrex::ParticleReal const T1, amrex::ParticleReal const T2, amrex::ParticleReal const q1, amrex::ParticleReal const q2, amrex::ParticleReal const m1, amrex::ParticleReal const m2, amrex::Real const dt, amrex::Real const dV, index_type coll_idx, @@ -99,12 +104,14 @@ public: ElasticCollisionPerez( I1s, I1e, I2s, I2e, I1, I2, - soa_1, soa_2, - q1, q2, m1, m2, -1.0_prt, -1.0_prt, + soa_1, soa_2, n1, n2, T1, T2, + q1, q2, m1, m2, dt, m_CoulombLog, dV, engine, m_isSameSpecies, coll_idx); } amrex::ParticleReal m_CoulombLog; + bool m_computeSpeciesDensities = true; + bool m_computeSpeciesTemperatures = false; bool m_isSameSpecies; }; diff --git a/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H b/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H index 3101047e211..93acc3d8aef 100644 --- a/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H +++ b/Source/Particles/Collision/BinaryCollision/Coulomb/UpdateMomentumPerezElastic.H @@ -18,9 +18,10 @@ /* \brief Update particle velocities according to * F. Perez et al., Phys.Plasmas.19.083104 (2012), * which is based on Nanbu's method, PhysRevE.55.4642 (1997). - * @param[in] LmdD is max(Debye length, minimal interparticle distance). + * @param[in] bmax is max(Debye length, minimal interparticle distance). * @param[in] L is the Coulomb log. A fixed L will be used if L > 0, * otherwise L will be calculated based on the algorithm. + * @param[in] n12 = max(w1,w2)*min(N1,N2)/dV is the effective density used for s12 * To see if there are nan or inf updated velocities, * compile with USE_ASSERTION=TRUE. * @@ -36,7 +37,7 @@ void UpdateMomentumPerezElastic ( T_PR const n1, T_PR const n2, T_PR const n12, T_PR const q1, T_PR const m1, T_PR const w1, T_PR const q2, T_PR const m2, T_PR const w2, - T_R const dt, T_PR const L, T_PR const lmdD, + T_R const dt, T_PR const L, T_PR const bmax, amrex::RandomEngine const& engine) { @@ -128,13 +129,13 @@ void UpdateMomentumPerezElastic ( // Compute the Coulomb log lnLmd lnLmd = amrex::max( T_PR(2.0), - T_PR(0.5)*std::log(T_PR(1.0)+lmdD*lmdD/(bmin*bmin)) ); + T_PR(0.5)*std::log(T_PR(1.0) + bmax*bmax/(bmin*bmin)) ); } // Compute s const auto tts = m1*g1s*m2*g2s/(inv_c2*p1sm*p1sm) + T_PR(1.0); const auto tts2 = tts*tts; - s = n1*n2/n12 * dt*lnLmd*q1*q1*q2*q2 / + s = n12 * dt*lnLmd*q1*q1*q2*q2 / ( T_PR(4.0) * MathConst::pi * PhysConst::ep0 * PhysConst::ep0 * m1*g1*m2*g2/(inv_c2*inv_c2) ) * gc*p1sm/mass_g * tts2; @@ -144,7 +145,7 @@ void UpdateMomentumPerezElastic ( const auto coeff = static_cast( std::pow(4.0*MathConst::pi/3.0,1.0/3.0)); T_PR const vrel = mass_g*p1sm/(m1*g1s*m2*g2s*gc); - T_PR const sp = coeff * n1*n2/n12 * dt * vrel * (m1+m2) / + T_PR const sp = coeff * n12 * dt * vrel * (m1+m2) / amrex::max( m1*cbrt_n1*cbrt_n1, m2*cbrt_n2*cbrt_n2); diff --git a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H index f28f3fb984c..30b466e2ec2 100644 --- a/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H +++ b/Source/Particles/Collision/BinaryCollision/DSMC/DSMCFunc.H @@ -96,6 +96,8 @@ public: index_type const* AMREX_RESTRICT I2, const SoaData_type& soa_1, const SoaData_type& soa_2, GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, + amrex::ParticleReal const /*n1*/, amrex::ParticleReal const /*n2*/, + amrex::ParticleReal const /*T1*/, amrex::ParticleReal const /*T2*/, amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, amrex::ParticleReal const m1, amrex::ParticleReal const m2, amrex::Real const dt, amrex::Real const dV, index_type coll_idx, @@ -189,6 +191,8 @@ public: } int m_process_count; + bool m_computeSpeciesDensities = false; + bool m_computeSpeciesTemperatures = false; ScatteringProcess::Executor* m_scattering_processes_data; }; diff --git a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H index b6d6fe171de..3113dc69839 100644 --- a/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H +++ b/Source/Particles/Collision/BinaryCollision/NuclearFusion/NuclearFusionFunc.H @@ -131,6 +131,8 @@ public: index_type const* AMREX_RESTRICT I2, const SoaData_type& soa_1, const SoaData_type& soa_2, GetParticlePosition /*get_position_1*/, GetParticlePosition /*get_position_2*/, + amrex::ParticleReal const /*n1*/, amrex::ParticleReal const /*n2*/, + amrex::ParticleReal const /*T1*/, amrex::ParticleReal const /*T2*/, amrex::ParticleReal const /*q1*/, amrex::ParticleReal const /*q2*/, amrex::ParticleReal const m1, amrex::ParticleReal const m2, amrex::Real const dt, amrex::Real const dV, index_type coll_idx, @@ -232,6 +234,8 @@ public: amrex::ParticleReal m_probability_threshold; amrex::ParticleReal m_probability_target_value; NuclearFusionType m_fusion_type; + bool m_computeSpeciesDensities = false; + bool m_computeSpeciesTemperatures = false; bool m_isSameSpecies; }; diff --git a/Source/Particles/Deposition/ChargeDeposition.H b/Source/Particles/Deposition/ChargeDeposition.H index 2912b769c57..d3d36dbfbda 100644 --- a/Source/Particles/Deposition/ChargeDeposition.H +++ b/Source/Particles/Deposition/ChargeDeposition.H @@ -28,8 +28,8 @@ ion_lev is a null pointer. * \param rho_fab FArrayBox of charge density, either full array or tile. * \param np_to_deposit Number of particles for which current is deposited. - * \param dx 3D cell size - * \param xyzmin Physical lower bounds of domain. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. @@ -40,37 +40,19 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, const int* ion_lev, amrex::FArrayBox& rho_fab, long np_to_deposit, - const std::array& dx, - const std::array xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes) + [[maybe_unused]] int n_rz_azimuthal_modes) { using namespace amrex; // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; - const amrex::Real dzi = 1.0_rt/dx[2]; -#if defined(WARPX_DIM_1D_Z) - const amrex::Real invvol = dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real invvol = dxi*dzi; -#elif defined(WARPX_DIM_3D) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real dyi = 1.0_rt/dx[1]; - const amrex::Real invvol = dxi*dyi*dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_3D) - const amrex::Real xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif - const amrex::Real zmin = xyzmin[2]; + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; amrex::Array4 const& rho_arr = rho_fab.array(); amrex::IntVect const rho_type = rho_fab.box().type(); @@ -98,19 +80,12 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, // Get particle position in grid coordinates #if defined(WARPX_DIM_RZ) const amrex::Real rp = std::sqrt(xp*xp + yp*yp); - amrex::Real costheta; - amrex::Real sintheta; - if (rp > 0.) { - costheta = xp/rp; - sintheta = yp/rp; - } else { - costheta = 1._rt; - sintheta = 0._rt; - } + const amrex::Real costheta = (rp > 0._rt ? xp/rp : 1._rt); + const amrex::Real sintheta = (rp > 0._rt ? yp/rp : 0._rt); const Complex xy0 = Complex{costheta, sintheta}; - const amrex::Real x = (rp - xmin)*dxi; + const amrex::Real x = (rp - xyzmin.x)*dinv.x; #else - const amrex::Real x = (xp - xmin)*dxi; + const amrex::Real x = (xp - xyzmin.x)*dinv.x; #endif // Compute shape factor along x @@ -125,7 +100,7 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, #endif //defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_3D) #if defined(WARPX_DIM_3D) // y direction - const amrex::Real y = (yp - ymin)*dyi; + const amrex::Real y = (yp - xyzmin.y)*dinv.y; amrex::Real sy[depos_order + 1] = {0._rt}; int j = 0; if (rho_type[1] == NODE) { @@ -135,7 +110,7 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, } #endif // z direction - const amrex::Real z = (zp - zmin)*dzi; + const amrex::Real z = (zp - xyzmin.z)*dinv.z; amrex::Real sz[depos_order + 1] = {0._rt}; int k = 0; if (rho_type[WARPX_ZINDEX] == NODE) { @@ -151,8 +126,7 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, &rho_arr(lo.x+k+iz, 0, 0, 0), sz[iz]*wq); } -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) for (int iz=0; iz<=depos_order; iz++){ for (int ix=0; ix<=depos_order; ix++){ amrex::Gpu::Atomic::AddNoRet( @@ -182,10 +156,6 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, #endif } ); - -#ifndef WARPX_DIM_RZ - amrex::ignore_unused(n_rz_azimuthal_modes); -#endif } /* \brief Perform charge deposition on a tile using shared memory @@ -198,8 +168,8 @@ void doChargeDepositionShapeN (const GetParticlePosition& GetPosition, * \param rho_fab FArrayBox of charge density, either full array or tile. * \param ix_type * \param np_to_deposit Number of particles for which charge is deposited. - * \param dx 3D cell size - * \param xyzmin Physical lower bounds of domain. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. @@ -216,11 +186,11 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio amrex::FArrayBox& rho_fab, const amrex::IntVect& ix_type, const long np_to_deposit, - const std::array& dx, - const std::array xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes, + [[maybe_unused]]const int n_rz_azimuthal_modes, const amrex::DenseBins& a_bins, const amrex::Box& box, const amrex::Geometry& geom, @@ -239,26 +209,8 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; - const amrex::Real dzi = 1.0_rt/dx[2]; -#if defined(WARPX_DIM_1D_Z) - const amrex::Real invvol = dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real invvol = dxi*dzi; -#elif defined(WARPX_DIM_3D) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real dyi = 1.0_rt/dx[1]; - const amrex::Real invvol = dxi*dyi*dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_3D) - const amrex::Real xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif - const amrex::Real zmin = xyzmin[2]; + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; amrex::Array4 const& rho_arr = rho_fab.array(); auto rho_box = rho_fab.box(); @@ -269,7 +221,6 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio // Loop over particles and deposit into rho_fab #if defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP) - const auto dxiarr = geom.InvCellSizeArray(); const auto plo = geom.ProbLoArray(); const auto domain = geom.Domain(); @@ -316,15 +267,14 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio ParticleReal xp, yp, zp; GetPosition(permutation[bin_start], xp, yp, zp); #if defined(WARPX_DIM_3D) - IntVect iv = IntVect(int(amrex::Math::floor((xp-plo[0])*dxiarr[0])), - int(amrex::Math::floor((yp-plo[1])*dxiarr[1])), - int(amrex::Math::floor((zp-plo[2])*dxiarr[2]))); + IntVect iv = IntVect(int(amrex::Math::floor((xp-plo[0])*dinv.x)), + int(amrex::Math::floor((yp-plo[1])*dinv.y)), + int(amrex::Math::floor((zp-plo[2])*dinv.z))); #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - IntVect iv = IntVect( - int(amrex::Math::floor((xp-plo[0])*dxiarr[0])), - int(amrex::Math::floor((zp-plo[1])*dxiarr[1]))); + IntVect iv = IntVect(int(amrex::Math::floor((xp-plo[0])*dinv.x)), + int(amrex::Math::floor((zp-plo[1])*dinv.z))); #elif defined(WARPX_DIM_1D_Z) - IntVect iv = IntVect(int(amrex::Math::floor((zp-plo[0])*dxiarr[0]))); + IntVect iv = IntVect(int(amrex::Math::floor((zp-plo[0])*dinv.z))); #endif iv += domain.smallEnd(); getTileIndex(iv, box, true, bin_size, buffer_box); @@ -381,9 +331,9 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio sintheta = 0._rt; } const Complex xy0 = Complex{costheta, sintheta}; - const amrex::Real x = (rp - xmin)*dxi; + const amrex::Real x = (rp - xyzmin.x)*dinv.x; #else - const amrex::Real x = (xp - xmin)*dxi; + const amrex::Real x = (xp - xyzmin.x)*dinv.x; #endif // Compute shape factor along x @@ -398,7 +348,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio #endif //defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) || defined(WARPX_DIM_3D) #if defined(WARPX_DIM_3D) // y direction - const amrex::Real y = (yp - ymin)*dyi; + const amrex::Real y = (yp - xyzmin.y)*dinv.y; amrex::Real sy[depos_order + 1] = {0._rt}; int j = 0; if (rho_type[1] == NODE) { @@ -408,7 +358,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio } #endif // z direction - const amrex::Real z = (zp - zmin)*dzi; + const amrex::Real z = (zp - xyzmin.z)*dinv.z; amrex::Real sz[depos_order + 1] = {0._rt}; int k = 0; if (rho_type[WARPX_ZINDEX] == NODE) { @@ -424,8 +374,7 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio &buf(lo.x+k+iz, 0, 0, 0), sz[iz]*wq); } -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) +#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) for (int iz=0; iz<=depos_order; iz++){ for (int ix=0; ix<=depos_order; ix++){ amrex::Gpu::Atomic::AddNoRet( @@ -462,10 +411,6 @@ void doChargeDepositionSharedShapeN (const GetParticlePosition& GetPositio #endif // defined(AMREX_USE_CUDA) || defined(AMREX_USE_HIP) } ); - -#ifndef WARPX_DIM_RZ - amrex::ignore_unused(n_rz_azimuthal_modes); -#endif } #endif // WARPX_CHARGEDEPOSITION_H_ diff --git a/Source/Particles/Deposition/CurrentDeposition.H b/Source/Particles/Deposition/CurrentDeposition.H index 3dce7d28534..cb56c559bc0 100644 --- a/Source/Particles/Deposition/CurrentDeposition.H +++ b/Source/Particles/Deposition/CurrentDeposition.H @@ -24,6 +24,7 @@ #include #include #include +#include #include /** @@ -38,17 +39,17 @@ * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match * the time of the deposition. - * \param dzi, dxi, dyi The inverse cell sizes - * \param zmin, xmin, ymin The lower bounds of the domain + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param invvol The inverse volume of a grid cell * \param lo Index lower bounds of domain. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. */ template AMREX_GPU_HOST_DEVICE AMREX_INLINE -void doDepositionShapeNKernel(const amrex::ParticleReal xp, - const amrex::ParticleReal yp, - const amrex::ParticleReal zp, +void doDepositionShapeNKernel([[maybe_unused]] const amrex::ParticleReal xp, + [[maybe_unused]] const amrex::ParticleReal yp, + [[maybe_unused]] const amrex::ParticleReal zp, const amrex::ParticleReal wq, const amrex::ParticleReal vx, const amrex::ParticleReal vy, @@ -60,28 +61,14 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, amrex::IntVect const& jy_type, amrex::IntVect const& jz_type, const amrex::Real relative_time, - AMREX_D_DECL(const amrex::Real dzi, - const amrex::Real dxi, - const amrex::Real dyi), - AMREX_D_DECL(const amrex::Real zmin, - const amrex::Real xmin, - const amrex::Real ymin), + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Real invvol, const amrex::Dim3 lo, - const int n_rz_azimuthal_modes) + [[maybe_unused]] const int n_rz_azimuthal_modes) { using namespace amrex::literals; -#if !defined(WARPX_DIM_RZ) - amrex::ignore_unused(n_rz_azimuthal_modes); -#endif -#if defined(WARPX_DIM_1D_Z) - amrex::ignore_unused(xp, yp); -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - amrex::ignore_unused(yp); -#endif - constexpr int zdir = WARPX_ZINDEX; constexpr int NODE = amrex::IndexType::NODE; constexpr int CELL = amrex::IndexType::CELL; @@ -92,15 +79,8 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, const amrex::Real xpmid = xp + relative_time*vx; const amrex::Real ypmid = yp + relative_time*vy; const amrex::Real rpmid = std::sqrt(xpmid*xpmid + ypmid*ypmid); - amrex::Real costheta; - amrex::Real sintheta; - if (rpmid > 0._rt) { - costheta = xpmid/rpmid; - sintheta = ypmid/rpmid; - } else { - costheta = 1._rt; - sintheta = 0._rt; - } + const amrex::Real costheta = (rpmid > 0._rt ? xpmid/rpmid : 1._rt); + const amrex::Real sintheta = (rpmid > 0._rt ? ypmid/rpmid : 0._rt); const Complex xy0 = Complex{costheta, sintheta}; const amrex::Real wqx = wq*invvol*(+vx*costheta + vy*sintheta); const amrex::Real wqy = wq*invvol*(-vx*sintheta + vy*costheta); @@ -115,12 +95,13 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, #if (AMREX_SPACEDIM >= 2) // x direction // Get particle position after 1/2 push back in position -#if defined(WARPX_DIM_RZ) // Keep these double to avoid bug in single precision - const double xmid = (rpmid - xmin)*dxi; +#if defined(WARPX_DIM_RZ) + const double xmid = (rpmid - xyzmin.x)*dinv.x; #else - const double xmid = ((xp - xmin) + relative_time*vx)*dxi; + const double xmid = ((xp - xyzmin.x) + relative_time*vx)*dinv.x; #endif + // j_j[xyz] leftmost grid point in x that the particle touches for the centering of each current // sx_j[xyz] shape factor along x for the centering of each current // There are only two possible centerings, node or cell centered, so at most only two shape factor @@ -155,7 +136,7 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, #if defined(WARPX_DIM_3D) // y direction // Keep these double to avoid bug in single precision - const double ymid = ((yp - ymin) + relative_time*vy)*dyi; + const double ymid = ((yp - xyzmin.y) + relative_time*vy)*dinv.y; double sy_node[depos_order + 1] = {0.}; double sy_cell[depos_order + 1] = {0.}; int k_node = 0; @@ -182,7 +163,8 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, // z direction // Keep these double to avoid bug in single precision - const double zmid = ((zp - zmin) + relative_time*vz)*dzi; + constexpr int zdir = WARPX_ZINDEX; + const double zmid = ((zp - xyzmin.z) + relative_time*vz)*dinv.z; double sz_node[depos_order + 1] = {0.}; double sz_cell[depos_order + 1] = {0.}; int l_node = 0; @@ -282,7 +264,7 @@ void doDepositionShapeNKernel(const amrex::ParticleReal xp, * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match * the time of the deposition. - * \param dx 3D cell size + * \param dinv 3D cell size inverse * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. * \param q species charge. @@ -300,41 +282,19 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, amrex::FArrayBox& jz_fab, long np_to_deposit, amrex::Real relative_time, - const std::array& dx, - const std::array& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes) + [[maybe_unused]]int n_rz_azimuthal_modes) { using namespace amrex::literals; -#if !defined(WARPX_DIM_RZ) - amrex::ignore_unused(n_rz_azimuthal_modes); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; - const amrex::Real dzi = 1.0_rt/dx[2]; -#if defined(WARPX_DIM_1D_Z) - const amrex::Real invvol = dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real invvol = dxi*dzi; -#elif defined(WARPX_DIM_3D) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real dyi = 1.0_rt/dx[1]; - const amrex::Real invvol = dxi*dyi*dzi; -#endif -#if (AMREX_SPACEDIM >= 2) - const amrex::Real xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif - const amrex::Real zmin = xyzmin[2]; + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; const amrex::Real clightsq = 1.0_rt/PhysConst::c/PhysConst::c; @@ -367,9 +327,7 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, doDepositionShapeNKernel(xp, yp, zp, wq, vx, vy, vz, jx_arr, jy_arr, jz_arr, jx_type, jy_type, jz_type, - relative_time, - AMREX_D_DECL(dzi, dxi, dyi), - AMREX_D_DECL(zmin, xmin, ymin), + relative_time, dinv, xyzmin, invvol, lo, n_rz_azimuthal_modes); } @@ -391,7 +349,7 @@ void doDepositionShapeN (const GetParticlePosition& GetPosition, ion_lev is a null pointer. * \param jx_fab,jy_fab,jz_fab FArrayBox of current density, either full array or tile. * \param np_to_deposit Number of particles for which current is deposited. - * \param dx 3D cell size + * \param dinv 3D cell size inverse * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. * \param q species charge. @@ -411,40 +369,19 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, amrex::FArrayBox& jy_fab, amrex::FArrayBox& jz_fab, const long np_to_deposit, - const std::array& dx, - const std::array& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes) + [[maybe_unused]]const int n_rz_azimuthal_modes) { using namespace amrex::literals; -#if !defined(WARPX_DIM_RZ) - amrex::ignore_unused(n_rz_azimuthal_modes); -#endif // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; - const amrex::Real dzi = 1.0_rt/dx[2]; -#if defined(WARPX_DIM_1D_Z) - const amrex::Real invvol = dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real invvol = dxi*dzi; -#elif defined(WARPX_DIM_3D) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real dyi = 1.0_rt/dx[1]; - const amrex::Real invvol = dxi*dyi*dzi; -#endif -#if (AMREX_SPACEDIM >= 2) - const amrex::Real xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif - const amrex::Real zmin = xyzmin[2]; + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; amrex::Array4 const& jx_arr = jx_fab.array(); amrex::Array4 const& jy_arr = jy_fab.array(); @@ -483,9 +420,7 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, const amrex::Real relative_time = 0._rt; doDepositionShapeNKernel(xp, yp, zp, wq, vx, vy, vz, jx_arr, jy_arr, jz_arr, jx_type, jy_type, jz_type, - relative_time, - AMREX_D_DECL(dzi, dxi, dyi), - AMREX_D_DECL(zmin, xmin, ymin), + relative_time, dinv, xyzmin, invvol, lo, n_rz_azimuthal_modes); } @@ -509,7 +444,7 @@ void doDepositionShapeNImplicit(const GetParticlePosition& GetPosition, * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match * the time of the deposition. - * \param dx 3D cell size + * \param dinv 3D cell size inverse * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. * \param q species charge. @@ -527,8 +462,8 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, amrex::FArrayBox& jz_fab, long np_to_deposit, const amrex::Real relative_time, - const std::array& dx, - const std::array& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, amrex::Dim3 lo, amrex::Real q, int n_rz_azimuthal_modes, @@ -634,7 +569,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, { const unsigned int ip = permutation[ip_orig]; depositComponent(GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_buff, jx_type, - relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, + relative_time, dinv, xyzmin, lo, q, n_rz_azimuthal_modes, ip, zdir, NODE, CELL, 0); } @@ -649,7 +584,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, { const unsigned int ip = permutation[ip_orig]; depositComponent(GetPosition, wp, uxp, uyp, uzp, ion_lev, jy_buff, jy_type, - relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, + relative_time, dinv, xyzmin, lo, q, n_rz_azimuthal_modes, ip, zdir, NODE, CELL, 1); } @@ -664,7 +599,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, { const unsigned int ip = permutation[ip_orig]; depositComponent(GetPosition, wp, uxp, uyp, uzp, ion_lev, jz_buff, jz_type, - relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, + relative_time, dinv, xyzmin, lo, q, n_rz_azimuthal_modes, ip, zdir, NODE, CELL, 2); } @@ -675,7 +610,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, // Note, you should never reach this part of the code. This funcion cannot be called unless // using HIP/CUDA, and those things are checked prior //don't use any args - ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes, a_bins, box, geom, a_tbox_max_size); + ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, n_rz_azimuthal_modes, a_bins, box, geom, a_tbox_max_size); WARPX_ABORT_WITH_MESSAGE("Shared memory only implemented for HIP/CUDA"); #endif } @@ -698,7 +633,7 @@ void doDepositionSharedShapeN (const GetParticlePosition& GetPosition, * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match * the time of the deposition. - * \param dx 3D cell size + * \param dinv 3D cell size inverse * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. * \param q species charge. @@ -717,53 +652,28 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, long np_to_deposit, amrex::Real dt, amrex::Real relative_time, - const std::array& dx, - std::array xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes) + [[maybe_unused]]int n_rz_azimuthal_modes) { using namespace amrex; using namespace amrex::literals; -#if !defined(WARPX_DIM_RZ) - ignore_unused(n_rz_azimuthal_modes); -#endif - // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) bool const do_ionization = ion_lev; -#if !defined(WARPX_DIM_1D_Z) - Real const dxi = 1.0_rt / dx[0]; -#endif -#if !defined(WARPX_DIM_1D_Z) - Real const xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - Real const dyi = 1.0_rt / dx[1]; - Real const ymin = xyzmin[1]; +#if !defined(WARPX_DIM_3D) + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; #endif - Real const dzi = 1.0_rt / dx[2]; - Real const zmin = xyzmin[2]; -#if defined(WARPX_DIM_3D) - Real const invdtdx = 1.0_rt / (dt*dx[1]*dx[2]); - Real const invdtdy = 1.0_rt / (dt*dx[0]*dx[2]); - Real const invdtdz = 1.0_rt / (dt*dx[0]*dx[1]); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - Real const invdtdx = 1.0_rt / (dt*dx[2]); - Real const invdtdz = 1.0_rt / (dt*dx[0]); - Real const invvol = 1.0_rt / (dx[0]*dx[2]); -#elif defined(WARPX_DIM_1D_Z) - Real const invdtdz = 1.0_rt / (dt*dx[0]); - Real const invvol = 1.0_rt / (dx[2]); -#endif + amrex::XDim3 const invdtd = amrex::XDim3{(1.0_rt/dt)*dinv.y*dinv.z, + (1.0_rt/dt)*dinv.x*dinv.z, + (1.0_rt/dt)*dinv.x*dinv.y}; -#if defined(WARPX_DIM_RZ) - Complex const I = Complex{0._rt, 1._rt}; -#endif + Real constexpr clightsq = 1.0_rt / ( PhysConst::c * PhysConst::c ); - Real const clightsq = 1.0_rt / ( PhysConst::c * PhysConst::c ); #if !defined(WARPX_DIM_1D_Z) Real constexpr one_third = 1.0_rt / 3.0_rt; Real constexpr one_sixth = 1.0_rt / 6.0_rt; @@ -778,7 +688,6 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, + uyp[ip]*uyp[ip]*clightsq + uzp[ip]*uzp[ip]*clightsq); - // wqx, wqy wqz are particle current in each direction Real wq = q*wp[ip]; if (do_ionization){ wq *= ion_lev[ip]; @@ -787,14 +696,6 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, ParticleReal xp, yp, zp; GetPosition(ip, xp, yp, zp); -#if !defined(WARPX_DIM_1D_Z) - Real const wqx = wq*invdtdx; -#endif -#if defined(WARPX_DIM_3D) - Real const wqy = wq*invdtdy; -#endif - Real const wqz = wq*invdtdz; - // computes current and old position in grid units #if defined(WARPX_DIM_RZ) Real const xp_new = xp + (relative_time + 0.5_rt*dt)*uxp[ip]*gaminv; @@ -806,51 +707,33 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, Real const rp_new = std::sqrt(xp_new*xp_new + yp_new*yp_new); Real const rp_mid = std::sqrt(xp_mid*xp_mid + yp_mid*yp_mid); Real const rp_old = std::sqrt(xp_old*xp_old + yp_old*yp_old); - Real costheta_new, sintheta_new; - if (rp_new > 0._rt) { - costheta_new = xp_new/rp_new; - sintheta_new = yp_new/rp_new; - } else { - costheta_new = 1._rt; - sintheta_new = 0._rt; - } - amrex::Real costheta_mid, sintheta_mid; - if (rp_mid > 0._rt) { - costheta_mid = xp_mid/rp_mid; - sintheta_mid = yp_mid/rp_mid; - } else { - costheta_mid = 1._rt; - sintheta_mid = 0._rt; - } - amrex::Real costheta_old, sintheta_old; - if (rp_old > 0._rt) { - costheta_old = xp_old/rp_old; - sintheta_old = yp_old/rp_old; - } else { - costheta_old = 1._rt; - sintheta_old = 0._rt; - } + const amrex::Real costheta_mid = (rp_mid > 0._rt ? xp_mid/rp_mid : 1._rt); + const amrex::Real sintheta_mid = (rp_mid > 0._rt ? yp_mid/rp_mid : 0._rt); + const amrex::Real costheta_new = (rp_new > 0._rt ? xp_new/rp_new : 1._rt); + const amrex::Real sintheta_new = (rp_new > 0._rt ? yp_new/rp_new : 0._rt); + const amrex::Real costheta_old = (rp_old > 0._rt ? xp_old/rp_old : 1._rt); + const amrex::Real sintheta_old = (rp_old > 0._rt ? yp_old/rp_old : 0._rt); const Complex xy_new0 = Complex{costheta_new, sintheta_new}; const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; const Complex xy_old0 = Complex{costheta_old, sintheta_old}; // Keep these double to avoid bug in single precision - double const x_new = (rp_new - xmin)*dxi; - double const x_old = (rp_old - xmin)*dxi; + double const x_new = (rp_new - xyzmin.x)*dinv.x; + double const x_old = (rp_old - xyzmin.x)*dinv.x; #else #if !defined(WARPX_DIM_1D_Z) // Keep these double to avoid bug in single precision - double const x_new = (xp - xmin + (relative_time + 0.5_rt*dt)*uxp[ip]*gaminv)*dxi; - double const x_old = x_new - dt*dxi*uxp[ip]*gaminv; + double const x_new = (xp - xyzmin.x + (relative_time + 0.5_rt*dt)*uxp[ip]*gaminv)*dinv.x; + double const x_old = x_new - dt*dinv.x*uxp[ip]*gaminv; #endif #endif #if defined(WARPX_DIM_3D) // Keep these double to avoid bug in single precision - double const y_new = (yp - ymin + (relative_time + 0.5_rt*dt)*uyp[ip]*gaminv)*dyi; - double const y_old = y_new - dt*dyi*uyp[ip]*gaminv; + double const y_new = (yp - xyzmin.y + (relative_time + 0.5_rt*dt)*uyp[ip]*gaminv)*dinv.y; + double const y_old = y_new - dt*dinv.y*uyp[ip]*gaminv; #endif // Keep these double to avoid bug in single precision - double const z_new = (zp - zmin + (relative_time + 0.5_rt*dt)*uzp[ip]*gaminv)*dzi; - double const z_old = z_new - dt*dzi*uzp[ip]*gaminv; + double const z_new = (zp - xyzmin.z + (relative_time + 0.5_rt*dt)*uzp[ip]*gaminv)*dinv.z; + double const z_old = z_new - dt*dinv.z*uzp[ip]*gaminv; #if defined(WARPX_DIM_RZ) Real const vy = (-uxp[ip]*sintheta_mid + uyp[ip]*costheta_mid)*gaminv; @@ -861,6 +744,12 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, Real const vy = uyp[ip]*gaminv; #endif + // --- Compute shape factors + // Compute shape factors for position as they are now and at old positions + // [ijk]_new: leftmost grid point that the particle touches + const Compute_shape_factor< depos_order > compute_shape_factor; + const Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; + // Shape factor arrays // Note that there are extra values above and below // to possibly hold the factor for the old particle @@ -869,30 +758,17 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, #if !defined(WARPX_DIM_1D_Z) double sx_new[depos_order + 3] = {0.}; double sx_old[depos_order + 3] = {0.}; -#endif -#if defined(WARPX_DIM_3D) - // Keep these double to avoid bug in single precision - double sy_new[depos_order + 3] = {0.}; - double sy_old[depos_order + 3] = {0.}; -#endif - // Keep these double to avoid bug in single precision - double sz_new[depos_order + 3] = {0.}; - double sz_old[depos_order + 3] = {0.}; - - // --- Compute shape factors - // Compute shape factors for position as they are now and at old positions - // [ijk]_new: leftmost grid point that the particle touches - const Compute_shape_factor< depos_order > compute_shape_factor; - const Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; - -#if !defined(WARPX_DIM_1D_Z) const int i_new = compute_shape_factor(sx_new+1, x_new); const int i_old = compute_shifted_shape_factor(sx_old, x_old, i_new); #endif #if defined(WARPX_DIM_3D) + double sy_new[depos_order + 3] = {0.}; + double sy_old[depos_order + 3] = {0.}; const int j_new = compute_shape_factor(sy_new+1, y_new); const int j_old = compute_shifted_shape_factor(sy_old, y_old, j_new); #endif + double sz_new[depos_order + 3] = {0.}; + double sz_old[depos_order + 3] = {0.}; const int k_new = compute_shape_factor(sz_new+1, z_new); const int k_old = compute_shifted_shape_factor(sz_old, z_old, k_new); @@ -917,7 +793,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, for (int j=djl; j<=depos_order+2-dju; j++) { amrex::Real sdxi = 0._rt; for (int i=dil; i<=depos_order+1-diu; i++) { - sdxi += wqx*(sx_old[i] - sx_new[i])*( + sdxi += wq*invdtd.x*(sx_old[i] - sx_new[i])*( one_third*(sy_new[j]*sz_new[k] + sy_old[j]*sz_old[k]) +one_sixth*(sy_new[j]*sz_old[k] + sy_old[j]*sz_new[k])); amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdxi); @@ -928,7 +804,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, for (int i=dil; i<=depos_order+2-diu; i++) { amrex::Real sdyj = 0._rt; for (int j=djl; j<=depos_order+1-dju; j++) { - sdyj += wqy*(sy_old[j] - sy_new[j])*( + sdyj += wq*invdtd.y*(sy_old[j] - sy_new[j])*( one_third*(sx_new[i]*sz_new[k] + sx_old[i]*sz_old[k]) +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdyj); @@ -939,7 +815,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, for (int i=dil; i<=depos_order+2-diu; i++) { amrex::Real sdzk = 0._rt; for (int k=dkl; k<=depos_order+1-dku; k++) { - sdzk += wqz*(sz_old[k] - sz_new[k])*( + sdzk += wq*invdtd.z*(sz_old[k] - sz_new[k])*( one_third*(sx_new[i]*sy_new[j] + sx_old[i]*sy_old[j]) +one_sixth*(sx_new[i]*sy_old[j] + sx_old[i]*sy_new[j])); amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdzk); @@ -952,7 +828,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, for (int k=dkl; k<=depos_order+2-dku; k++) { amrex::Real sdxi = 0._rt; for (int i=dil; i<=depos_order+1-diu; i++) { - sdxi += wqx*(sx_old[i] - sx_new[i])*0.5_rt*(sz_new[k] + sz_old[k]); + sdxi += wq*invdtd.x*(sx_old[i] - sx_new[i])*0.5_rt*(sz_new[k] + sz_old[k]); amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdxi); #if defined(WARPX_DIM_RZ) Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} @@ -973,6 +849,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdyj); #if defined(WARPX_DIM_RZ) + Complex const I = Complex{0._rt, 1._rt}; Complex xy_new = xy_new0; Complex xy_mid = xy_mid0; Complex xy_old = xy_old0; @@ -980,7 +857,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { // The factor 2 comes from the normalization of the modes // The minus sign comes from the different convention with respect to Davidson et al. - const Complex djt_cmplx = -2._rt * I*(i_new-1 + i + xmin*dxi)*wq*invdtdx/(amrex::Real)imode + const Complex djt_cmplx = -2._rt * I*(i_new-1 + i + xyzmin.x*dinv.x)*wq*invdtd.x/(amrex::Real)imode *(Complex(sx_new[i]*sz_new[k], 0._rt)*(xy_new - xy_mid) + Complex(sx_old[i]*sz_old[k], 0._rt)*(xy_mid - xy_old)); amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode-1), djt_cmplx.real()); @@ -995,7 +872,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, for (int i=dil; i<=depos_order+2-diu; i++) { Real sdzk = 0._rt; for (int k=dkl; k<=depos_order+1-dku; k++) { - sdzk += wqz*(sz_old[k] - sz_new[k])*0.5_rt*(sx_new[i] + sx_old[i]); + sdzk += wq*invdtd.z*(sz_old[k] - sz_new[k])*0.5_rt*(sx_new[i] + sx_old[i]); amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdzk); #if defined(WARPX_DIM_RZ) Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} @@ -1021,7 +898,7 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, } amrex::Real sdzk = 0._rt; for (int k=dkl; k<=depos_order+1-dku; k++) { - sdzk += wqz*(sz_old[k] - sz_new[k]); + sdzk += wq*invdtd.z*(sz_old[k] - sz_new[k]); amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+k_new-1+k, 0, 0, 0), sdzk); } #endif @@ -1047,16 +924,16 @@ void doEsirkepovDepositionShapeN (const GetParticlePosition& GetPosition, * \param Jx_arr,Jy_arr,Jz_arr Array4 of current density, either full array or tile. * \param np_to_deposit Number of particles for which current is deposited. * \param dt Time step for particle level - * \param dx 3D cell size + * \param dinv 3D cell size inverse * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. */ template -void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * const xp_n, - const amrex::ParticleReal * const yp_n, - const amrex::ParticleReal * const zp_n, +void doChargeConservingDepositionShapeNImplicit ([[maybe_unused]]const amrex::ParticleReal * const xp_n, + [[maybe_unused]]const amrex::ParticleReal * const yp_n, + [[maybe_unused]]const amrex::ParticleReal * const zp_n, const GetParticlePosition& GetPosition, const amrex::ParticleReal * const wp, [[maybe_unused]]const amrex::ParticleReal * const uxp_n, @@ -1071,49 +948,25 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con const amrex::Array4& Jz_arr, const long np_to_deposit, const amrex::Real dt, - const std::array& dx, - const std::array xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes) + [[maybe_unused]] const int n_rz_azimuthal_modes) { using namespace amrex; -#if !defined(WARPX_DIM_RZ) - ignore_unused(n_rz_azimuthal_modes); -#endif // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) bool const do_ionization = ion_lev; -#if !defined(WARPX_DIM_1D_Z) - Real const dxi = 1.0_rt / dx[0]; -#endif -#if !defined(WARPX_DIM_1D_Z) - Real const xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - Real const dyi = 1.0_rt / dx[1]; - Real const ymin = xyzmin[1]; -#endif - Real const dzi = 1.0_rt / dx[2]; - Real const zmin = xyzmin[2]; -#if defined(WARPX_DIM_3D) - Real const invdtdx = 1.0_rt / (dt*dx[1]*dx[2]); - Real const invdtdy = 1.0_rt / (dt*dx[0]*dx[2]); - Real const invdtdz = 1.0_rt / (dt*dx[0]*dx[1]); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - Real const invdtdx = 1.0_rt / (dt*dx[2]); - Real const invdtdz = 1.0_rt / (dt*dx[0]); - Real const invvol = 1.0_rt / (dx[0]*dx[2]); -#elif defined(WARPX_DIM_1D_Z) - Real const invdtdz = 1.0_rt / (dt*dx[0]); - Real const invvol = 1.0_rt / (dx[2]); +#if !defined(WARPX_DIM_3D) + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; #endif -#if defined(WARPX_DIM_RZ) - Complex const I = Complex{0._rt, 1._rt}; -#endif + amrex::XDim3 const invdtd = amrex::XDim3{(1.0_rt/dt)*dinv.y*dinv.z, + (1.0_rt/dt)*dinv.x*dinv.z, + (1.0_rt/dt)*dinv.x*dinv.y}; #if !defined(WARPX_DIM_1D_Z) Real constexpr one_third = 1.0_rt / 3.0_rt; @@ -1137,7 +990,6 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con const amrex::ParticleReal gaminv = 2.0_prt/(gamma_n + gamma_np1); #endif - // wqx, wqy wqz are particle current in each direction Real wq = q*wp[ip]; if (do_ionization){ wq *= ion_lev[ip]; @@ -1148,24 +1000,12 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con #if !defined(WARPX_DIM_1D_Z) ParticleReal const xp_np1 = 2._prt*xp_nph - xp_n[ip]; -#else - ignore_unused(xp_n); #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) ParticleReal const yp_np1 = 2._prt*yp_nph - yp_n[ip]; -#else - ignore_unused(yp_n); #endif ParticleReal const zp_np1 = 2._prt*zp_nph - zp_n[ip]; -#if !defined(WARPX_DIM_1D_Z) - amrex::Real const wqx = wq*invdtdx; -#endif -#if defined(WARPX_DIM_3D) - amrex::Real const wqy = wq*invdtdy; -#endif - amrex::Real const wqz = wq*invdtdz; - // computes current and old position in grid units #if defined(WARPX_DIM_RZ) amrex::Real const xp_new = xp_np1; @@ -1177,51 +1017,33 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con amrex::Real const rp_new = std::sqrt(xp_new*xp_new + yp_new*yp_new); amrex::Real const rp_old = std::sqrt(xp_old*xp_old + yp_old*yp_old); amrex::Real const rp_mid = (rp_new + rp_old)/2._rt; - amrex::Real costheta_new, sintheta_new; - if (rp_new > 0._rt) { - costheta_new = xp_new/rp_new; - sintheta_new = yp_new/rp_new; - } else { - costheta_new = 1._rt; - sintheta_new = 0._rt; - } - amrex::Real costheta_mid, sintheta_mid; - if (rp_mid > 0._rt) { - costheta_mid = xp_mid/rp_mid; - sintheta_mid = yp_mid/rp_mid; - } else { - costheta_mid = 1._rt; - sintheta_mid = 0._rt; - } - amrex::Real costheta_old, sintheta_old; - if (rp_old > 0._rt) { - costheta_old = xp_old/rp_old; - sintheta_old = yp_old/rp_old; - } else { - costheta_old = 1._rt; - sintheta_old = 0._rt; - } + const amrex::Real costheta_mid = (rp_mid > 0._rt ? xp_mid/rp_mid : 1._rt); + const amrex::Real sintheta_mid = (rp_mid > 0._rt ? yp_mid/rp_mid : 0._rt); + const amrex::Real costheta_new = (rp_new > 0._rt ? xp_new/rp_new : 1._rt); + const amrex::Real sintheta_new = (rp_new > 0._rt ? yp_new/rp_new : 0._rt); + const amrex::Real costheta_old = (rp_old > 0._rt ? xp_old/rp_old : 1._rt); + const amrex::Real sintheta_old = (rp_old > 0._rt ? yp_old/rp_old : 0._rt); const Complex xy_new0 = Complex{costheta_new, sintheta_new}; const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; const Complex xy_old0 = Complex{costheta_old, sintheta_old}; // Keep these double to avoid bug in single precision - double const x_new = (rp_new - xmin)*dxi; - double const x_old = (rp_old - xmin)*dxi; + double const x_new = (rp_new - xyzmin.x)*dinv.x; + double const x_old = (rp_old - xyzmin.x)*dinv.x; #else #if !defined(WARPX_DIM_1D_Z) // Keep these double to avoid bug in single precision - double const x_new = (xp_np1 - xmin)*dxi; - double const x_old = (xp_n[ip] - xmin)*dxi; + double const x_new = (xp_np1 - xyzmin.x)*dinv.x; + double const x_old = (xp_n[ip] - xyzmin.x)*dinv.x; #endif #endif #if defined(WARPX_DIM_3D) // Keep these double to avoid bug in single precision - double const y_new = (yp_np1 - ymin)*dyi; - double const y_old = (yp_n[ip] - ymin)*dyi; + double const y_new = (yp_np1 - xyzmin.y)*dinv.y; + double const y_old = (yp_n[ip] - xyzmin.y)*dinv.y; #endif // Keep these double to avoid bug in single precision - double const z_new = (zp_np1 - zmin)*dzi; - double const z_old = (zp_n[ip] - zmin)*dzi; + double const z_new = (zp_np1 - xyzmin.z)*dinv.z; + double const z_old = (zp_n[ip] - xyzmin.z)*dinv.z; #if defined(WARPX_DIM_RZ) amrex::Real const vy = (-uxp_nph[ip]*sintheta_mid + uyp_nph[ip]*costheta_mid)*gaminv; @@ -1232,6 +1054,12 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con amrex::Real const vy = uyp_nph[ip]*gaminv; #endif + // --- Compute shape factors + // Compute shape factors for position as they are now and at old positions + // [ijk]_new: leftmost grid point that the particle touches + const Compute_shape_factor< depos_order > compute_shape_factor; + const Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; + // Shape factor arrays // Note that there are extra values above and below // to possibly hold the factor for the old particle @@ -1240,30 +1068,17 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con #if !defined(WARPX_DIM_1D_Z) double sx_new[depos_order + 3] = {0.}; double sx_old[depos_order + 3] = {0.}; -#endif -#if defined(WARPX_DIM_3D) - // Keep these double to avoid bug in single precision - double sy_new[depos_order + 3] = {0.}; - double sy_old[depos_order + 3] = {0.}; -#endif - // Keep these double to avoid bug in single precision - double sz_new[depos_order + 3] = {0.}; - double sz_old[depos_order + 3] = {0.}; - - // --- Compute shape factors - // Compute shape factors for position as they are now and at old positions - // [ijk]_new: leftmost grid point that the particle touches - const Compute_shape_factor< depos_order > compute_shape_factor; - const Compute_shifted_shape_factor< depos_order > compute_shifted_shape_factor; - -#if !defined(WARPX_DIM_1D_Z) const int i_new = compute_shape_factor(sx_new+1, x_new); const int i_old = compute_shifted_shape_factor(sx_old, x_old, i_new); #endif #if defined(WARPX_DIM_3D) + double sy_new[depos_order + 3] = {0.}; + double sy_old[depos_order + 3] = {0.}; const int j_new = compute_shape_factor(sy_new+1, y_new); const int j_old = compute_shifted_shape_factor(sy_old, y_old, j_new); #endif + double sz_new[depos_order + 3] = {0.}; + double sz_old[depos_order + 3] = {0.}; const int k_new = compute_shape_factor(sz_new+1, z_new); const int k_old = compute_shifted_shape_factor(sz_old, z_old, k_new); @@ -1288,7 +1103,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con for (int j=djl; j<=depos_order+2-dju; j++) { amrex::Real sdxi = 0._rt; for (int i=dil; i<=depos_order+1-diu; i++) { - sdxi += wqx*(sx_old[i] - sx_new[i])*( + sdxi += wq*invdtd.x*(sx_old[i] - sx_new[i])*( one_third*(sy_new[j]*sz_new[k] + sy_old[j]*sz_old[k]) +one_sixth*(sy_new[j]*sz_old[k] + sy_old[j]*sz_new[k])); amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdxi); @@ -1299,7 +1114,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con for (int i=dil; i<=depos_order+2-diu; i++) { amrex::Real sdyj = 0._rt; for (int j=djl; j<=depos_order+1-dju; j++) { - sdyj += wqy*(sy_old[j] - sy_new[j])*( + sdyj += wq*invdtd.y*(sy_old[j] - sy_new[j])*( one_third*(sx_new[i]*sz_new[k] + sx_old[i]*sz_old[k]) +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdyj); @@ -1310,7 +1125,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con for (int i=dil; i<=depos_order+2-diu; i++) { amrex::Real sdzk = 0._rt; for (int k=dkl; k<=depos_order+1-dku; k++) { - sdzk += wqz*(sz_old[k] - sz_new[k])*( + sdzk += wq*invdtd.z*(sz_old[k] - sz_new[k])*( one_third*(sx_new[i]*sy_new[j] + sx_old[i]*sy_old[j]) +one_sixth*(sx_new[i]*sy_old[j] + sx_old[i]*sy_new[j])); amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+j_new-1+j, lo.z+k_new-1+k), sdzk); @@ -1323,7 +1138,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con for (int k=dkl; k<=depos_order+2-dku; k++) { amrex::Real sdxi = 0._rt; for (int i=dil; i<=depos_order+1-diu; i++) { - sdxi += wqx*(sx_old[i] - sx_new[i])*0.5_rt*(sz_new[k] + sz_old[k]); + sdxi += wq*invdtd.x*(sx_old[i] - sx_new[i])*0.5_rt*(sz_new[k] + sz_old[k]); amrex::Gpu::Atomic::AddNoRet( &Jx_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdxi); #if defined(WARPX_DIM_RZ) Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} @@ -1344,6 +1159,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con +one_sixth*(sx_new[i]*sz_old[k] + sx_old[i]*sz_new[k])); amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdyj); #if defined(WARPX_DIM_RZ) + Complex const I = Complex{0._rt, 1._rt}; Complex xy_new = xy_new0; Complex xy_mid = xy_mid0; Complex xy_old = xy_old0; @@ -1351,7 +1167,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con for (int imode=1 ; imode < n_rz_azimuthal_modes ; imode++) { // The factor 2 comes from the normalization of the modes // The minus sign comes from the different convention with respect to Davidson et al. - const Complex djt_cmplx = -2._rt * I*(i_new-1 + i + xmin*dxi)*wq*invdtdx/(amrex::Real)imode + const Complex djt_cmplx = -2._rt * I*(i_new-1 + i + xyzmin.x*dinv.x)*wq*invdtd.x/(amrex::Real)imode *(Complex(sx_new[i]*sz_new[k], 0._rt)*(xy_new - xy_mid) + Complex(sx_old[i]*sz_old[k], 0._rt)*(xy_mid - xy_old)); amrex::Gpu::Atomic::AddNoRet( &Jy_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 2*imode-1), djt_cmplx.real()); @@ -1366,7 +1182,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con for (int i=dil; i<=depos_order+2-diu; i++) { Real sdzk = 0._rt; for (int k=dkl; k<=depos_order+1-dku; k++) { - sdzk += wqz*(sz_old[k] - sz_new[k])*0.5_rt*(sx_new[i] + sx_old[i]); + sdzk += wq*invdtd.z*(sz_old[k] - sz_new[k])*0.5_rt*(sx_new[i] + sx_old[i]); amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+i_new-1+i, lo.y+k_new-1+k, 0, 0), sdzk); #if defined(WARPX_DIM_RZ) Complex xy_mid = xy_mid0; // Throughout the following loop, xy_mid takes the value e^{i m theta} @@ -1392,7 +1208,7 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con } amrex::Real sdzk = 0._rt; for (int k=dkl; k<=depos_order+1-dku; k++) { - sdzk += wqz*(sz_old[k] - sz_new[k]); + sdzk += wq*invdtd.z*(sz_old[k] - sz_new[k]); amrex::Gpu::Atomic::AddNoRet( &Jz_arr(lo.x+k_new-1+k, 0, 0, 0), sdzk); } #endif @@ -1420,16 +1236,16 @@ void doChargeConservingDepositionShapeNImplicit (const amrex::ParticleReal * con * \param Jx_arr,Jy_arr,Jz_arr Array4 of current density, either full array or tile. * \param np_to_deposit Number of particles for which current is deposited. * \param dt Time step for particle level - * \param dx 3D cell size + * \param dinv 3D cell size inverse * \param xyzmin Physical lower bounds of domain. * \param lo Index lower bounds of domain. * \param q species charge. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry. */ template -void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_n, - const amrex::ParticleReal * const yp_n, - const amrex::ParticleReal * const zp_n, +void doVillasenorDepositionShapeNImplicit ([[maybe_unused]]const amrex::ParticleReal * const xp_n, + [[maybe_unused]]const amrex::ParticleReal * const yp_n, + [[maybe_unused]]const amrex::ParticleReal * const zp_n, const GetParticlePosition& GetPosition, const amrex::ParticleReal * const wp, [[maybe_unused]]const amrex::ParticleReal * const uxp_n, @@ -1444,42 +1260,21 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ const amrex::Array4& Jz_arr, const long np_to_deposit, const amrex::Real dt, - const std::array& dx, - const std::array xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3 lo, const amrex::Real q, - const int n_rz_azimuthal_modes) + [[maybe_unused]] const int n_rz_azimuthal_modes) { using namespace amrex; -#if !defined(WARPX_DIM_RZ) - ignore_unused(n_rz_azimuthal_modes); -#endif // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) bool const do_ionization = ion_lev; -#if !defined(WARPX_DIM_1D_Z) - Real const dxi = 1.0_rt / dx[0]; -#endif -#if !defined(WARPX_DIM_1D_Z) - Real const xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - Real const dyi = 1.0_rt / dx[1]; - Real const ymin = xyzmin[1]; -#endif - Real const dzi = 1.0_rt / dx[2]; - Real const zmin = xyzmin[2]; -#if defined(WARPX_DIM_3D) - Real const invvol = 1.0_rt / (dx[0]*dx[1]*dx[2]); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - Real const invvol = 1.0_rt / (dx[0]*dx[2]); -#elif defined(WARPX_DIM_1D_Z) - Real const invvol = 1.0_rt / (dx[2]); -#endif + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; -#if !defined(WARPX_DIM_1D_Z) +#if (AMREX_SPACEDIM > 1) Real constexpr one_third = 1.0_rt / 3.0_rt; Real constexpr one_sixth = 1.0_rt / 6.0_rt; #endif @@ -1502,7 +1297,6 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ const amrex::ParticleReal gaminv = 2.0_prt/(gamma_n + gamma_np1); #endif - // wqx, wqy wqz are particle current in each direction Real wq = q*wp[ip]; if (do_ionization){ wq *= ion_lev[ip]; @@ -1513,13 +1307,9 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ #if !defined(WARPX_DIM_1D_Z) ParticleReal const xp_np1 = 2._prt*xp_nph - xp_n[ip]; -#else - ignore_unused(xp_n); #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) ParticleReal const yp_np1 = 2._prt*yp_nph - yp_n[ip]; -#else - ignore_unused(yp_n); #endif ParticleReal const zp_np1 = 2._prt*zp_nph - zp_n[ip]; @@ -1545,14 +1335,14 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; // Keep these double to avoid bug in single precision - double const x_new = (rp_new - xmin)*dxi; - double const x_old = (rp_old - xmin)*dxi; + double const x_new = (rp_new - xyzmin.x)*dinv.x; + double const x_old = (rp_old - xyzmin.x)*dinv.x; amrex::Real const vx = (rp_new - rp_old)/dt; amrex::Real const vy = (-uxp_nph[ip]*sintheta_mid + uyp_nph[ip]*costheta_mid)*gaminv; #elif defined(WARPX_DIM_XZ) // Keep these double to avoid bug in single precision - double const x_new = (xp_np1 - xmin)*dxi; - double const x_old = (xp_n[ip] - xmin)*dxi; + double const x_new = (xp_np1 - xyzmin.x)*dinv.x; + double const x_old = (xp_n[ip] - xyzmin.x)*dinv.x; amrex::Real const vx = (xp_np1 - xp_n[ip])/dt; amrex::Real const vy = uyp_nph[ip]*gaminv; #elif defined(WARPX_DIM_1D_Z) @@ -1560,17 +1350,17 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ amrex::Real const vy = uyp_nph[ip]*gaminv; #elif defined(WARPX_DIM_3D) // Keep these double to avoid bug in single precision - double const x_new = (xp_np1 - xmin)*dxi; - double const x_old = (xp_n[ip] - xmin)*dxi; - double const y_new = (yp_np1 - ymin)*dyi; - double const y_old = (yp_n[ip] - ymin)*dyi; + double const x_new = (xp_np1 - xyzmin.x)*dinv.x; + double const x_old = (xp_n[ip] - xyzmin.x)*dinv.x; + double const y_new = (yp_np1 - xyzmin.y)*dinv.y; + double const y_old = (yp_n[ip] - xyzmin.y)*dinv.y; amrex::Real const vx = (xp_np1 - xp_n[ip])/dt; amrex::Real const vy = (yp_np1 - yp_n[ip])/dt; #endif // Keep these double to avoid bug in single precision - double const z_new = (zp_np1 - zmin)*dzi; - double const z_old = (zp_n[ip] - zmin)*dzi; + double const z_new = (zp_np1 - xyzmin.z)*dinv.z; + double const z_old = (zp_n[ip] - xyzmin.z)*dinv.z; amrex::Real const vz = (zp_np1 - zp_n[ip])/dt; // Define velocity kernals to deposit @@ -2057,7 +1847,7 @@ void doVillasenorDepositionShapeNImplicit (const amrex::ParticleReal * const xp_ * current positions of the particles. When different than 0, * the particle position will be temporarily modified to match * the time of the deposition. - * \param[in] dx 3D cell size + * \param[in] dinv 3D cell size inverse * \param[in] xyzmin 3D lower bounds of physical domain * \param[in] lo Dimension-agnostic lower bounds of index domain * \param[in] q Species charge @@ -2076,57 +1866,37 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, long np_to_deposit, amrex::Real dt, amrex::Real relative_time, - const std::array& dx, - const std::array& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, amrex::Dim3 lo, amrex::Real q, - int n_rz_azimuthal_modes) + [[maybe_unused]]int n_rz_azimuthal_modes) { using namespace amrex::literals; #if defined(WARPX_DIM_RZ) amrex::ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, Dx_fab, Dy_fab, Dz_fab, - np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes); + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q); WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented in RZ geometry"); #endif #if defined(WARPX_DIM_1D_Z) amrex::ignore_unused(GetPosition, wp, uxp, uyp, uzp, ion_lev, Dx_fab, Dy_fab, Dz_fab, - np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, n_rz_azimuthal_modes); - WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented in cartesian 1D geometry"); + np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q); + WARPX_ABORT_WITH_MESSAGE("Vay deposition not implemented in 1D geometry"); #endif #if !(defined WARPX_DIM_RZ || defined WARPX_DIM_1D_Z) - amrex::ignore_unused(n_rz_azimuthal_modes); // If ion_lev is a null pointer, then do_ionization=0, else do_ionization=1 const bool do_ionization = ion_lev; - // Inverse cell volume in each direction - const amrex::Real dxi = 1._rt / dx[0]; - const amrex::Real dzi = 1._rt / dx[2]; -#if defined(WARPX_DIM_3D) - const amrex::Real dyi = 1._rt / dx[1]; -#endif - // Inverse of time step const amrex::Real invdt = 1._rt / dt; - // Total inverse cell volume -#if defined(WARPX_DIM_XZ) - const amrex::Real invvol = dxi * dzi; -#elif defined(WARPX_DIM_3D) - const amrex::Real invvol = dxi * dyi * dzi; -#endif - - // Lower bound of physical domain in each direction - const amrex::Real xmin = xyzmin[0]; - const amrex::Real zmin = xyzmin[2]; -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; // Allocate temporary arrays #if defined(WARPX_DIM_3D) @@ -2172,23 +1942,18 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, yp += relative_time * vy; zp += relative_time * vz; - // Particle current densities -#if defined(WARPX_DIM_XZ) - const amrex::Real wqy = wq * vy * invvol; -#endif - // Current and old particle positions in grid units // Keep these double to avoid bug in single precision. - double const x_new = (xp - xmin + 0.5_rt*dt*vx) * dxi; - double const x_old = (xp - xmin - 0.5_rt*dt*vx) * dxi; + double const x_new = (xp - xyzmin.x + 0.5_rt*dt*vx) * dinv.x; + double const x_old = (xp - xyzmin.x - 0.5_rt*dt*vx) * dinv.x; #if defined(WARPX_DIM_3D) // Keep these double to avoid bug in single precision. - double const y_new = (yp - ymin + 0.5_rt*dt*vy) * dyi; - double const y_old = (yp - ymin - 0.5_rt*dt*vy) * dyi; + double const y_new = (yp - xyzmin.y + 0.5_rt*dt*vy) * dinv.y; + double const y_old = (yp - xyzmin.y - 0.5_rt*dt*vy) * dinv.y; #endif // Keep these double to avoid bug in single precision. - double const z_new = (zp - zmin + 0.5_rt*dt*vz) * dzi; - double const z_old = (zp - zmin - 0.5_rt*dt*vz) * dzi; + double const z_new = (zp - xyzmin.z + 0.5_rt*dt*vz) * dinv.z; + double const z_old = (zp - xyzmin.z - 0.5_rt*dt*vz) * dinv.z; // Shape factor arrays for current and old positions (nodal) // Keep these double to avoid bug in single precision. @@ -2235,6 +2000,7 @@ void doVayDepositionShapeN (const GetParticlePosition& GetPosition, // Deposit current into Dx_arr, Dy_arr and Dz_arr #if defined(WARPX_DIM_XZ) + const amrex::Real wqy = wq * vy * invvol; for (int k=0; k<=depos_order; k++) { for (int i=0; i<=depos_order; i++) { diff --git a/Source/Particles/Deposition/SharedDepositionUtils.H b/Source/Particles/Deposition/SharedDepositionUtils.H index 1e2294be3a2..e6deb51b5df 100644 --- a/Source/Particles/Deposition/SharedDepositionUtils.H +++ b/Source/Particles/Deposition/SharedDepositionUtils.H @@ -74,8 +74,8 @@ void depositComponent (const GetParticlePosition& GetPosition, amrex::Array4 const& j_buff, amrex::IntVect const j_type, const amrex::Real relative_time, - const std::array& dx, - const std::array& xyzmin, + const amrex::XDim3 dinv, + const amrex::XDim3 xyzmin, const amrex::Dim3 lo, const amrex::Real q, const int n_rz_azimuthal_modes, @@ -91,26 +91,8 @@ void depositComponent (const GetParticlePosition& GetPosition, // Whether ion_lev is a null pointer (do_ionization=0) or a real pointer // (do_ionization=1) const bool do_ionization = ion_lev; - const amrex::Real dzi = 1.0_rt/dx[2]; -#if defined(WARPX_DIM_1D_Z) - const amrex::Real invvol = dzi; -#endif -#if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real invvol = dxi*dzi; -#elif defined(WARPX_DIM_3D) - const amrex::Real dxi = 1.0_rt/dx[0]; - const amrex::Real dyi = 1.0_rt/dx[1]; - const amrex::Real invvol = dxi*dyi*dzi; -#endif -#if (AMREX_SPACEDIM >= 2) - const amrex::Real xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif - const amrex::Real zmin = xyzmin[2]; + const amrex::Real invvol = dinv.x*dinv.y*dinv.z; const amrex::Real clightsq = 1.0_rt/PhysConst::c/PhysConst::c; @@ -171,9 +153,9 @@ void depositComponent (const GetParticlePosition& GetPosition, // Get particle position after 1/2 push back in position #if defined(WARPX_DIM_RZ) // Keep these double to avoid bug in single precision - const double xmid = (rpmid - xmin)*dxi; + const double xmid = (rpmid - xyzmin.x)*dinv.x; #else - const double xmid = ((xp - xmin) + relative_time*vx)*dxi; + const double xmid = ((xp - xyzmin.x) + relative_time*vx)*dinv.x; #endif // j_j[xyz] leftmost grid point in x that the particle touches for the centering of each current // sx_j[xyz] shape factor along x for the centering of each current @@ -203,7 +185,7 @@ void depositComponent (const GetParticlePosition& GetPosition, #if defined(WARPX_DIM_3D) // y direction // Keep these double to avoid bug in single precision - const double ymid = ((yp - ymin) + relative_time*vy)*dyi; + const double ymid = ((yp - xyzmin.y) + relative_time*vy)*dinv.y; double sy_node[depos_order + 1] = {0.}; double sy_cell[depos_order + 1] = {0.}; int k_node = 0; @@ -224,7 +206,7 @@ void depositComponent (const GetParticlePosition& GetPosition, // z direction // Keep these double to avoid bug in single precision - const double zmid = ((zp - zmin) + relative_time*vz)*dzi; + const double zmid = ((zp - xyzmin.z) + relative_time*vz)*dinv.z; double sz_node[depos_order + 1] = {0.}; double sz_cell[depos_order + 1] = {0.}; int l_node = 0; diff --git a/Source/Particles/ElementaryProcess/Ionization.H b/Source/Particles/ElementaryProcess/Ionization.H index cee7ec07eb5..6f98c18959a 100644 --- a/Source/Particles/ElementaryProcess/Ionization.H +++ b/Source/Particles/ElementaryProcess/Ionization.H @@ -62,8 +62,8 @@ struct IonizationFilterFunc amrex::IndexType m_by_type; amrex::IndexType m_bz_type; - amrex::GpuArray m_dx_arr; - amrex::GpuArray m_xyzmin_arr; + amrex::XDim3 m_dinv; + amrex::XDim3 m_xyzmin; bool m_galerkin_interpolation; int m_nox; @@ -117,7 +117,7 @@ struct IonizationFilterFunc doGatherShapeN(xp, yp, zp, ex, ey, ez, bx, by, bz, m_ex_arr, m_ey_arr, m_ez_arr, m_bx_arr, m_by_arr, m_bz_arr, m_ex_type, m_ey_type, m_ez_type, m_bx_type, m_by_type, m_bz_type, - m_dx_arr, m_xyzmin_arr, m_lo, m_n_rz_azimuthal_modes, + m_dinv, m_xyzmin, m_lo, m_n_rz_azimuthal_modes, m_nox, m_galerkin_interpolation); m_get_externalEB(i, ex, ey, ez, bx, by, bz); diff --git a/Source/Particles/ElementaryProcess/Ionization.cpp b/Source/Particles/ElementaryProcess/Ionization.cpp index b7b91e4d4e3..0568e302eec 100644 --- a/Source/Particles/ElementaryProcess/Ionization.cpp +++ b/Source/Particles/ElementaryProcess/Ionization.cpp @@ -76,11 +76,10 @@ IonizationFilterFunc::IonizationFilterFunc (const WarpXParIter& a_pti, int lev, box.grow(ngEB); const std::array& dx = WarpX::CellSize(std::max(lev, 0)); - m_dx_arr = {dx[0], dx[1], dx[2]}; + m_dinv = amrex::XDim3{1._rt/dx[0], 1._rt/dx[1], 1._rt/dx[2]}; // Lower corner of tile box physical domain (take into account Galilean shift) - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - m_xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + m_xyzmin = WarpX::LowerCorner(box, lev, 0._rt); m_lo = amrex::lbound(box); } diff --git a/Source/Particles/ElementaryProcess/QEDPairGeneration.H b/Source/Particles/ElementaryProcess/QEDPairGeneration.H index d550c5f03ee..180fdf0fb35 100644 --- a/Source/Particles/ElementaryProcess/QEDPairGeneration.H +++ b/Source/Particles/ElementaryProcess/QEDPairGeneration.H @@ -144,7 +144,7 @@ public: doGatherShapeN(xp, yp, zp, ex, ey, ez, bx, by, bz, m_ex_arr, m_ey_arr, m_ez_arr, m_bx_arr, m_by_arr, m_bz_arr, m_ex_type, m_ey_type, m_ez_type, m_bx_type, m_by_type, m_bz_type, - m_dx_arr, m_xyzmin_arr, m_lo, m_n_rz_azimuthal_modes, + m_dinv, m_xyzmin, m_lo, m_n_rz_azimuthal_modes, m_nox, m_galerkin_interpolation); //Despite the names of the variables, positrons and electrons @@ -198,8 +198,8 @@ private: amrex::IndexType m_by_type; amrex::IndexType m_bz_type; - amrex::GpuArray m_dx_arr; - amrex::GpuArray m_xyzmin_arr; + amrex::XDim3 m_dinv; + amrex::XDim3 m_xyzmin; bool m_galerkin_interpolation; int m_nox; diff --git a/Source/Particles/ElementaryProcess/QEDPairGeneration.cpp b/Source/Particles/ElementaryProcess/QEDPairGeneration.cpp index 2b380d454f4..82546e43ef4 100644 --- a/Source/Particles/ElementaryProcess/QEDPairGeneration.cpp +++ b/Source/Particles/ElementaryProcess/QEDPairGeneration.cpp @@ -64,11 +64,10 @@ PairGenerationTransformFunc (BreitWheelerGeneratePairs const generate_functor, box.grow(ngEB); const std::array& dx = WarpX::CellSize(std::max(lev, 0)); - m_dx_arr = {dx[0], dx[1], dx[2]}; + m_dinv = amrex::XDim3{1._rt/dx[0], 1._rt/dx[1], 1._rt/dx[2]}; // Lower corner of tile box physical domain (take into account Galilean shift) - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - m_xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + m_xyzmin = WarpX::LowerCorner(box, lev, 0._rt); m_lo = amrex::lbound(box); } diff --git a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H index dd5f57a8f24..514526374bd 100644 --- a/Source/Particles/ElementaryProcess/QEDPhotonEmission.H +++ b/Source/Particles/ElementaryProcess/QEDPhotonEmission.H @@ -156,7 +156,7 @@ public: doGatherShapeN(xp, yp, zp, ex, ey, ez, bx, by, bz, m_ex_arr, m_ey_arr, m_ez_arr, m_bx_arr, m_by_arr, m_bz_arr, m_ex_type, m_ey_type, m_ez_type, m_bx_type, m_by_type, m_bz_type, - m_dx_arr, m_xyzmin_arr, m_lo, m_n_rz_azimuthal_modes, + m_dinv, m_xyzmin, m_lo, m_n_rz_azimuthal_modes, m_nox, m_galerkin_interpolation); auto& ux = src.m_rdata[PIdx::ux][i_src]; @@ -209,8 +209,8 @@ private: amrex::IndexType m_by_type; amrex::IndexType m_bz_type; - amrex::GpuArray m_dx_arr; - amrex::GpuArray m_xyzmin_arr; + amrex::XDim3 m_dinv; + amrex::XDim3 m_xyzmin; bool m_galerkin_interpolation; int m_nox; diff --git a/Source/Particles/ElementaryProcess/QEDPhotonEmission.cpp b/Source/Particles/ElementaryProcess/QEDPhotonEmission.cpp index 077a4659ce5..688c3c0184d 100644 --- a/Source/Particles/ElementaryProcess/QEDPhotonEmission.cpp +++ b/Source/Particles/ElementaryProcess/QEDPhotonEmission.cpp @@ -67,13 +67,10 @@ PhotonEmissionTransformFunc (QuantumSynchrotronGetOpticalDepth opt_depth_functor box.grow(ngEB); const std::array& dx = WarpX::CellSize(std::max(lev, 0)); - m_dx_arr = {dx[0], dx[1], dx[2]}; + m_dinv = amrex::XDim3{1._rt/dx[0], 1._rt/dx[1], 1._rt/dx[2]}; // Lower corner of tile box physical domain (take into account Galilean shift) - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); - m_xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - - + m_xyzmin = WarpX::LowerCorner(box, lev, 0._rt); m_lo = amrex::lbound(box); } diff --git a/Source/Particles/Gather/FieldGather.H b/Source/Particles/Gather/FieldGather.H index b5bd4376ba1..4b4590b8642 100644 --- a/Source/Particles/Gather/FieldGather.H +++ b/Source/Particles/Gather/FieldGather.H @@ -28,16 +28,16 @@ * \param bx_arr,by_arr,bz_arr Array4 of the magnetic field, either full array or tile. * \param ex_type,ey_type,ez_type IndexType of the electric field * \param bx_type,by_type,bz_type IndexType of the magnetic field - * \param dx 3D cell spacing - * \param xyzmin Physical lower bounds of domain in x, y, z. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE -void doGatherShapeN (const amrex::ParticleReal xp, - const amrex::ParticleReal yp, - const amrex::ParticleReal zp, +void doGatherShapeN ([[maybe_unused]] const amrex::ParticleReal xp, + [[maybe_unused]] const amrex::ParticleReal yp, + [[maybe_unused]] const amrex::ParticleReal zp, amrex::ParticleReal& Exp, amrex::ParticleReal& Eyp, amrex::ParticleReal& Ezp, @@ -56,41 +56,13 @@ void doGatherShapeN (const amrex::ParticleReal xp, const amrex::IndexType bx_type, const amrex::IndexType by_type, const amrex::IndexType bz_type, - const amrex::GpuArray& dx, - const amrex::GpuArray& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3& lo, - const int n_rz_azimuthal_modes) + [[maybe_unused]] const int n_rz_azimuthal_modes) { using namespace amrex; -#if defined(WARPX_DIM_XZ) - amrex::ignore_unused(yp); -#endif - -#if defined(WARPX_DIM_1D_Z) - amrex::ignore_unused(xp,yp); -#endif - -#ifndef WARPX_DIM_RZ - amrex::ignore_unused(n_rz_azimuthal_modes); -#endif - -#if (AMREX_SPACEDIM >= 2) - const amrex::Real dxi = 1.0_rt/dx[0]; -#endif - const amrex::Real dzi = 1.0_rt/dx[2]; -#if defined(WARPX_DIM_3D) - const amrex::Real dyi = 1.0_rt/dx[1]; -#endif - -#if (AMREX_SPACEDIM >= 2) - const amrex::Real xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - const amrex::Real ymin = xyzmin[1]; -#endif - const amrex::Real zmin = xyzmin[2]; - constexpr int zdir = WARPX_ZINDEX; constexpr int NODE = amrex::IndexType::NODE; constexpr int CELL = amrex::IndexType::CELL; @@ -105,9 +77,9 @@ void doGatherShapeN (const amrex::ParticleReal xp, // Get particle position #ifdef WARPX_DIM_RZ const amrex::Real rp = std::sqrt(xp*xp + yp*yp); - const amrex::Real x = (rp - xmin)*dxi; + const amrex::Real x = (rp - xyzmin.x)*dinv.x; #else - const amrex::Real x = (xp-xmin)*dxi; + const amrex::Real x = (xp-xyzmin.x)*dinv.x; #endif // j_[eb][xyz] leftmost grid point in x that the particle touches for the centering of each current @@ -151,7 +123,7 @@ void doGatherShapeN (const amrex::ParticleReal xp, #if defined(WARPX_DIM_3D) // y direction - const amrex::Real y = (yp-ymin)*dyi; + const amrex::Real y = (yp-xyzmin.y)*dinv.y; amrex::Real sy_node[depos_order + 1]; amrex::Real sy_cell[depos_order + 1]; amrex::Real sy_node_v[depos_order + 1 - galerkin_interpolation]; @@ -187,7 +159,7 @@ void doGatherShapeN (const amrex::ParticleReal xp, #endif // z direction - const amrex::Real z = (zp-zmin)*dzi; + const amrex::Real z = (zp-xyzmin.z)*dinv.z; amrex::Real sz_node[depos_order + 1]; amrex::Real sz_cell[depos_order + 1]; amrex::Real sz_node_v[depos_order + 1 - galerkin_interpolation]; @@ -464,10 +436,10 @@ void doGatherShapeN (const amrex::ParticleReal xp, * \param Bx_arr,By_arr,Bz_arr Array4 of the magnetic field, either full array or tile. * \param Ex_type,Ey_type,Ez_type IndexType of the electric field * \param Bx_type,By_type,Bz_type IndexType of the magnetic field - * \param dx 3D cell spacing - * \param xyzmin Physical lower bounds of domain in x, y, z. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. - * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry + * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry */ template AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE @@ -496,8 +468,8 @@ void doGatherShapeNEsirkepovStencilImplicit ( [[maybe_unused]] const amrex::IndexType Bx_type, [[maybe_unused]] const amrex::IndexType By_type, [[maybe_unused]] const amrex::IndexType Bz_type, - const amrex::GpuArray& dx, - const amrex::GpuArray& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3& lo, const int n_rz_azimuthal_modes) { @@ -506,19 +478,6 @@ void doGatherShapeNEsirkepovStencilImplicit ( ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(WARPX_DIM_1D_Z) - Real const dxi = 1.0_rt / dx[0]; -#endif -#if !defined(WARPX_DIM_1D_Z) - Real const xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - Real const dyi = 1.0_rt / dx[1]; - Real const ymin = xyzmin[1]; -#endif - Real const dzi = 1.0_rt / dx[2]; - Real const zmin = xyzmin[2]; - #if !defined(WARPX_DIM_1D_Z) Real constexpr one_third = 1.0_rt / 3.0_rt; Real constexpr one_sixth = 1.0_rt / 6.0_rt; @@ -553,23 +512,23 @@ void doGatherShapeNEsirkepovStencilImplicit ( } const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; // Keep these double to avoid bug in single precision - double const x_new = (rp_new - xmin)*dxi; - double const x_old = (rp_old - xmin)*dxi; + double const x_new = (rp_new - xyzmin.x)*dinv.x; + double const x_old = (rp_old - xyzmin.x)*dinv.x; #else #if !defined(WARPX_DIM_1D_Z) // Keep these double to avoid bug in single precision - double const x_new = (xp_np1 - xmin)*dxi; - double const x_old = (xp_n - xmin)*dxi; + double const x_new = (xp_np1 - xyzmin.x)*dinv.x; + double const x_old = (xp_n - xyzmin.x)*dinv.x; #endif #endif #if defined(WARPX_DIM_3D) // Keep these double to avoid bug in single precision - double const y_new = (yp_np1 - ymin)*dyi; - double const y_old = (yp_n - ymin)*dyi; + double const y_new = (yp_np1 - xyzmin.y)*dinv.y; + double const y_old = (yp_n - xyzmin.y)*dinv.y; #endif // Keep these double to avoid bug in single precision - double const z_new = (zp_np1 - zmin)*dzi; - double const z_old = (zp_n - zmin)*dzi; + double const z_new = (zp_np1 - xyzmin.z)*dinv.z; + double const z_old = (zp_n - xyzmin.z)*dinv.z; // Shape factor arrays // Note that there are extra values above and below @@ -894,8 +853,8 @@ void doGatherShapeNEsirkepovStencilImplicit ( * \param Bx_arr,By_arr,Bz_arr Array4 of the magnetic field, either full array or tile. * \param Ex_type,Ey_type,Ez_type IndexType of the electric field * \param Bx_type,By_type,Bz_type IndexType of the magnetic field - * \param dx 3D cell spacing - * \param xyzmin Physical lower bounds of domain in x, y, z. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry */ @@ -926,8 +885,8 @@ void doGatherPicnicShapeN ( [[maybe_unused]] const amrex::IndexType Bx_type, [[maybe_unused]] const amrex::IndexType By_type, [[maybe_unused]] const amrex::IndexType Bz_type, - const amrex::GpuArray& dx, - const amrex::GpuArray& xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3& lo, const int n_rz_azimuthal_modes) { @@ -936,19 +895,6 @@ void doGatherPicnicShapeN ( ignore_unused(n_rz_azimuthal_modes); #endif -#if !defined(WARPX_DIM_1D_Z) - Real const dxi = 1.0_rt / dx[0]; -#endif -#if !defined(WARPX_DIM_1D_Z) - Real const xmin = xyzmin[0]; -#endif -#if defined(WARPX_DIM_3D) - Real const dyi = 1.0_rt / dx[1]; - Real const ymin = xyzmin[1]; -#endif - Real const dzi = 1.0_rt / dx[2]; - Real const zmin = xyzmin[2]; - #if !defined(WARPX_DIM_1D_Z) const ParticleReal xp_np1 = 2._prt*xp_nph - xp_n; #endif @@ -983,25 +929,25 @@ void doGatherPicnicShapeN ( } const Complex xy_mid0 = Complex{costheta_mid, sintheta_mid}; // Keep these double to avoid bug in single precision - double const x_new = (rp_new - xmin)*dxi; - double const x_old = (rp_old - xmin)*dxi; - double const x_bar = (rp_mid - xmin)*dxi; + double const x_new = (rp_new - xyzmin.x)*dinv.x; + double const x_old = (rp_old - xyzmin.x)*dinv.x; + double const x_bar = (rp_mid - xyzmin.x)*dinv.x; #elif !defined(WARPX_DIM_1D_Z) // Keep these double to avoid bug in single precision - double const x_new = (xp_np1 - xmin)*dxi; - double const x_old = (xp_n - xmin)*dxi; - double const x_bar = (xp_nph - xmin)*dxi; + double const x_new = (xp_np1 - xyzmin.x)*dinv.x; + double const x_old = (xp_n - xyzmin.x)*dinv.x; + double const x_bar = (xp_nph - xyzmin.x)*dinv.x; #endif #if defined(WARPX_DIM_3D) // Keep these double to avoid bug in single precision - double const y_new = (yp_np1 - ymin)*dyi; - double const y_old = (yp_n - ymin)*dyi; - double const y_bar = (yp_nph - ymin)*dyi; + double const y_new = (yp_np1 - xyzmin.y)*dinv.y; + double const y_old = (yp_n - xyzmin.y)*dinv.y; + double const y_bar = (yp_nph - xyzmin.y)*dinv.y; #endif // Keep these double to avoid bug in single precision - double const z_new = (zp_np1 - zmin)*dzi; - double const z_old = (zp_n - zmin)*dzi; - double const z_bar = (zp_nph - zmin)*dzi; + double const z_new = (zp_np1 - xyzmin.z)*dinv.z; + double const z_old = (zp_n - xyzmin.z)*dinv.z; + double const z_bar = (zp_nph - xyzmin.z)*dinv.z; // 1) Determine the number of segments. // 2) Loop over segments and gather electric field. @@ -1567,8 +1513,8 @@ void doGatherPicnicShapeN ( * \param exfab,eyfab,ezfab Array4 of the electric field, either full array or tile. * \param bxfab,byfab,bzfab Array4 of the magnetic field, either full array or tile. * \param np_to_gather Number of particles for which field is gathered. - * \param dx 3D cell size - * \param xyzmin Physical lower bounds of domain. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry */ @@ -1585,15 +1531,12 @@ void doGatherShapeN(const GetParticlePosition& getPosition, amrex::FArrayBox const * const byfab, amrex::FArrayBox const * const bzfab, const long np_to_gather, - const std::array& dx, - const std::array xyzmin, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3 lo, const int n_rz_azimuthal_modes) { - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - amrex::Array4 const& ex_arr = exfab->array(); amrex::Array4 const& ey_arr = eyfab->array(); amrex::Array4 const& ez_arr = ezfab->array(); @@ -1622,7 +1565,7 @@ void doGatherShapeN(const GetParticlePosition& getPosition, xp, yp, zp, Exp[ip], Eyp[ip], Ezp[ip], Bxp[ip], Byp[ip], Bzp[ip], ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } ); } @@ -1637,8 +1580,8 @@ void doGatherShapeN(const GetParticlePosition& getPosition, * \param bx_arr,by_arr,bz_arr Array4 of the magnetic field, either full array or tile. * \param ex_type,ey_type,ez_type IndexType of the electric field * \param bx_type,by_type,bz_type IndexType of the magnetic field - * \param dx_arr 3D cell spacing - * \param xyzmin_arr Physical lower bounds of domain in x, y, z. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry * \param nox order of the particle shape function @@ -1666,8 +1609,8 @@ void doGatherShapeN (const amrex::ParticleReal xp, const amrex::IndexType bx_type, const amrex::IndexType by_type, const amrex::IndexType bz_type, - const amrex::GpuArray& dx_arr, - const amrex::GpuArray& xyzmin_arr, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3& lo, const int n_rz_azimuthal_modes, const int nox, @@ -1678,44 +1621,44 @@ void doGatherShapeN (const amrex::ParticleReal xp, doGatherShapeN<1,1>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 2) { doGatherShapeN<2,1>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 3) { doGatherShapeN<3,1>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 4) { doGatherShapeN<4,1>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } } else { if (nox == 1) { doGatherShapeN<1,0>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 2) { doGatherShapeN<2,0>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 3) { doGatherShapeN<3,0>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 4) { doGatherShapeN<4,0>(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } } } @@ -1732,8 +1675,8 @@ void doGatherShapeN (const amrex::ParticleReal xp, * \param bx_arr,by_arr,bz_arr Array4 of the magnetic field, either full array or tile. * \param ex_type,ey_type,ez_type IndexType of the electric field * \param bx_type,by_type,bz_type IndexType of the magnetic field - * \param dx_arr 3D cell spacing - * \param xyzmin_arr Physical lower bounds of domain in x, y, z. + * \param dinv 3D cell size inverse + * \param xyzmin The lower bounds of the domain * \param lo Index lower bounds of domain. * \param n_rz_azimuthal_modes Number of azimuthal modes when using RZ geometry * \param nox order of the particle shape function @@ -1766,8 +1709,8 @@ void doGatherShapeNImplicit ( const amrex::IndexType bx_type, const amrex::IndexType by_type, const amrex::IndexType bz_type, - const amrex::GpuArray& dx_arr, - const amrex::GpuArray& xyzmin_arr, + const amrex::XDim3 & dinv, + const amrex::XDim3 & xyzmin, const amrex::Dim3& lo, const int n_rz_azimuthal_modes, const int nox, @@ -1779,25 +1722,25 @@ void doGatherShapeNImplicit ( Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 2) { doGatherShapeNEsirkepovStencilImplicit<2>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 3) { doGatherShapeNEsirkepovStencilImplicit<3>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 4) { doGatherShapeNEsirkepovStencilImplicit<4>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } } else if (depos_type==3) { // CurrentDepositionAlgo::Villasenor @@ -1806,25 +1749,25 @@ void doGatherShapeNImplicit ( Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 2) { doGatherPicnicShapeN<2>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 3) { doGatherPicnicShapeN<3>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 4) { doGatherPicnicShapeN<4>(xp_n, yp_n, zp_n, xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } } else if (depos_type==1) { // CurrentDepositionAlgo::Direct @@ -1832,22 +1775,22 @@ void doGatherShapeNImplicit ( doGatherShapeN<1,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 2) { doGatherShapeN<2,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 3) { doGatherShapeN<3,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } else if (nox == 4) { doGatherShapeN<4,0>(xp_nph, yp_nph, zp_nph, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes); + dinv, xyzmin, lo, n_rz_azimuthal_modes); } } } diff --git a/Source/Particles/Gather/GetExternalFields.cpp b/Source/Particles/Gather/GetExternalFields.cpp index 8e3c679aa3a..bb55f79f394 100644 --- a/Source/Particles/Gather/GetExternalFields.cpp +++ b/Source/Particles/Gather/GetExternalFields.cpp @@ -83,6 +83,18 @@ GetExternalEBField::GetExternalEBField (const WarpXParIter& a_pti, long a_offset m_repeated_plasma_lens_strengths_B = mypc.d_repeated_plasma_lens_strengths_B.data(); } + // When the external particle fields are read from file, + // the external fields are not added directly inside the gather kernel. + // (Hence of `None`, which ensures that the gather kernel is compiled without support + // for external fields.) Instead, the external fields are added to the MultiFab + // Efield_aux and Bfield_aux before the particles gather from these MultiFab. + if (mypc.m_E_ext_particle_s == "read_from_file") { + m_Etype = ExternalFieldInitType::None; + } + if (mypc.m_B_ext_particle_s == "read_from_file") { + m_Btype = ExternalFieldInitType::None; + } + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_Etype != Unknown, "Unknown E_ext_particle_init_style"); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_Btype != Unknown, "Unknown B_ext_particle_init_style"); diff --git a/Source/Particles/LaserParticleContainer.H b/Source/Particles/LaserParticleContainer.H index fac94ff20a3..197cb897602 100644 --- a/Source/Particles/LaserParticleContainer.H +++ b/Source/Particles/LaserParticleContainer.H @@ -94,7 +94,7 @@ public: amrex::ParticleReal * AMREX_RESTRICT puzp, amrex::ParticleReal const * AMREX_RESTRICT pwp, amrex::Real const * AMREX_RESTRICT amplitude, - amrex::Real dt); + amrex::Real dt, PushType push_type=PushType::Explicit); protected: diff --git a/Source/Particles/LaserParticleContainer.cpp b/Source/Particles/LaserParticleContainer.cpp index bf846c674f0..bd266ab368a 100644 --- a/Source/Particles/LaserParticleContainer.cpp +++ b/Source/Particles/LaserParticleContainer.cpp @@ -19,6 +19,7 @@ #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" +#include "WarpX.H" #include @@ -652,7 +653,7 @@ LaserParticleContainer::Evolve (int lev, // Calculate the corresponding momentum and position for the particles update_laser_particle(pti, static_cast(np), uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(), wp.dataPtr(), - amplitude_E.dataPtr(), dt); + amplitude_E.dataPtr(), dt, push_type ); WARPX_PROFILE_VAR_STOP(blp_pp); // Current Deposition @@ -718,7 +719,7 @@ LaserParticleContainer::ComputeSpacing (int lev, Real& Sx, Real& Sy) const #if !defined(WARPX_DIM_RZ) constexpr float small_float_coeff = 1.e-25f; constexpr double small_double_coeff = 1.e-50; - constexpr Real small_coeff = std::is_same::value ? + constexpr Real small_coeff = std::is_same_v ? static_cast(small_float_coeff) : static_cast(small_double_coeff); const auto eps = static_cast(dx[0]*small_coeff); @@ -850,7 +851,7 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti, ParticleReal * AMREX_RESTRICT const puzp, ParticleReal const * AMREX_RESTRICT const pwp, Real const * AMREX_RESTRICT const amplitude, - const Real dt) + const Real dt, PushType push_type) { const auto GetPosition = GetParticlePosition(pti); auto SetPosition = SetParticlePosition(pti); @@ -862,6 +863,26 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti, const Real tmp_nvec_1 = m_nvec[1]; const Real tmp_nvec_2 = m_nvec[2]; + // When using the implicit solver, this function is called multiple times per timestep + // (within the linear and nonlinear solver). Thus, the position of the particles needs to be reset + // to the initial position (at the beginning of the timestep), before updating the particle position +#if (AMREX_SPACEDIM >= 2) + ParticleReal* x_n = nullptr; + if (push_type == PushType::Implicit) { + x_n = pti.GetAttribs(particle_comps["x_n"]).dataPtr(); + } +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + ParticleReal* y_n = nullptr; + if (push_type == PushType::Implicit) { + y_n = pti.GetAttribs(particle_comps["y_n"]).dataPtr(); + } +#endif + ParticleReal* z_n = nullptr; + if (push_type == PushType::Implicit) { + z_n = pti.GetAttribs(particle_comps["z_n"]).dataPtr(); + } + // Copy member variables to tmp copies for GPU runs. const Real tmp_mobility = m_mobility; const Real gamma_boost = WarpX::gamma_boost; @@ -893,15 +914,33 @@ LaserParticleContainer::update_laser_particle (WarpXParIter& pti, puzp[i] = gamma * vz; // Push the the particle positions - ParticleReal x, y, z; - GetPosition(i, x, y, z); + + // When using the implicit solver, this function is called multiple times per timestep + // (within the linear and nonlinear solver). Thus, the position of the particles needs to be reset + // to the initial position (at the beginning of the timestep), before updating the particle position + + ParticleReal x=0., y=0., z=0.; + if (push_type == PushType::Explicit) { + GetPosition(i, x, y, z); + } + #if !defined(WARPX_DIM_1D_Z) + if (push_type == PushType::Implicit) { + x = x_n[i]; + } x += vx * dt; #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + if (push_type == PushType::Implicit) { + y = y_n[i]; + } y += vy * dt; #endif + if (push_type == PushType::Implicit) { + z = z_n[i]; + } z += vz * dt; + SetPosition(i, x, y, z); } ); diff --git a/Source/Particles/ParticleBoundaryBuffer.cpp b/Source/Particles/ParticleBoundaryBuffer.cpp index 345a0bfa592..25068b2e65c 100644 --- a/Source/Particles/ParticleBoundaryBuffer.cpp +++ b/Source/Particles/ParticleBoundaryBuffer.cpp @@ -160,6 +160,9 @@ struct FindEmbeddedBoundaryIntersection { #else amrex::ignore_unused(x_temp, y_temp, z_temp,normal); #endif + + // flip id to positive in destination + amrex::ParticleIDWrapper{dst.m_idcpu[dst_i]}.make_valid(); } }; #endif @@ -199,6 +202,9 @@ struct CopyAndTimestamp { dst.m_runtime_rdata[m_normal_index][dst_i]= n[0]; dst.m_runtime_rdata[m_normal_index+1][dst_i]= n[1]; dst.m_runtime_rdata[m_normal_index+2][dst_i]= n[2]; + + // flip id to positive in destination + amrex::ParticleIDWrapper{dst.m_idcpu[dst_i]}.make_valid(); } }; @@ -384,7 +390,15 @@ void ParticleBoundaryBuffer::gatherParticlesFromDomainBoundaries (MultiParticleC buffer[i].AddRealComp("ny", false); buffer[i].AddRealComp("nz", false); } + auto& species_buffer = buffer[i]; + for (int lev = 0; lev < pc.numLevels(); ++lev){ + for(PIter pti(pc, lev); pti.isValid(); ++pti){ + species_buffer.DefineAndReturnParticleTile( + lev, pti.index(), pti.LocalTileIndex()); + } + } + for (int lev = 0; lev < pc.numLevels(); ++lev) { const auto& plevel = pc.GetParticles(lev); @@ -468,7 +482,15 @@ void ParticleBoundaryBuffer::gatherParticlesFromEmbeddedBoundaries ( buffer[i].AddRealComp("nz", false); } + auto& species_buffer = buffer[i]; + for (int lev = 0; lev < pc.numLevels(); ++lev){ + for(PIter pti(pc, lev); pti.isValid(); ++pti){ + species_buffer.DefineAndReturnParticleTile( + lev, pti.index(), pti.LocalTileIndex()); + } + } + for (int lev = 0; lev < pc.numLevels(); ++lev) { const auto& plevel = pc.GetParticles(lev); diff --git a/Source/Particles/PhotonParticleContainer.cpp b/Source/Particles/PhotonParticleContainer.cpp index aa9f04be224..1f15d5210f5 100644 --- a/Source/Particles/PhotonParticleContainer.cpp +++ b/Source/Particles/PhotonParticleContainer.cpp @@ -93,8 +93,8 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, int lev, int gather_lev, amrex::Real dt, ScaleFields /*scaleFields*/, DtType a_dt_type) { - // Get cell size on gather_lev - const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); + // Get inverse cell size on gather_lev + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(gather_lev,0)); // Get box from which field is gathered. // If not gathering from the finest level, the box is coarsened. @@ -142,7 +142,7 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; // Lower corner of tile box physical domain (take into account Galilean shift) - const std::array& xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); const Dim3 lo = lbound(box); @@ -150,9 +150,6 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, const int nox = WarpX::nox; const int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - amrex::Array4 const& ex_arr = exfab->array(); amrex::Array4 const& ey_arr = eyfab->array(); amrex::Array4 const& ez_arr = ezfab->array(); @@ -201,7 +198,7 @@ PhotonParticleContainer::PushPX (WarpXParIter& pti, doGatherShapeN(x, y, z, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, + dinv, xyzmin, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); } diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index e48645c70a1..f1900257035 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -61,6 +61,7 @@ #include #include #include +#include #include #include #include @@ -1064,7 +1065,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int static_cast(std::round((overlap_realbox.lo(dir)-problo[dir])/dx[dir])); // shifted is exact in non-moving-window direction. That's all we care. } - if (no_overlap == 1) { + if (no_overlap) { continue; // Go to the next tile } @@ -1639,7 +1640,7 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // shifted is exact in non-moving-window direction. That's all we care. } } - if (no_overlap == 1) { + if (no_overlap) { continue; // Go to the next tile } @@ -2562,7 +2563,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, if (do_not_push) { return; } - const std::array& dx = WarpX::CellSize(std::max(lev,0)); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(lev,0)); #ifdef AMREX_USE_OMP #pragma omp parallel @@ -2594,7 +2595,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, const amrex::ParticleReal By_external_particle = m_B_external_particle[1]; const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, lev, 0._rt); const Dim3 lo = lbound(box); @@ -2602,9 +2603,6 @@ PhysicalParticleContainer::PushP (int lev, Real dt, const int nox = WarpX::nox; const int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - amrex::Array4 const& ex_arr = exfab.array(); amrex::Array4 const& ey_arr = eyfab.array(); amrex::Array4 const& ez_arr = ezfab.array(); @@ -2661,7 +2659,7 @@ PhysicalParticleContainer::PushP (int lev, Real dt, doGatherShapeN(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, + dinv, xyzmin, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); } @@ -2759,7 +2757,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, if (np_to_push == 0) { return; } // Get cell size on gather_lev - const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(gather_lev,0)); // Get box from which field is gathered. // If not gathering from the finest level, the box is coarsened. @@ -2787,7 +2785,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; // Lower corner of tile box physical domain (take into account Galilean shift) - const std::array& xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); const Dim3 lo = lbound(box); @@ -2795,9 +2793,6 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, const int nox = WarpX::nox; const int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - amrex::Array4 const& ex_arr = exfab->array(); amrex::Array4 const& ey_arr = eyfab->array(); amrex::Array4 const& ez_arr = ezfab->array(); @@ -2912,7 +2907,7 @@ PhysicalParticleContainer::PushPX (WarpXParIter& pti, doGatherShapeN(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, + dinv, xyzmin, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); } @@ -3012,7 +3007,7 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, if (np_to_push == 0) { return; } // Get cell size on gather_lev - const std::array& dx = WarpX::CellSize(std::max(gather_lev,0)); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(gather_lev,0)); // Get box from which field is gathered. // If not gathering from the finest level, the box is coarsened. @@ -3039,7 +3034,7 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; // Lower corner of tile box physical domain (take into account Galilean shift) - const std::array& xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, gather_lev, 0._rt); const Dim3 lo = lbound(box); @@ -3047,9 +3042,6 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, const int nox = WarpX::nox; const int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; - amrex::Array4 const& ex_arr = exfab->array(); amrex::Array4 const& ey_arr = eyfab->array(); amrex::Array4 const& ez_arr = ezfab->array(); @@ -3123,6 +3115,12 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, const int qed_runtime_flag = no_qed; #endif + const int max_iterations = WarpX::max_particle_its_in_implicit_scheme; + const amrex::ParticleReal particle_tolerance = WarpX::particle_tol_in_implicit_scheme; + + amrex::Gpu::Buffer unconverged_particles({0}); + amrex::Long* unconverged_particles_ptr = unconverged_particles.data(); + // Using this version of ParallelFor with compile time options // improves performance when qed or external EB are not used by reducing // register pressure. @@ -3134,107 +3132,163 @@ PhysicalParticleContainer::ImplicitPushXP (WarpXParIter& pti, { // Position advance starts from the position at the start of the step // but uses the most recent velocity. + #if (AMREX_SPACEDIM >= 2) amrex::ParticleReal xp = x_n[ip]; const amrex::ParticleReal xp_n = x_n[ip]; #else - amrex::ParticleReal xp = 0._rt; + const amrex::ParticleReal xp = 0._rt; const amrex::ParticleReal xp_n = 0._rt; #endif #if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) amrex::ParticleReal yp = y_n[ip]; const amrex::ParticleReal yp_n = y_n[ip]; #else - amrex::ParticleReal yp = 0._rt; + const amrex::ParticleReal yp = 0._rt; const amrex::ParticleReal yp_n = 0._rt; #endif amrex::ParticleReal zp = z_n[ip]; const amrex::ParticleReal zp_n = z_n[ip]; - UpdatePositionImplicit(xp, yp, zp, ux_n[ip], uy_n[ip], uz_n[ip], ux[ip], uy[ip], uz[ip], 0.5_rt*dt); - setPosition(ip, xp, yp, zp); + amrex::ParticleReal dxp, dxp_save; + amrex::ParticleReal dyp, dyp_save; + amrex::ParticleReal dzp, dzp_save; + auto idxg2 = static_cast(dinv.x*dinv.x); + auto idyg2 = static_cast(dinv.y*dinv.y); + auto idzg2 = static_cast(dinv.z*dinv.z); - amrex::ParticleReal Exp = Ex_external_particle; - amrex::ParticleReal Eyp = Ey_external_particle; - amrex::ParticleReal Ezp = Ez_external_particle; - amrex::ParticleReal Bxp = Bx_external_particle; - amrex::ParticleReal Byp = By_external_particle; - amrex::ParticleReal Bzp = Bz_external_particle; + amrex::ParticleReal step_norm = 1._prt; + for (int iter=0; iter(ux[ip], uy[ip], uz[ip], - Exp, Eyp, Ezp, Bxp, Byp, Bzp, - ion_lev ? ion_lev[ip] : 1, - m, q, pusher_algo, do_crr, -#ifdef WARPX_QED - t_chi_max, + if (!do_sync) #endif - dt); - } -#ifdef WARPX_QED - else { - if constexpr (qed_control == has_qed) { - doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip], + { + doParticleMomentumPush<0>(ux[ip], uy[ip], uz[ip], Exp, Eyp, Ezp, Bxp, Byp, Bzp, ion_lev ? ion_lev[ip] : 1, m, q, pusher_algo, do_crr, +#ifdef WARPX_QED t_chi_max, +#endif dt); } - } +#ifdef WARPX_QED + else { + if constexpr (qed_control == has_qed) { + doParticleMomentumPush<1>(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp, Bxp, Byp, Bzp, + ion_lev ? ion_lev[ip] : 1, + m, q, pusher_algo, do_crr, + t_chi_max, + dt); + } + } #endif #ifdef WARPX_QED - [[maybe_unused]] auto foo_local_has_quantum_sync = local_has_quantum_sync; - [[maybe_unused]] auto *foo_podq = p_optical_depth_QSR; - [[maybe_unused]] const auto& foo_evolve_opt = evolve_opt; // have to do all these for nvcc - if constexpr (qed_control == has_qed) { - if (local_has_quantum_sync) { - evolve_opt(ux[ip], uy[ip], uz[ip], - Exp, Eyp, Ezp,Bxp, Byp, Bzp, - dt, p_optical_depth_QSR[ip]); + [[maybe_unused]] auto foo_local_has_quantum_sync = local_has_quantum_sync; + [[maybe_unused]] auto *foo_podq = p_optical_depth_QSR; + [[maybe_unused]] const auto& foo_evolve_opt = evolve_opt; // have to do all these for nvcc + if constexpr (qed_control == has_qed) { + if (local_has_quantum_sync) { + evolve_opt(ux[ip], uy[ip], uz[ip], + Exp, Eyp, Ezp,Bxp, Byp, Bzp, + dt, p_optical_depth_QSR[ip]); + } } - } #else amrex::ignore_unused(qed_control); #endif - // Take average to get the time centered value - ux[ip] = 0.5_rt*(ux[ip] + ux_n[ip]); - uy[ip] = 0.5_rt*(uy[ip] + uy_n[ip]); - uz[ip] = 0.5_rt*(uz[ip] + uz_n[ip]); + // Take average to get the time centered value + ux[ip] = 0.5_rt*(ux[ip] + ux_n[ip]); + uy[ip] = 0.5_rt*(uy[ip] + uy_n[ip]); + uz[ip] = 0.5_rt*(uz[ip] + uz_n[ip]); + + iter++; + + // particle did not converge + if ( iter > 1 && iter == max_iterations ) { +#if !defined(AMREX_USE_GPU) + std::stringstream convergenceMsg; + convergenceMsg << "Picard solver for particle failed to converge after " << + iter << " iterations. " << std::endl; + convergenceMsg << "Position step norm is " << step_norm << + " and the tolerance is " << particle_tolerance << std::endl; + convergenceMsg << " ux = " << ux[ip] << ", uy = " << uy[ip] << ", uz = " << uz[ip] << std::endl; + convergenceMsg << " xp = " << xp << ", yp = " << yp << ", zp = " << zp; + ablastr::warn_manager::WMRecordWarning("ImplicitPushXP", convergenceMsg.str()); +#endif + + // write signaling flag: how many particles did not converge? + amrex::Gpu::Atomic::Add(unconverged_particles_ptr, amrex::Long(1)); + } + + } // end Picard iterations }); + + auto const num_unconverged_particles = *(unconverged_particles.copyToHost()); + if (num_unconverged_particles > 0) { + ablastr::warn_manager::WMRecordWarning("ImplicitPushXP", + "Picard solver for " + + std::to_string(num_unconverged_particles) + + " particles failed to converge after " + + std::to_string(max_iterations) + " iterations." + ); + } } void diff --git a/Source/Particles/Pusher/UpdatePosition.H b/Source/Particles/Pusher/UpdatePosition.H index 6e2e82632ee..89c2de88e47 100644 --- a/Source/Particles/Pusher/UpdatePosition.H +++ b/Source/Particles/Pusher/UpdatePosition.H @@ -85,4 +85,46 @@ void UpdatePositionImplicit(amrex::ParticleReal& x, amrex::ParticleReal& y, amre z += uz * inv_gamma * dt; } +/** \brief Check particle position for convergence. This is used by the theta-implicit + * and semi-implicit time solvers to obtain a self-consistent time-centered update + * of the particles for given electric and magnetic fields on the grid. + */ +AMREX_GPU_HOST_DEVICE AMREX_INLINE +void PositionNorm( amrex::ParticleReal dxp, amrex::ParticleReal dyp, amrex::ParticleReal dzp, + amrex::ParticleReal& dxp_save, amrex::ParticleReal& dyp_save, amrex::ParticleReal& dzp_save, + amrex::ParticleReal idxg2, amrex::ParticleReal idyg2, amrex::ParticleReal idzg2, + amrex::ParticleReal& step_norm, const int iter ) +{ + using namespace amrex::literals; + +#if defined(WARPX_DIM_1D_Z) + amrex::ignore_unused(dxp, dxp_save, idxg2); +#endif +#if !defined(WARPX_DIM_3D) + amrex::ignore_unused(dyp, dyp_save, idyg2); +#endif + + if (iter==0) { step_norm = 1.0_prt; } + else { + step_norm = (dzp - dzp_save)*(dzp - dzp_save)*idzg2; +#if !defined(WARPX_DIM_1D_Z) + step_norm += (dxp - dxp_save)*(dxp - dxp_save)*idxg2; +#endif +#if defined(WARPX_DIM_3D) + step_norm += (dyp - dyp_save)*(dyp - dyp_save)*idyg2; +#elif defined(WARPX_DIM_RZ) + step_norm += (dyp - dyp_save)*(dyp - dyp_save)*idxg2; +#endif + step_norm = std::sqrt(step_norm); + } + dzp_save = dzp; +#if !defined(WARPX_DIM_1D_Z) + dxp_save = dxp; +#endif +#if defined(WARPX_DIM_3D) || defined(WARPX_DIM_RZ) + dyp_save = dyp; +#endif + +} + #endif // WARPX_PARTICLES_PUSHER_UPDATEPOSITION_H_ diff --git a/Source/Particles/Resampling/VelocityCoincidenceThinning.H b/Source/Particles/Resampling/VelocityCoincidenceThinning.H index b8d67d198a4..bb325734777 100644 --- a/Source/Particles/Resampling/VelocityCoincidenceThinning.H +++ b/Source/Particles/Resampling/VelocityCoincidenceThinning.H @@ -200,5 +200,6 @@ private: int m_ntheta, m_nphi; amrex::ParticleReal m_delta_ur; amrex::Vector m_delta_u; + amrex::ParticleReal m_cluster_weight = std::numeric_limits::max(); }; #endif // WARPX_VELOCITY_COINCIDENCE_THINNING_H_ diff --git a/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp b/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp index db4ddb801c6..2880ace200d 100644 --- a/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp +++ b/Source/Particles/Resampling/VelocityCoincidenceThinning.cpp @@ -24,6 +24,14 @@ VelocityCoincidenceThinning::VelocityCoincidenceThinning (const std::string& spe "Resampling min_ppc should be greater than or equal to 1" ); + amrex::ParticleReal target_weight = 0; + if (utils::parser::queryWithParser( + pp_species_name, "resampling_algorithm_target_weight", target_weight + )) { + // factor of 2 since each cluster is reduced to 2 particles + m_cluster_weight = target_weight * 2.0_prt; + } + std::string velocity_grid_type_str = "spherical"; pp_species_name.query( "resampling_algorithm_velocity_grid_type", velocity_grid_type_str @@ -84,7 +92,7 @@ void VelocityCoincidenceThinning::operator() (WarpXParIter& pti, const int lev, auto *const cell_offsets = bins.offsetsPtr(); const auto min_ppc = m_min_ppc; - + const auto cluster_weight = m_cluster_weight; const auto mass = pc->getMass(); // check if species mass > 0 @@ -207,10 +215,13 @@ void VelocityCoincidenceThinning::operator() (WarpXParIter& pti, const int lev, ux[part_idx], uy[part_idx], uz[part_idx], mass ); - // check if this is the last particle in the current momentum bin + // check if this is the last particle in the current momentum bin, + // or if the next particle would push the current cluster weight + // to exceed the maximum specified cluster weight if ( (i == cell_stop - 1) || (momentum_bin_number_data[sorted_indices_data[i]] != momentum_bin_number_data[sorted_indices_data[i + 1]]) + || (total_weight + w[indices[sorted_indices_data[i+1]]] > cluster_weight) ) { // check if the bin has more than 2 particles in it if ( particles_in_bin > 2 && total_weight > std::numeric_limits::min() ){ diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index eed556bf998..c3ec4c41131 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -337,7 +337,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, if (do_not_push) { return; } - const std::array& dx = WarpX::CellSize(std::max(lev,0)); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(lev,0)); #ifdef AMREX_USE_OMP #pragma omp parallel @@ -369,7 +369,6 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, const amrex::ParticleReal By_external_particle = m_B_external_particle[1]; const amrex::ParticleReal Bz_external_particle = m_B_external_particle[2]; - const std::array& xyzmin = WarpX::LowerCorner(box, lev, 0._rt); const Dim3 lo = lbound(box); @@ -377,8 +376,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, const int nox = WarpX::nox; const int n_rz_azimuthal_modes = WarpX::n_rz_azimuthal_modes; - const amrex::GpuArray dx_arr = {dx[0], dx[1], dx[2]}; - const amrex::GpuArray xyzmin_arr = {xyzmin[0], xyzmin[1], xyzmin[2]}; + const amrex::XDim3 xyzmin = WarpX::LowerCorner(box, lev, 0._rt); amrex::Array4 const& ex_arr = exfab.array(); amrex::Array4 const& ey_arr = eyfab.array(); @@ -445,7 +443,7 @@ RigidInjectedParticleContainer::PushP (int lev, Real dt, doGatherShapeN(xp, yp, zp, Exp, Eyp, Ezp, Bxp, Byp, Bzp, ex_arr, ey_arr, ez_arr, bx_arr, by_arr, bz_arr, ex_type, ey_type, ez_type, bx_type, by_type, bz_type, - dx_arr, xyzmin_arr, lo, n_rz_azimuthal_modes, + dinv, xyzmin, lo, n_rz_azimuthal_modes, nox, galerkin_interpolation); [[maybe_unused]] const auto& getExternalEB_tmp = getExternalEB; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index 8692dff3302..bdce18b7b2b 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -396,7 +396,8 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, amrex::numParticlesOutOfRange(pti, range) == 0, "Particles shape does not fit within tile (CPU) or guard cells (GPU) used for current deposition"); - const std::array& dx = WarpX::CellSize(std::max(depos_lev,0)); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(depos_lev,0)); + const amrex::ParticleReal q = this->charge; WARPX_PROFILE_VAR_NS("WarpXParticleContainer::DepositCurrent::Sorting", blp_sort); @@ -467,7 +468,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, // Note that this includes guard cells since it is after tilebox.ngrow const Dim3 lo = lbound(tilebox); // Take into account Galilean shift - const std::array& xyzmin = WarpX::LowerCorner(tilebox, depos_lev, 0.5_rt*dt); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(tilebox, depos_lev, 0.5_rt*dt); if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov || WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor) { @@ -544,28 +545,28 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, doDepositionSharedShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size); } else if (WarpX::nox == 2){ doDepositionSharedShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size); } else if (WarpX::nox == 3){ doDepositionSharedShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size); } else if (WarpX::nox == 4){ doDepositionSharedShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size); } @@ -580,25 +581,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, doEsirkepovDepositionShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doEsirkepovDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doEsirkepovDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doEsirkepovDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } else if (push_type == PushType::Implicit) { @@ -625,7 +626,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doChargeConservingDepositionShapeNImplicit<2>( @@ -633,7 +634,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doChargeConservingDepositionShapeNImplicit<3>( @@ -641,7 +642,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doChargeConservingDepositionShapeNImplicit<4>( @@ -649,7 +650,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } @@ -678,7 +679,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doVillasenorDepositionShapeNImplicit<2>( @@ -686,7 +687,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doVillasenorDepositionShapeNImplicit<3>( @@ -694,7 +695,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doVillasenorDepositionShapeNImplicit<4>( @@ -702,7 +703,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, GetPosition, wp.dataPtr() + offset, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dx, xyzmin, lo, q, + jx_arr, jy_arr, jz_arr, np_to_deposit, dt, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } @@ -717,25 +718,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, doVayDepositionShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doVayDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doVayDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doVayDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dx, xyzmin, lo, q, + jx_fab, jy_fab, jz_fab, np_to_deposit, dt, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } else { // Direct deposition @@ -744,25 +745,25 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, doDepositionShapeN<1>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doDepositionShapeN<2>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doDepositionShapeN<3>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doDepositionShapeN<4>( GetPosition, wp.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, relative_time, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } else if (push_type == PushType::Implicit) { @@ -775,7 +776,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 2){ doDepositionShapeNImplicit<2>( @@ -783,7 +784,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 3){ doDepositionShapeNImplicit<3>( @@ -791,7 +792,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } else if (WarpX::nox == 4){ doDepositionShapeNImplicit<4>( @@ -799,7 +800,7 @@ WarpXParticleContainer::DepositCurrent (WarpXParIter& pti, uxp_n.dataPtr() + offset, uyp_n.dataPtr() + offset, uzp_n.dataPtr() + offset, uxp.dataPtr() + offset, uyp.dataPtr() + offset, uzp.dataPtr() + offset, ion_lev, - jx_fab, jy_fab, jz_fab, np_to_deposit, dx, + jx_fab, jy_fab, jz_fab, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes); } } @@ -931,7 +932,6 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, "Particles shape does not fit within tile (CPU) or guard cells (GPU) used for charge deposition"); amrex::ignore_unused(range); // In case the assertion isn't compiled - const std::array& dx = WarpX::CellSize(std::max(depos_lev,0)); const Real q = this->charge; WARPX_PROFILE_VAR_NS("WarpXParticleContainer::DepositCharge::Sorting", blp_sort); @@ -981,12 +981,13 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, // Take into account Galilean shift const Real dt = warpx.getdt(lev); const amrex::Real time_shift_delta = (icomp == 0 ? 0.0_rt : dt); - const std::array& xyzmin = WarpX::LowerCorner( - tilebox, depos_lev, time_shift_delta); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(tilebox, depos_lev, time_shift_delta); // Indices of the lower bound const Dim3 lo = lbound(tilebox); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(depos_lev,0)); + // HACK - sort particles by bin here. WARPX_PROFILE_VAR_START(blp_sort); amrex::DenseBins bins; @@ -1088,25 +1089,25 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, if (WarpX::nox == 1){ doChargeDepositionSharedShapeN<1>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 2){ doChargeDepositionSharedShapeN<2>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 3){ doChargeDepositionSharedShapeN<3>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); } else if (WarpX::nox == 4){ doChargeDepositionSharedShapeN<4>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, ix_type, np_to_deposit, dx, xyzmin, lo, q, + rho_fab, ix_type, np_to_deposit, dinv, xyzmin, lo, q, WarpX::n_rz_azimuthal_modes, bins, box, geom, max_tbox_size, WarpX::shared_tilesize); @@ -1125,7 +1126,6 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, // note: this is smaller than rho->nGrowVect() for PSATD const amrex::IntVect& ng_rho = warpx.get_ng_depos_rho(); - const std::array& dx = WarpX::CellSize(std::max(depos_lev,0)); amrex::IntVect ref_ratio; if (lev == depos_lev) { ref_ratio = IntVect(AMREX_D_DECL(1, 1, 1 )); @@ -1150,7 +1150,8 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, // Take into account Galilean shift const amrex::Real dt = warpx.getdt(lev); const amrex::Real time_shift_delta = (icomp == 0 ? 0.0_rt : dt); - const std::array& xyzmin = WarpX::LowerCorner(tilebox, depos_lev, time_shift_delta); + const amrex::XDim3 xyzmin = WarpX::LowerCorner(tilebox, depos_lev, time_shift_delta); + const amrex::XDim3 dinv = WarpX::InvCellSize(std::max(depos_lev,0)); AMREX_ALWAYS_ASSERT(WarpX::nox == WarpX::noy); AMREX_ALWAYS_ASSERT(WarpX::nox == WarpX::noz); @@ -1158,12 +1159,10 @@ WarpXParticleContainer::DepositCharge (WarpXParIter& pti, RealVector const& wp, ablastr::particles::deposit_charge( pti, wp, this->charge, ion_lev, rho, local_rho[thread_num], - WarpX::noz, dx, xyzmin, WarpX::n_rz_azimuthal_modes, + WarpX::noz, dinv, xyzmin, WarpX::n_rz_azimuthal_modes, ng_rho, depos_lev, ref_ratio, offset, np_to_deposit, - icomp, nc, - WarpX::do_device_synchronize - ); + icomp, nc); } } diff --git a/Source/Python/WarpX.cpp b/Source/Python/WarpX.cpp index 4b0ad56a0ce..b3bd5414303 100644 --- a/Source/Python/WarpX.cpp +++ b/Source/Python/WarpX.cpp @@ -14,7 +14,7 @@ #include #include #include -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include # ifdef WARPX_DIM_RZ # include diff --git a/Source/Utils/Parser/ParserUtils.H b/Source/Utils/Parser/ParserUtils.H index dbc4845d1bf..96937abdbbe 100644 --- a/Source/Utils/Parser/ParserUtils.H +++ b/Source/Utils/Parser/ParserUtils.H @@ -147,7 +147,7 @@ namespace utils::parser auto parser = makeParser(str_val, {}); - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { val = safeCastToInt( static_cast(std::round(parser.compileHost<0>()())), str); @@ -174,7 +174,7 @@ namespace utils::parser val.resize(n); for (int i=0 ; i < n ; i++) { auto parser = makeParser(tmp_str_arr[i], {}); - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { val[i] = safeCastToInt( static_cast(std::round(parser.compileHost<0>()())), str); } @@ -217,7 +217,7 @@ namespace utils::parser val.resize(n); for (int i=0 ; i < n ; i++) { auto parser = makeParser(tmp_str_arr[i], {}); - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { val[i] = safeCastToInt( static_cast(std::round(parser.compileHost<0>()())), str); } @@ -250,7 +250,7 @@ namespace utils::parser Store_parserString(a_pp, str, str_val); auto parser = makeParser(str_val, {}); - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { val = safeCastToInt( static_cast(std::round(parser.compileHost<0>()())), str); } @@ -270,7 +270,7 @@ namespace utils::parser val.resize(n); for (int i=0 ; i < n ; i++) { auto parser = makeParser(tmp_str_arr[i], {}); - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { val[i] = safeCastToInt( static_cast(std::round(parser.compileHost<0>()())), str); } @@ -308,7 +308,7 @@ namespace utils::parser val.resize(n); for (int i=0 ; i < n ; i++) { auto parser = makeParser(tmp_str_arr[i], {}); - if constexpr (std::is_same::value) { + if constexpr (std::is_same_v) { val[i] = safeCastToInt( static_cast(std::round(parser.compileHost<0>()())), str); } diff --git a/Source/Utils/WarpXAlgorithmSelection.H b/Source/Utils/WarpXAlgorithmSelection.H index c34516aedc5..b9105557462 100644 --- a/Source/Utils/WarpXAlgorithmSelection.H +++ b/Source/Utils/WarpXAlgorithmSelection.H @@ -9,9 +9,13 @@ #define WARPX_UTILS_WARPXALGORITHMSELECTION_H_ #include +#include #include +using namespace ablastr::utils::enums; // NOLINT(google-global-names-in-headers) + + /** * \brief struct to determine the computational medium, i.e., vacuum or material/macroscopic default is vacuum. @@ -29,8 +33,8 @@ struct MediumForEM { struct EvolveScheme { enum { Explicit = 0, - ImplicitPicard = 1, - SemiImplicitPicard = 2 + ThetaImplicitEM = 1, + SemiImplicitEM = 2 }; }; @@ -47,20 +51,6 @@ struct MacroscopicSolverAlgo { }; }; -struct GridType { - enum { - Collocated = 0, - Staggered = 1, - Hybrid = 2 - }; -}; - -enum struct PatchType -{ - fine, - coarse -}; - struct ElectromagneticSolverAlgo { enum { None = 0, @@ -175,7 +165,8 @@ enum struct ParticleBoundaryType { Open = 1, //!< particles cross domain boundary leave with damped j Reflecting = 2, //!< particles are reflected Periodic = 3, //!< particles are introduced from the periodic boundary - Thermal = 4 + Thermal = 4, + None = 5 //!< For r=0 boundary with RZ simulations }; /** MPI reductions diff --git a/Source/Utils/WarpXAlgorithmSelection.cpp b/Source/Utils/WarpXAlgorithmSelection.cpp index b1f93c2f6c8..9e2360315b8 100644 --- a/Source/Utils/WarpXAlgorithmSelection.cpp +++ b/Source/Utils/WarpXAlgorithmSelection.cpp @@ -10,8 +10,9 @@ #include "WarpXAlgorithmSelection.H" #include "Utils/TextMsg.H" -#include +#include +#include #include #include @@ -24,17 +25,17 @@ // and corresponding integer for use inside the code const std::map evolve_scheme_to_int = { - {"explicit", EvolveScheme::Explicit }, - {"implicit_picard", EvolveScheme::ImplicitPicard }, - {"semi_implicit_picard", EvolveScheme::SemiImplicitPicard }, - {"default", EvolveScheme::Explicit } + {"explicit", EvolveScheme::Explicit }, + {"theta_implicit_em", EvolveScheme::ThetaImplicitEM }, + {"semi_implicit_em", EvolveScheme::SemiImplicitEM }, + {"default", EvolveScheme::Explicit } }; const std::map grid_to_int = { - {"collocated", GridType::Collocated}, - {"staggered", GridType::Staggered}, - {"hybrid", GridType::Hybrid}, - {"default", GridType::Staggered} + {"collocated", static_cast(ablastr::utils::enums::GridType::Collocated)}, + {"staggered", static_cast(ablastr::utils::enums::GridType::Staggered)}, + {"hybrid", static_cast(ablastr::utils::enums::GridType::Hybrid)}, + {"default", static_cast(ablastr::utils::enums::GridType::Staggered)} }; const std::map electromagnetic_solver_algo_to_int = { @@ -142,6 +143,7 @@ const std::map ParticleBCType_algo_to_enum = {"reflecting", ParticleBoundaryType::Reflecting}, {"periodic", ParticleBoundaryType::Periodic}, {"thermal", ParticleBoundaryType::Thermal}, + {"none", ParticleBoundaryType::None}, {"default", ParticleBoundaryType::Absorbing} }; @@ -152,8 +154,8 @@ const std::map ReductionType_algo_to_int = { }; int -GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ){ - +GetAlgorithmInteger(const amrex::ParmParse& pp, const char* pp_search_key ) +{ // Read user input ; use "default" if it is not found std::string algo = "default"; pp.query( pp_search_key, algo ); diff --git a/Source/Utils/WarpXMovingWindow.cpp b/Source/Utils/WarpXMovingWindow.cpp index eb7eceaa2fd..73696838cd4 100644 --- a/Source/Utils/WarpXMovingWindow.cpp +++ b/Source/Utils/WarpXMovingWindow.cpp @@ -9,7 +9,7 @@ #include "WarpX.H" #include "BoundaryConditions/PML.H" -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) # include "BoundaryConditions/PML_RZ.H" #endif #include "Initialization/ExternalField.H" @@ -19,6 +19,7 @@ #include "Utils/TextMsg.H" #include "Utils/WarpXConst.H" #include "Utils/WarpXProfilerWrapper.H" +#include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H" #include @@ -252,7 +253,7 @@ WarpX::MoveWindow (const int step, bool move_j) shiftMF(*pml_B[dim], geom[lev], num_shift, dir, lev, dont_update_cost); shiftMF(*pml_E[dim], geom[lev], num_shift, dir, lev, dont_update_cost); } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (pml_rz[lev] && dim < 2) { const std::array& pml_rz_B = pml_rz[lev]->GetB_fp(); const std::array& pml_rz_E = pml_rz[lev]->GetE_fp(); @@ -452,6 +453,17 @@ WarpX::MoveWindow (const int step, bool move_j) } } + // Recompute macroscopic properties of the medium + if (WarpX::em_solver_medium == MediumForEM::Macroscopic) { + const int lev_zero = 0; + m_macroscopic_properties->InitData( + Geom(lev_zero), + getField(warpx::fields::FieldType::Efield_fp, lev_zero,0).ixType().toIntVect(), + getField(warpx::fields::FieldType::Efield_fp, lev_zero,1).ixType().toIntVect(), + getField(warpx::fields::FieldType::Efield_fp, lev_zero,2).ixType().toIntVect() + ); + } + return num_shift_base; } @@ -522,7 +534,7 @@ WarpX::shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (amrex::MFIter mfi(tmpmf); mfi.isValid(); ++mfi ) + for (amrex::MFIter mfi(tmpmf, TilingIfNotGPU()); mfi.isValid(); ++mfi ) { if (cost && WarpX::load_balance_costs_update_algo == LoadBalanceCostsUpdateAlgo::Timers) { @@ -533,7 +545,7 @@ WarpX::shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, auto const& dstfab = mf.array(mfi); auto const& srcfab = tmpmf.array(mfi); - const amrex::Box& outbox = mfi.fabbox() & adjBox; + const amrex::Box& outbox = mfi.growntilebox() & adjBox; if (outbox.ok()) { if (!useparser) { @@ -598,7 +610,7 @@ WarpX::shiftMF (amrex::MultiFab& mf, const amrex::Geometry& geom, } } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) if (WarpX::GetInstance().getPMLRZ()) { // This does the exchange of data in the corner guard cells, the cells that are in the // guard region both radially and longitudinally. These are the PML cells in the overlapping diff --git a/Source/Utils/WarpXProfilerWrapper.H b/Source/Utils/WarpXProfilerWrapper.H index 856fc9d0e02..8b13b0df2ab 100644 --- a/Source/Utils/WarpXProfilerWrapper.H +++ b/Source/Utils/WarpXProfilerWrapper.H @@ -8,17 +8,13 @@ #ifndef WARPX_PROFILERWRAPPER_H_ #define WARPX_PROFILERWRAPPER_H_ -#include "WarpX.H" -#include "ablastr/profiler/ProfilerWrapper.H" +#include - -// `BL_PROFILE_PASTE(SYNC_SCOPE_, __COUNTER__)` and `SYNC_V_##vname` used to make unique names for -// synchronizeOnDestruct objects, like `SYNC_SCOPE_0` and `SYNC_V_pmain` -#define WARPX_PROFILE(fname) ABLASTR_PROFILE(fname, WarpX::do_device_synchronize) -#define WARPX_PROFILE_VAR(fname, vname) ABLASTR_PROFILE_VAR(fname, vname, WarpX::do_device_synchronize) -#define WARPX_PROFILE_VAR_NS(fname, vname) ABLASTR_PROFILE_VAR_NS(fname, vname, WarpX::do_device_synchronize) -#define WARPX_PROFILE_VAR_START(vname) ABLASTR_PROFILE_VAR_START(vname, WarpX::do_device_synchronize) -#define WARPX_PROFILE_VAR_STOP(vname) ABLASTR_PROFILE_VAR_STOP(vname, WarpX::do_device_synchronize) -#define WARPX_PROFILE_REGION(rname) ABLASTR_PROFILE_REGION(rname, WarpX::do_device_synchronize) +#define WARPX_PROFILE(fname) BL_PROFILE(fname) +#define WARPX_PROFILE_VAR(fname, vname) BL_PROFILE_VAR(fname, vname) +#define WARPX_PROFILE_VAR_NS(fname, vname) BL_PROFILE_VAR_NS(fname, vname) +#define WARPX_PROFILE_VAR_START(vname) BL_PROFILE_VAR_START(vname) +#define WARPX_PROFILE_VAR_STOP(vname) BL_PROFILE_VAR_STOP(vname) +#define WARPX_PROFILE_REGION(rname) BL_PROFILE_REGION(rname) #endif // WARPX_PROFILERWRAPPER_H_ diff --git a/Source/Utils/WarpXTagging.cpp b/Source/Utils/WarpXTagging.cpp index f88380e2589..b6eb2fda5d2 100644 --- a/Source/Utils/WarpXTagging.cpp +++ b/Source/Utils/WarpXTagging.cpp @@ -61,7 +61,7 @@ WarpX::ErrorEst (int lev, TagBoxArray& tags, Real /*time*/, int /*ngrow*/) } else { tag_val = (pos > ftlo && pos < fthi); } - if ( tag_val == 1) { + if (tag_val) { fab(i,j,k) = TagBox::SET; } }); diff --git a/Source/Utils/WarpXUtil.cpp b/Source/Utils/WarpXUtil.cpp index d2691685c53..2ef4ee55d6e 100644 --- a/Source/Utils/WarpXUtil.cpp +++ b/Source/Utils/WarpXUtil.cpp @@ -232,8 +232,8 @@ void NullifyMF(amrex::MultiFab& mf, int lev, amrex::Real zmin, amrex::Real zmax) for(amrex::MFIter mfi(mf, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi){ const amrex::Box& bx = mfi.tilebox(); // Get box lower and upper physical z bound, and dz - const amrex::Real zmin_box = WarpX::LowerCorner(bx, lev, 0._rt)[2]; - const amrex::Real zmax_box = WarpX::UpperCorner(bx, lev, 0._rt)[2]; + const amrex::Real zmin_box = WarpX::LowerCorner(bx, lev, 0._rt).z; + const amrex::Real zmax_box = WarpX::UpperCorner(bx, lev, 0._rt).z; const amrex::Real dz = WarpX::CellSize(lev)[2]; // Get box lower index in the z direction #if defined(WARPX_DIM_3D) diff --git a/Source/Utils/WarpX_Complex.H b/Source/Utils/WarpX_Complex.H index 7a1f1669286..54209b1d5d8 100644 --- a/Source/Utils/WarpX_Complex.H +++ b/Source/Utils/WarpX_Complex.H @@ -8,7 +8,7 @@ #ifndef WARPX_COMPLEX_H_ #define WARPX_COMPLEX_H_ -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include #endif @@ -21,7 +21,7 @@ // Defines a complex type on GPU & CPU using Complex = amrex::GpuComplex; -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT static_assert(sizeof(Complex) == sizeof(ablastr::math::anyfft::Complex), "The complex type in WarpX and the FFT library do not match."); #endif diff --git a/Source/WarpX.H b/Source/WarpX.H index 8ad7820e4cd..3361869f4c8 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -27,7 +27,7 @@ #include "Fluids/MultiFluidContainer_fwd.H" #include "Fluids/WarpXFluidContainer_fwd.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # ifdef WARPX_DIM_RZ # include "FieldSolver/SpectralSolver/SpectralSolverRZ_fwd.H" # include "BoundaryConditions/PML_RZ_fwd.H" @@ -41,12 +41,16 @@ #include "FieldSolver/Fields.H" #include "FieldSolver/ElectrostaticSolver.H" #include "FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H" +#include "FieldSolver/ImplicitSolvers/ImplicitSolver.H" +#include "FieldSolver/ImplicitSolvers/WarpXSolverVec.H" #include "Filter/BilinearFilter.H" #include "Parallelization/GuardCellManager.H" #include "Utils/Parser/IntervalsParser.H" #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/export.H" +#include + #include #include #include @@ -113,11 +117,27 @@ public: void Evolve (int numsteps = -1); - void EvolveImplicitPicardInit (int lev); - void SaveParticlesAtImplicitStepStart (WarpXParticleContainer& pc, int lev); - void FinishImplicitParticleUpdate (WarpXParticleContainer& pc, int lev); - void FinishImplicitFieldUpdate(amrex::Vector, 3 > >& Efield_fp, - amrex::Vector, 3 > >& Efield_n); + // + // Functions used by implicit solvers + // + void ImplicitPreRHSOp ( amrex::Real cur_time, + amrex::Real a_full_dt, + int a_nl_iter, + bool a_from_jacobian ); + void SaveParticlesAtImplicitStepStart (); + void FinishImplicitParticleUpdate (); + void SetElectricFieldAndApplyBCs ( const WarpXSolverVec& a_E ); + void UpdateMagneticFieldAndApplyBCs ( const amrex::Vector, 3 > >& a_Bn, + amrex::Real a_thetadt ); + void ApplyMagneticFieldBCs (); + void FinishMagneticFieldAndApplyBCs ( const amrex::Vector, 3 > >& a_Bn, + amrex::Real a_theta ); + void FinishImplicitField ( amrex::Vector, 3 > >& Field_fp, + const amrex::Vector, 3 > >& Field_n, + amrex::Real theta ); + void ImplicitComputeRHSE ( amrex::Real dt, WarpXSolverVec& a_Erhs_vec); + void ImplicitComputeRHSE (int lev, amrex::Real dt, WarpXSolverVec& a_Erhs_vec); + void ImplicitComputeRHSE (int lev, PatchType patch_type, amrex::Real dt, WarpXSolverVec& a_Erhs_vec); MultiParticleContainer& GetPartContainer () { return *mypc; } MultiFluidContainer& GetFluidContainer () { return *myfl; } @@ -159,14 +179,12 @@ public: static short particle_pusher_algo; //! Integer that corresponds to the type of Maxwell solver (Yee, CKC, PSATD, ECT) static short electromagnetic_solver_id; - //! Integer that corresponds to the evolve scheme (explicit, implicit_picard, semi_implicit_picard) + //! Integer that corresponds to the evolve scheme (explicit, semi_implicit_em, theta_implicit_em) static short evolve_scheme; - //! The maximum number of Picard iterations to do each time step - static int max_picard_iterations; - //! The tolerance for the Picard iteration convergence - static amrex::Real picard_iteration_tolerance; - //! Flags whether the Picard iterations are required to converge - static bool require_picard_convergence; + //! Maximum iterations used for self-consistent particle update in implicit particle-suppressed evolve schemes + static int max_particle_its_in_implicit_scheme; + //! Relative tolerance used for self-consistent particle update in implicit particle-suppressed evolve schemes + static amrex::ParticleReal particle_tol_in_implicit_scheme; /** Records a number corresponding to the load balance cost update strategy * being used (0 or 1 corresponding to timers or heuristic). */ @@ -219,7 +237,7 @@ public: //! If true, a correction is applied to the current in Fourier space, // to satisfy the continuity equation and charge conservation - bool current_correction; + bool current_correction = true; //! If true, the PSATD update equation for E contains both J and rho //! (default is false for standard PSATD and true for Galilean PSATD) @@ -359,7 +377,7 @@ public: //! Integer that corresponds to the type of grid used in the simulation //! (collocated, staggered, hybrid) - static short grid_type; + static ablastr::utils::enums::GridType grid_type; // Global rho nodal flag to know about rho index type when rho MultiFab is not allocated amrex::IntVect m_rho_nodal_flag; @@ -510,10 +528,21 @@ public: [[nodiscard]] const amrex::MultiFab& getField(warpx::fields::FieldType field_type, int lev, int direction = 0) const; + /** + * \brief + * Get a constant reference to the specified vector field on the different MR levels + * + * \param field_type[in] the field type + * + * \return a vector (which one element per MR level) of arrays of three pointers (for 3 vector components) amrex::MultiFab* containing the field data + */ + [[nodiscard]] const amrex::Vector,3>>& + getMultiLevelField(warpx::fields::FieldType field_type) const; + [[nodiscard]] bool DoPML () const {return do_pml;} [[nodiscard]] bool DoFluidSpecies () const {return do_fluid_species;} -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) const PML_RZ* getPMLRZ() {return pml_rz[0].get();} #endif @@ -756,7 +785,7 @@ public: static bool isAnyParticleBoundaryThermal(); PML* GetPML (int lev); -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) PML_RZ* GetPML_RZ (int lev); #endif @@ -877,6 +906,7 @@ public: amrex::Vector& output_geom ) const; static std::array CellSize (int lev); + static amrex::XDim3 InvCellSize (int lev); static amrex::RealBox getRealBox(const amrex::Box& bx, int lev); /** @@ -887,7 +917,7 @@ public: * (when v_galilean is not zero) * \return An array of the position coordinates */ - static std::array LowerCorner (const amrex::Box& bx, int lev, amrex::Real time_shift_delta); + static amrex::XDim3 LowerCorner (const amrex::Box& bx, int lev, amrex::Real time_shift_delta); /** * \brief Return the upper corner of the box in real units. * \param bx The box @@ -896,7 +926,7 @@ public: * (when v_galilean is not zero) * \return An array of the position coordinates */ - static std::array UpperCorner (const amrex::Box& bx, int lev, amrex::Real time_shift_delta); + static amrex::XDim3 UpperCorner (const amrex::Box& bx, int lev, amrex::Real time_shift_delta); static amrex::IntVect RefRatio (int lev); @@ -1029,7 +1059,7 @@ public: * \brief Load field values from a user-specified openPMD file, * for the fields Ex, Ey, Ez, Bx, By, Bz */ - void LoadExternalFieldsFromFile (int lev); + void LoadExternalFields (int lev); /** * \brief Load field values from a user-specified openPMD file @@ -1066,7 +1096,7 @@ public: * \param[in] n_order order of the finite-difference approximation * \param[in] a_grid_type type of grid (collocated or not) */ - static amrex::Vector getFornbergStencilCoefficients(int n_order, short a_grid_type); + static amrex::Vector getFornbergStencilCoefficients (int n_order, ablastr::utils::enums::GridType a_grid_type); // Device vectors of stencil coefficients used for finite-order centering of fields amrex::Gpu::DeviceVector device_field_centering_stencil_coeffs_x; @@ -1176,7 +1206,7 @@ public: */ void PSATDSubtractCurrentPartialSumsAvg (); -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # ifdef WARPX_DIM_RZ SpectralSolverRZ& @@ -1272,13 +1302,11 @@ private: void FillBoundaryB_avg (int lev, PatchType patch_type, amrex::IntVect ng); void FillBoundaryE_avg (int lev, PatchType patch_type, amrex::IntVect ng); - void AddExternalFields (); + void AddExternalFields (int lev); void OneStep_nosub (amrex::Real cur_time); void OneStep_sub1 (amrex::Real cur_time); - void OneStep_ImplicitPicard(amrex::Real cur_time); - /** * \brief Perform one PIC iteration, with the multiple J deposition per time step */ @@ -1424,14 +1452,14 @@ private: int centering_nox, int centering_noy, int centering_noz, - short a_grid_type); + ablastr::utils::enums::GridType a_grid_type); void AllocLevelMFs (int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm, const amrex::IntVect& ngEB, amrex::IntVect& ngJ, const amrex::IntVect& ngRho, const amrex::IntVect& ngF, const amrex::IntVect& ngG, bool aux_is_nodal); -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # ifdef WARPX_DIM_RZ void AllocLevelSpectralSolverRZ (amrex::Vector>& spectral_solver, int lev, @@ -1486,12 +1514,6 @@ private: amrex::Vector, 3 > > Efield_avg_fp; amrex::Vector, 3 > > Bfield_avg_fp; - // Implicit, fields at start of step and from the previous iteration - amrex::Vector, 3 > > Efield_n; - amrex::Vector, 3 > > Bfield_n; - amrex::Vector, 3 > > Efield_save; - amrex::Vector, 3 > > Bfield_save; - // Memory buffers for computing magnetostatic fields // Vector Potential A and previous step. Time buffer needed for computing dA/dt to first order amrex::Vector, 3 > > vector_potential_fp_nodal; @@ -1501,6 +1523,8 @@ private: // Same as Bfield_fp/Efield_fp for reading external field data amrex::Vector, 3 > > Efield_fp_external; amrex::Vector, 3 > > Bfield_fp_external; + amrex::Vector, 3 > > E_external_particle_field; + amrex::Vector, 3 > > B_external_particle_field; //! EB: Lengths of the mesh edges amrex::Vector, 3 > > m_edge_lengths; @@ -1600,7 +1624,7 @@ private: amrex::Vector do_pml_Lo; amrex::Vector do_pml_Hi; amrex::Vector > pml; -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) amrex::Vector > pml_rz; #endif amrex::Real v_particle_pml; @@ -1775,7 +1799,7 @@ private: void PushPSATD (); -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT /** * \brief Forward FFT of E,B on all mesh refinement levels @@ -1945,6 +1969,10 @@ private: amrex::Vector> m_fdtd_solver_fp; amrex::Vector> m_fdtd_solver_cp; + + // implicit solver object + std::unique_ptr m_implicit_solver; + }; #endif diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index d724ecd7571..a9aacdf1fc6 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -18,7 +18,7 @@ #include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceSolver.H" #include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H" #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H" -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # include "FieldSolver/SpectralSolver/SpectralKSpace.H" # ifdef WARPX_DIM_RZ # include "FieldSolver/SpectralSolver/SpectralSolverRZ.H" @@ -41,6 +41,8 @@ #include "Utils/WarpXProfilerWrapper.H" #include "Utils/WarpXUtil.H" +#include "FieldSolver/ImplicitSolvers/ImplicitSolverLibrary.H" + #include #include @@ -113,9 +115,8 @@ short WarpX::field_gathering_algo; short WarpX::particle_pusher_algo; short WarpX::electromagnetic_solver_id; short WarpX::evolve_scheme; -int WarpX::max_picard_iterations = 10; -Real WarpX::picard_iteration_tolerance = 1.e-7; -bool WarpX::require_picard_convergence = true; +int WarpX::max_particle_its_in_implicit_scheme = 21; +ParticleReal WarpX::particle_tol_in_implicit_scheme = 1.e-10; short WarpX::psatd_solution_type; short WarpX::J_in_time; short WarpX::rho_in_time; @@ -210,15 +211,9 @@ IntVect WarpX::filter_npass_each_dir(1); int WarpX::n_field_gather_buffer = -1; int WarpX::n_current_deposition_buffer = -1; -short WarpX::grid_type; +ablastr::utils::enums::GridType WarpX::grid_type; amrex::IntVect m_rho_nodal_flag; -#ifdef AMREX_USE_GPU -bool WarpX::do_device_synchronize = true; -#else -bool WarpX::do_device_synchronize = false; -#endif - WarpX* WarpX::m_instance = nullptr; void WarpX::MakeWarpX () @@ -292,7 +287,7 @@ WarpX::WarpX () // Loop over species (particles and lasers) // and set current injection position per species - if (do_moving_window){ + if (do_moving_window){ const int n_containers = mypc->nContainers(); for (int i=0; i(GetAlgorithmInteger(pp_warpx, "grid_type")); + grid_type = static_cast(GetAlgorithmInteger(pp_warpx, "grid_type")); // Use same shape factors in all directions, for gathering if (grid_type == GridType::Collocated) { galerkin_interpolation = false; } @@ -1165,7 +1160,7 @@ WarpX::ReadParameters () WARPX_ALWAYS_ASSERT_WITH_MESSAGE( electromagnetic_solver_id != ElectromagneticSolverAlgo::CKC, "algo.maxwell_solver = ckc is not (yet) available for RZ geometry"); #endif -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT WARPX_ALWAYS_ASSERT_WITH_MESSAGE( electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD, "algo.maxwell_solver = psatd is not supported because WarpX was built without spectral solvers"); #endif @@ -1179,6 +1174,14 @@ WarpX::ReadParameters () WARPX_ALWAYS_ASSERT_WITH_MESSAGE( WarpX::field_boundary_lo[0] == FieldBoundaryType::None, "Error : Field boundary at r=0 must be ``none``. \n"); + + const ParmParse pp_boundary("boundary"); + if (pp_boundary.contains("particle_lo")) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + WarpX::particle_boundary_lo[0] == ParticleBoundaryType::None, + "Error : Particle boundary at r=0 must be ``none``. \n"); + } + } if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { @@ -1197,6 +1200,21 @@ WarpX::ReadParameters () particle_pusher_algo = static_cast(GetAlgorithmInteger(pp_algo, "particle_pusher")); evolve_scheme = static_cast(GetAlgorithmInteger(pp_algo, "evolve_scheme")); + // check for implicit evolve scheme + if (evolve_scheme == EvolveScheme::SemiImplicitEM) { + m_implicit_solver = std::make_unique(); + } + else if (evolve_scheme == EvolveScheme::ThetaImplicitEM) { + m_implicit_solver = std::make_unique(); + } + + // implicit evolve schemes not setup to use mirrors + if (evolve_scheme == EvolveScheme::SemiImplicitEM || + evolve_scheme == EvolveScheme::ThetaImplicitEM) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( num_mirrors == 0, + "Mirrors cannot be used with Implicit evolve schemes."); + } + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( current_deposition_algo != CurrentDepositionAlgo::Esirkepov || !do_current_centering, @@ -1233,8 +1251,8 @@ WarpX::ReadParameters () if (current_deposition_algo == CurrentDepositionAlgo::Villasenor) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( - evolve_scheme == EvolveScheme::ImplicitPicard || - evolve_scheme == EvolveScheme::SemiImplicitPicard, + evolve_scheme == EvolveScheme::SemiImplicitEM || + evolve_scheme == EvolveScheme::ThetaImplicitEM, "Villasenor current deposition can only" "be used with Implicit evolve schemes."); } @@ -1270,8 +1288,16 @@ WarpX::ReadParameters () } } - // Use same shape factors in all directions, for gathering - if (field_gathering_algo == GatheringAlgo::MomentumConserving) { galerkin_interpolation = false; } + // Use same shape factors in all directions + // - with momentum-conserving field gathering + if (field_gathering_algo == GatheringAlgo::MomentumConserving) {galerkin_interpolation = false;} + // - with direct current deposition and the EM solver + if( electromagnetic_solver_id != ElectromagneticSolverAlgo::None && + electromagnetic_solver_id != ElectromagneticSolverAlgo::HybridPIC ) { + if (current_deposition_algo == CurrentDepositionAlgo::Direct) { + galerkin_interpolation = false; + } + } { const ParmParse pp_interpolation("interpolation"); @@ -1307,12 +1333,8 @@ WarpX::ReadParameters () } } - if (evolve_scheme == EvolveScheme::ImplicitPicard || - evolve_scheme == EvolveScheme::SemiImplicitPicard) { - utils::parser::queryWithParser(pp_algo, "max_picard_iterations", max_picard_iterations); - utils::parser::queryWithParser(pp_algo, "picard_iteration_tolerance", picard_iteration_tolerance); - utils::parser::queryWithParser(pp_algo, "require_picard_convergence", require_picard_convergence); - + if (evolve_scheme == EvolveScheme::SemiImplicitEM || + evolve_scheme == EvolveScheme::ThetaImplicitEM) { WARPX_ALWAYS_ASSERT_WITH_MESSAGE( current_deposition_algo == CurrentDepositionAlgo::Esirkepov || current_deposition_algo == CurrentDepositionAlgo::Villasenor || @@ -1535,7 +1557,6 @@ WarpX::ReadParameters () // Current correction activated by default, unless a charge-conserving // current deposition (Esirkepov, Vay) or the div(E) cleaning scheme // are used - current_correction = true; if (WarpX::current_deposition_algo == CurrentDepositionAlgo::Esirkepov || WarpX::current_deposition_algo == CurrentDepositionAlgo::Villasenor || WarpX::current_deposition_algo == CurrentDepositionAlgo::Vay || @@ -2105,7 +2126,7 @@ WarpX::ClearLevel (int lev) G_cp [lev].reset(); rho_cp[lev].reset(); -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { spectral_solver_fp[lev].reset(); spectral_solver_cp[lev].reset(); @@ -2121,13 +2142,7 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d { const bool aux_is_nodal = (field_gathering_algo == GatheringAlgo::MomentumConserving); -#if defined(WARPX_DIM_1D_Z) - const amrex::RealVect dx(WarpX::CellSize(lev)[2]); -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - const amrex::RealVect dx = {WarpX::CellSize(lev)[0], WarpX::CellSize(lev)[2]}; -#elif defined(WARPX_DIM_3D) - const amrex::RealVect dx = {WarpX::CellSize(lev)[0], WarpX::CellSize(lev)[1], WarpX::CellSize(lev)[2]}; -#endif + const Real* dx = Geom(lev).CellSize(); // Initialize filter before guard cells manager // (needs info on length of filter's stencil) @@ -2191,11 +2206,6 @@ WarpX::AllocLevelData (int lev, const BoxArray& ba, const DistributionMapping& d AllocLevelMFs(lev, ba, dm, guard_cells.ng_alloc_EB, guard_cells.ng_alloc_J, guard_cells.ng_alloc_Rho, guard_cells.ng_alloc_F, guard_cells.ng_alloc_G, aux_is_nodal); - if (evolve_scheme == EvolveScheme::ImplicitPicard || - evolve_scheme == EvolveScheme::SemiImplicitPicard) { - EvolveImplicitPicardInit(lev); - } - m_accelerator_lattice[lev] = std::make_unique(); m_accelerator_lattice[lev]->InitElementFinder(lev, ba, dm); @@ -2319,18 +2329,6 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(current_fp[lev][1], amrex::convert(ba, jy_nodal_flag), dm, ncomps, ngJ, lev, "current_fp[y]", 0.0_rt); AllocInitMultiFab(current_fp[lev][2], amrex::convert(ba, jz_nodal_flag), dm, ncomps, ngJ, lev, "current_fp[z]", 0.0_rt); - // Match external field MultiFabs to fine patch - if (m_p_ext_field_params->B_ext_grid_type == ExternalFieldType::read_from_file) { - AllocInitMultiFab(Bfield_fp_external[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[x]", 0.0_rt); - AllocInitMultiFab(Bfield_fp_external[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[y]", 0.0_rt); - AllocInitMultiFab(Bfield_fp_external[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_fp_external[z]", 0.0_rt); - } - if (m_p_ext_field_params->E_ext_grid_type == ExternalFieldType::read_from_file) { - AllocInitMultiFab(Efield_fp_external[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[x]", 0.0_rt); - AllocInitMultiFab(Efield_fp_external[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[y]", 0.0_rt); - AllocInitMultiFab(Efield_fp_external[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_fp_external[z]", 0.0_rt); - } - if (do_current_centering) { amrex::BoxArray const& nodal_ba = amrex::convert(ba, amrex::IntVect::TheNodeVector()); @@ -2387,6 +2385,13 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm myfl->InitData(lev, geom[lev].Domain(),cur_time); } + // Allocate extra multifabs for macroscopic properties of the medium + if (em_solver_medium == MediumForEM::Macroscopic) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( lev==0, + "Macroscopic properties are not supported with mesh refinement."); + m_macroscopic_properties->AllocateLevelMFs(ba, dm, ngEB); + } + if (fft_do_time_averaging) { AllocInitMultiFab(Bfield_avg_fp[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_avg_fp[x]", 0.0_rt); @@ -2471,7 +2476,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(phi_fp[lev], amrex::convert(ba, phi_nodal_flag), dm, ncomps, ngPhi, lev, "phi_fp", 0.0_rt); } - if (do_subcycling == 1 && lev == 0) + if (do_subcycling && lev == 0) { AllocInitMultiFab(current_store[lev][0], amrex::convert(ba,jx_nodal_flag),dm,ncomps,ngJ,lev, "current_store[x]"); AllocInitMultiFab(current_store[lev][1], amrex::convert(ba,jy_nodal_flag),dm,ncomps,ngJ,lev, "current_store[y]"); @@ -2491,7 +2496,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { // Allocate and initialize the spectral solver -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT WARPX_ALWAYS_ASSERT_WITH_MESSAGE( false, "WarpX::AllocLevelMFs: PSATD solver requires WarpX build with spectral solver support."); #else @@ -2562,32 +2567,82 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm AllocInitMultiFab(Efield_aux[lev][1], nba, dm, ncomps, ngEB, lev, "Efield_aux[y]", 0.0_rt); AllocInitMultiFab(Efield_aux[lev][2], nba, dm, ncomps, ngEB, lev, "Efield_aux[z]", 0.0_rt); } else if (lev == 0) { - if (!WarpX::fft_do_time_averaging) { - // In this case, the aux grid is simply an alias of the fp grid - AliasInitMultiFab(Efield_aux[lev][0], *Efield_fp[lev][0], 0, ncomps, lev, "Efield_aux[x]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][1], *Efield_fp[lev][1], 0, ncomps, lev, "Efield_aux[y]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][2], *Efield_fp[lev][2], 0, ncomps, lev, "Efield_aux[z]", 0.0_rt); - - AliasInitMultiFab(Bfield_aux[lev][0], *Bfield_fp[lev][0], 0, ncomps, lev, "Bfield_aux[x]", 0.0_rt); - AliasInitMultiFab(Bfield_aux[lev][1], *Bfield_fp[lev][1], 0, ncomps, lev, "Bfield_aux[y]", 0.0_rt); - AliasInitMultiFab(Bfield_aux[lev][2], *Bfield_fp[lev][2], 0, ncomps, lev, "Bfield_aux[z]", 0.0_rt); - } else { - AliasInitMultiFab(Efield_aux[lev][0], *Efield_avg_fp[lev][0], 0, ncomps, lev, "Efield_aux[x]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][1], *Efield_avg_fp[lev][1], 0, ncomps, lev, "Efield_aux[y]", 0.0_rt); - AliasInitMultiFab(Efield_aux[lev][2], *Efield_avg_fp[lev][2], 0, ncomps, lev, "Efield_aux[z]", 0.0_rt); - + if (WarpX::fft_do_time_averaging) { AliasInitMultiFab(Bfield_aux[lev][0], *Bfield_avg_fp[lev][0], 0, ncomps, lev, "Bfield_aux[x]", 0.0_rt); AliasInitMultiFab(Bfield_aux[lev][1], *Bfield_avg_fp[lev][1], 0, ncomps, lev, "Bfield_aux[y]", 0.0_rt); AliasInitMultiFab(Bfield_aux[lev][2], *Bfield_avg_fp[lev][2], 0, ncomps, lev, "Bfield_aux[z]", 0.0_rt); + + AliasInitMultiFab(Efield_aux[lev][0], *Efield_avg_fp[lev][0], 0, ncomps, lev, "Efield_aux[x]", 0.0_rt); + AliasInitMultiFab(Efield_aux[lev][1], *Efield_avg_fp[lev][1], 0, ncomps, lev, "Efield_aux[y]", 0.0_rt); + AliasInitMultiFab(Efield_aux[lev][2], *Efield_avg_fp[lev][2], 0, ncomps, lev, "Efield_aux[z]", 0.0_rt); + } else { + if (mypc->m_B_ext_particle_s == "read_from_file") { + AllocInitMultiFab(Bfield_aux[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[x]"); + AllocInitMultiFab(Bfield_aux[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[y]"); + AllocInitMultiFab(Bfield_aux[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[z]"); + } else { + // In this case, the aux grid is simply an alias of the fp grid (most common case in WarpX) + AliasInitMultiFab(Bfield_aux[lev][0], *Bfield_fp[lev][0], 0, ncomps, lev, "Bfield_aux[x]", 0.0_rt); + AliasInitMultiFab(Bfield_aux[lev][1], *Bfield_fp[lev][1], 0, ncomps, lev, "Bfield_aux[y]", 0.0_rt); + AliasInitMultiFab(Bfield_aux[lev][2], *Bfield_fp[lev][2], 0, ncomps, lev, "Bfield_aux[z]", 0.0_rt); + } + if (mypc->m_E_ext_particle_s == "read_from_file") { + AllocInitMultiFab(Efield_aux[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[x]"); + AllocInitMultiFab(Efield_aux[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[y]"); + AllocInitMultiFab(Efield_aux[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[z]"); + } else { + // In this case, the aux grid is simply an alias of the fp grid (most common case in WarpX) + AliasInitMultiFab(Efield_aux[lev][0], *Efield_fp[lev][0], 0, ncomps, lev, "Efield_aux[x]", 0.0_rt); + AliasInitMultiFab(Efield_aux[lev][1], *Efield_fp[lev][1], 0, ncomps, lev, "Efield_aux[y]", 0.0_rt); + AliasInitMultiFab(Efield_aux[lev][2], *Efield_fp[lev][2], 0, ncomps, lev, "Efield_aux[z]", 0.0_rt); + } } } else { - AllocInitMultiFab(Bfield_aux[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[x]"); - AllocInitMultiFab(Bfield_aux[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[y]"); - AllocInitMultiFab(Bfield_aux[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[z]"); - - AllocInitMultiFab(Efield_aux[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[x]"); - AllocInitMultiFab(Efield_aux[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[y]"); - AllocInitMultiFab(Efield_aux[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[z]"); + AllocInitMultiFab(Bfield_aux[lev][0], amrex::convert(ba, Bx_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[x]", 0.0_rt); + AllocInitMultiFab(Bfield_aux[lev][1], amrex::convert(ba, By_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[y]", 0.0_rt); + AllocInitMultiFab(Bfield_aux[lev][2], amrex::convert(ba, Bz_nodal_flag), dm, ncomps, ngEB, lev, "Bfield_aux[z]", 0.0_rt); + + AllocInitMultiFab(Efield_aux[lev][0], amrex::convert(ba, Ex_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[x]", 0.0_rt); + AllocInitMultiFab(Efield_aux[lev][1], amrex::convert(ba, Ey_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[y]", 0.0_rt); + AllocInitMultiFab(Efield_aux[lev][2], amrex::convert(ba, Ez_nodal_flag), dm, ncomps, ngEB, lev, "Efield_aux[z]", 0.0_rt); + } + + // The external fields that are read from file + if (m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::default_zero && m_p_ext_field_params->B_ext_grid_type != ExternalFieldType::constant) { + // These fields will be added directly to the grid, i.e. to fp, and need to match the index type + AllocInitMultiFab(Bfield_fp_external[lev][0], amrex::convert(ba, Bfield_fp[lev][0]->ixType()), + dm, ncomps, ngEB, lev, "Bfield_fp_external[x]", 0.0_rt); + AllocInitMultiFab(Bfield_fp_external[lev][1], amrex::convert(ba, Bfield_fp[lev][1]->ixType()), + dm, ncomps, ngEB, lev, "Bfield_fp_external[y]", 0.0_rt); + AllocInitMultiFab(Bfield_fp_external[lev][2], amrex::convert(ba, Bfield_fp[lev][2]->ixType()), + dm, ncomps, ngEB, lev, "Bfield_fp_external[z]", 0.0_rt); + } + if (mypc->m_B_ext_particle_s == "read_from_file") { + // These fields will be added to the fields that the particles see, and need to match the index type + AllocInitMultiFab(B_external_particle_field[lev][0], amrex::convert(ba, Bfield_aux[lev][0]->ixType()), + dm, ncomps, ngEB, lev, "B_external_particle_field[x]", 0.0_rt); + AllocInitMultiFab(B_external_particle_field[lev][1], amrex::convert(ba, Bfield_aux[lev][1]->ixType()), + dm, ncomps, ngEB, lev, "B_external_particle_field[y]", 0.0_rt); + AllocInitMultiFab(B_external_particle_field[lev][2], amrex::convert(ba, Bfield_aux[lev][2]->ixType()), + dm, ncomps, ngEB, lev, "B_external_particle_field[z]", 0.0_rt); + } + if (m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::default_zero && m_p_ext_field_params->E_ext_grid_type != ExternalFieldType::constant) { + // These fields will be added directly to the grid, i.e. to fp, and need to match the index type + AllocInitMultiFab(Efield_fp_external[lev][0], amrex::convert(ba, Efield_fp[lev][0]->ixType()), + dm, ncomps, ngEB, lev, "Efield_fp_external[x]", 0.0_rt); + AllocInitMultiFab(Efield_fp_external[lev][1], amrex::convert(ba, Efield_fp[lev][1]->ixType()), + dm, ncomps, ngEB, lev, "Efield_fp_external[y]", 0.0_rt); + AllocInitMultiFab(Efield_fp_external[lev][2], amrex::convert(ba, Efield_fp[lev][2]->ixType()), + dm, ncomps, ngEB, lev, "Efield_fp_external[z]", 0.0_rt); + } + if (mypc->m_E_ext_particle_s == "read_from_file") { + // These fields will be added to the fields that the particles see, and need to match the index type + AllocInitMultiFab(E_external_particle_field[lev][0], amrex::convert(ba, Efield_aux[lev][0]->ixType()), + dm, ncomps, ngEB, lev, "E_external_particle_field[x]", 0.0_rt); + AllocInitMultiFab(E_external_particle_field[lev][1], amrex::convert(ba, Efield_aux[lev][1]->ixType()), + dm, ncomps, ngEB, lev, "E_external_particle_field[y]", 0.0_rt); + AllocInitMultiFab(E_external_particle_field[lev][2], amrex::convert(ba, Efield_aux[lev][2]->ixType()), + dm, ncomps, ngEB, lev, "E_external_particle_field[z]", 0.0_rt); } // @@ -2649,7 +2704,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { // Allocate and initialize the spectral solver -#ifndef WARPX_USE_PSATD +#ifndef WARPX_USE_FFT WARPX_ALWAYS_ASSERT_WITH_MESSAGE( false, "WarpX::AllocLevelMFs: PSATD solver requires WarpX build with spectral solver support."); #else @@ -2700,22 +2755,22 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm if (n_field_gather_buffer > 0 || mypc->nSpeciesGatherFromMainGrid() > 0) { if (aux_is_nodal) { BoxArray const& cnba = amrex::convert(cba,IntVect::TheNodeVector()); - AllocInitMultiFab(Bfield_cax[lev][0], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[x]"); - AllocInitMultiFab(Bfield_cax[lev][1], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[y]"); - AllocInitMultiFab(Bfield_cax[lev][2], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[z]"); - AllocInitMultiFab(Efield_cax[lev][0], cnba,dm,ncomps,ngEB,lev, "Efield_cax[x]"); - AllocInitMultiFab(Efield_cax[lev][1], cnba,dm,ncomps,ngEB,lev, "Efield_cax[y]"); - AllocInitMultiFab(Efield_cax[lev][2], cnba,dm,ncomps,ngEB,lev, "Efield_cax[z]"); + AllocInitMultiFab(Bfield_cax[lev][0], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[x]", 0.0_rt); + AllocInitMultiFab(Bfield_cax[lev][1], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[y]", 0.0_rt); + AllocInitMultiFab(Bfield_cax[lev][2], cnba,dm,ncomps,ngEB,lev, "Bfield_cax[z]", 0.0_rt); + AllocInitMultiFab(Efield_cax[lev][0], cnba,dm,ncomps,ngEB,lev, "Efield_cax[x]", 0.0_rt); + AllocInitMultiFab(Efield_cax[lev][1], cnba,dm,ncomps,ngEB,lev, "Efield_cax[y]", 0.0_rt); + AllocInitMultiFab(Efield_cax[lev][2], cnba,dm,ncomps,ngEB,lev, "Efield_cax[z]", 0.0_rt); } else { // Create the MultiFabs for B - AllocInitMultiFab(Bfield_cax[lev][0], amrex::convert(cba,Bx_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[x]"); - AllocInitMultiFab(Bfield_cax[lev][1], amrex::convert(cba,By_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[y]"); - AllocInitMultiFab(Bfield_cax[lev][2], amrex::convert(cba,Bz_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[z]"); + AllocInitMultiFab(Bfield_cax[lev][0], amrex::convert(cba,Bx_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[x]", 0.0_rt); + AllocInitMultiFab(Bfield_cax[lev][1], amrex::convert(cba,By_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[y]", 0.0_rt); + AllocInitMultiFab(Bfield_cax[lev][2], amrex::convert(cba,Bz_nodal_flag),dm,ncomps,ngEB,lev, "Bfield_cax[z]", 0.0_rt); // Create the MultiFabs for E - AllocInitMultiFab(Efield_cax[lev][0], amrex::convert(cba,Ex_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[x]"); - AllocInitMultiFab(Efield_cax[lev][1], amrex::convert(cba,Ey_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[y]"); - AllocInitMultiFab(Efield_cax[lev][2], amrex::convert(cba,Ez_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[z]"); + AllocInitMultiFab(Efield_cax[lev][0], amrex::convert(cba,Ex_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[x]", 0.0_rt); + AllocInitMultiFab(Efield_cax[lev][1], amrex::convert(cba,Ey_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[y]", 0.0_rt); + AllocInitMultiFab(Efield_cax[lev][2], amrex::convert(cba,Ez_nodal_flag),dm,ncomps,ngEB,lev, "Efield_cax[z]", 0.0_rt); } AllocInitMultiFab(gather_buffer_masks[lev], ba, dm, ncomps, amrex::IntVect(1), lev, "gather_buffer_masks"); @@ -2743,7 +2798,7 @@ WarpX::AllocLevelMFs (int lev, const BoxArray& ba, const DistributionMapping& dm } } -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT # ifdef WARPX_DIM_RZ /* \brief Allocate spectral Maxwell solver (RZ dimensions) at a level * @@ -2858,6 +2913,13 @@ WarpX::CellSize (int lev) #endif } +amrex::XDim3 +WarpX::InvCellSize (int lev) +{ + std::array dx = WarpX::CellSize(lev); + return {1._rt/dx[0], 1._rt/dx[1], 1._rt/dx[2]}; +} + amrex::RealBox WarpX::getRealBox(const Box& bx, int lev) { @@ -2866,13 +2928,13 @@ WarpX::getRealBox(const Box& bx, int lev) return( grid_box ); } -std::array +amrex::XDim3 WarpX::LowerCorner(const Box& bx, const int lev, const amrex::Real time_shift_delta) { auto & warpx = GetInstance(); const RealBox grid_box = getRealBox( bx, lev ); - const Real* xyzmin = grid_box.lo(); + const Real* grid_min = grid_box.lo(); const amrex::Real cur_time = warpx.gett_new(lev); const amrex::Real time_shift = (cur_time + time_shift_delta - warpx.time_of_last_gal_shift); @@ -2881,23 +2943,23 @@ WarpX::LowerCorner(const Box& bx, const int lev, const amrex::Real time_shift_de warpx.m_v_galilean[2]*time_shift }; #if defined(WARPX_DIM_3D) - return { xyzmin[0] + galilean_shift[0], xyzmin[1] + galilean_shift[1], xyzmin[2] + galilean_shift[2] }; + return { grid_min[0] + galilean_shift[0], grid_min[1] + galilean_shift[1], grid_min[2] + galilean_shift[2] }; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - return { xyzmin[0] + galilean_shift[0], std::numeric_limits::lowest(), xyzmin[1] + galilean_shift[2] }; + return { grid_min[0] + galilean_shift[0], std::numeric_limits::lowest(), grid_min[1] + galilean_shift[2] }; #elif defined(WARPX_DIM_1D_Z) - return { std::numeric_limits::lowest(), std::numeric_limits::lowest(), xyzmin[0] + galilean_shift[2] }; + return { std::numeric_limits::lowest(), std::numeric_limits::lowest(), grid_min[0] + galilean_shift[2] }; #endif } -std::array +amrex::XDim3 WarpX::UpperCorner(const Box& bx, const int lev, const amrex::Real time_shift_delta) { auto & warpx = GetInstance(); const RealBox grid_box = getRealBox( bx, lev ); - const Real* xyzmax = grid_box.hi(); + const Real* grid_max = grid_box.hi(); const amrex::Real cur_time = warpx.gett_new(lev); const amrex::Real time_shift = (cur_time + time_shift_delta - warpx.time_of_last_gal_shift); @@ -2906,13 +2968,13 @@ WarpX::UpperCorner(const Box& bx, const int lev, const amrex::Real time_shift_de warpx.m_v_galilean[2]*time_shift }; #if defined(WARPX_DIM_3D) - return { xyzmax[0] + galilean_shift[0], xyzmax[1] + galilean_shift[1], xyzmax[2] + galilean_shift[2] }; + return { grid_max[0] + galilean_shift[0], grid_max[1] + galilean_shift[1], grid_max[2] + galilean_shift[2] }; #elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - return { xyzmax[0] + galilean_shift[0], std::numeric_limits::max(), xyzmax[1] + galilean_shift[1] }; + return { grid_max[0] + galilean_shift[0], std::numeric_limits::max(), grid_max[1] + galilean_shift[1] }; #elif defined(WARPX_DIM_1D_Z) - return { std::numeric_limits::max(), std::numeric_limits::max(), xyzmax[0] + galilean_shift[0] }; + return { std::numeric_limits::max(), std::numeric_limits::max(), grid_max[0] + galilean_shift[0] }; #endif } @@ -3000,7 +3062,7 @@ void WarpX::ComputeDivE(amrex::MultiFab& divE, const int lev) { if ( WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD ) { -#ifdef WARPX_USE_PSATD +#ifdef WARPX_USE_FFT spectral_solver_fp[lev]->ComputeSpectralDivE( lev, Efield_aux[lev], divE ); #else WARPX_ABORT_WITH_MESSAGE( @@ -3011,7 +3073,7 @@ WarpX::ComputeDivE(amrex::MultiFab& divE, const int lev) } } -#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_PSATD) +#if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT) PML_RZ* WarpX::GetPML_RZ (int lev) { @@ -3153,7 +3215,7 @@ WarpX::BuildBufferMasksInBox ( const amrex::Box tbx, amrex::IArrayBox &buffer_ma }); } -amrex::Vector WarpX::getFornbergStencilCoefficients(const int n_order, const short a_grid_type) +amrex::Vector WarpX::getFornbergStencilCoefficients (const int n_order, ablastr::utils::enums::GridType a_grid_type) { AMREX_ALWAYS_ASSERT_WITH_MESSAGE(n_order % 2 == 0, "n_order must be even"); @@ -3215,7 +3277,7 @@ void WarpX::AllocateCenteringCoefficients (amrex::Gpu::DeviceVector const int centering_nox, const int centering_noy, const int centering_noz, - const short a_grid_type) + ablastr::utils::enums::GridType a_grid_type) { // Vectors of Fornberg stencil coefficients amrex::Vector Fornberg_stencil_coeffs_x; @@ -3527,3 +3589,32 @@ WarpX::getField(FieldType field_type, const int lev, const int direction) const { return *getFieldPointer(field_type, lev, direction); } + +const amrex::Vector,3>>& +WarpX::getMultiLevelField(warpx::fields::FieldType field_type) const +{ + switch(field_type) + { + case FieldType::Efield_aux : + return Efield_aux; + case FieldType::Bfield_aux : + return Bfield_aux; + case FieldType::Efield_fp : + return Efield_fp; + case FieldType::Bfield_fp : + return Bfield_fp; + case FieldType::current_fp : + return current_fp; + case FieldType::current_fp_nodal : + return current_fp_nodal; + case FieldType::Efield_cp : + return Efield_cp; + case FieldType::Bfield_cp : + return Bfield_cp; + case FieldType::current_cp : + return current_cp; + default: + WARPX_ABORT_WITH_MESSAGE("Invalid field type"); + return Efield_fp; + } +} diff --git a/Source/ablastr/fields/CMakeLists.txt b/Source/ablastr/fields/CMakeLists.txt index f7dfb6763ec..56acc678217 100644 --- a/Source/ablastr/fields/CMakeLists.txt +++ b/Source/ablastr/fields/CMakeLists.txt @@ -1,6 +1,6 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) - if(WarpX_PSATD AND D EQUAL 3) + if(ABLASTR_FFT AND D EQUAL 3) target_sources(ablastr_${SD} PRIVATE IntegratedGreenFunctionSolver.cpp diff --git a/Source/ablastr/fields/Make.package b/Source/ablastr/fields/Make.package index 4fb7bd88e5f..01392991559 100644 --- a/Source/ablastr/fields/Make.package +++ b/Source/ablastr/fields/Make.package @@ -1,4 +1,4 @@ -ifeq ($(USE_PSATD),TRUE) +ifeq ($(USE_FFT),TRUE) ifeq ($(DIM),3) CEXE_sources += IntegratedGreenFunctionSolver.cpp endif diff --git a/Source/ablastr/fields/PoissonSolver.H b/Source/ablastr/fields/PoissonSolver.H index da0078f8b5a..26a4c72208d 100644 --- a/Source/ablastr/fields/PoissonSolver.H +++ b/Source/ablastr/fields/PoissonSolver.H @@ -9,13 +9,14 @@ #include #include +#include #include #include #include #include #include -#if defined(WARPX_USE_PSATD) && defined(WARPX_DIM_3D) +#if defined(ABLASTR_USE_FFT) && defined(WARPX_DIM_3D) #include #endif @@ -81,8 +82,9 @@ namespace ablastr::fields { * \param[in] geom the geometry per level (e.g., from AmrMesh) * \param[in] dmap the distribution mapping per level (e.g., from AmrMesh) * \param[in] grids the grids per level (e.g., from AmrMesh) + * \param[in] grid_type Integer that corresponds to the type of grid used in the simulation (collocated, staggered, hybrid) * \param[in] boundary_handler a handler for boundary conditions, for example @see ElectrostaticSolver::PoissonBoundaryHandler - * \param[in] is_solver_multigrid boolean to select the Poisson solver: 1 for Multigrid, 0 for FFT + * \param[in] is_solver_igf_on_lev0 boolean to select the Poisson solver: 1 for FFT on level 0 & Multigrid on other levels, 0 for Multigrid on all levels * \param[in] do_single_precision_comms perform communications in single precision * \param[in] rel_ref_ratio mesh refinement ratio between levels (default: 1) * \param[in] post_phi_calculation perform a calculation per level directly after phi was calculated; required for embedded boundaries (default: none) @@ -105,8 +107,9 @@ computePhi (amrex::Vector const & rho, amrex::Vector const& geom, amrex::Vector const& dmap, amrex::Vector const& grids, + utils::enums::GridType grid_type, T_BoundaryHandler const boundary_handler, - [[maybe_unused]] bool is_solver_multigrid, + bool is_solver_igf_on_lev0, bool const do_single_precision_comms = false, std::optional > rel_ref_ratio = std::nullopt, [[maybe_unused]] T_PostPhiCalculationFunctor post_phi_calculation = std::nullopt, @@ -116,7 +119,7 @@ computePhi (amrex::Vector const & rho, { using namespace amrex::literals; - ABLASTR_PROFILE("computePhi", false); + ABLASTR_PROFILE("computePhi"); if (!rel_ref_ratio.has_value()) { ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(rho.size() == 1u, @@ -160,9 +163,19 @@ computePhi (amrex::Vector const & rho, {{ beta[0], beta[1], beta[2] }}; #endif -#if (defined(WARPX_USE_PSATD) && defined(WARPX_DIM_3D)) +#if !defined(ABLASTR_USE_FFT) + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( !is_solver_igf_on_lev0, + "Must compile with FFT support to use the IGF solver!"); +#endif + +#if !defined(WARPX_DIM_3D) + ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE( !is_solver_igf_on_lev0, + "The FFT Poisson solver is currently only implemented for 3D!"); +#endif + +#if (defined(ABLASTR_USE_FFT) && defined(WARPX_DIM_3D)) // Use the Integrated Green Function solver (FFT) on the coarsest level if it was selected - if(!is_solver_multigrid && lev==0){ + if(is_solver_igf_on_lev0 && lev==0){ amrex::Array const dx_igf {AMREX_D_DECL(geom[lev].CellSize(0)/std::sqrt(1._rt-beta_solver[0]*beta_solver[0]), geom[lev].CellSize(1)/std::sqrt(1._rt-beta_solver[1]*beta_solver[1]), @@ -253,10 +266,15 @@ computePhi (amrex::Vector const & rho, mlmg.setVerbose(verbosity); mlmg.setMaxIter(max_iters); mlmg.setAlwaysUseBNorm(always_use_bnorm); + if (grid_type == utils::enums::GridType::Collocated) { + // In this case, computeE needs to use ghost nodes data. So we + // ask MLMG to fill BC for us after it solves the problem. + mlmg.setFinalFillBC(true); + } // Solve Poisson equation at lev mlmg.solve( {phi[lev]}, {rho[lev]}, - relative_tolerance, absolute_tolerance ); + relative_tolerance, absolute_tolerance ); // needed for solving the levels by levels: // - coarser level is initial guess for finer level @@ -270,10 +288,14 @@ computePhi (amrex::Vector const & rho, const amrex::IntVect& refratio = rel_ref_ratio.value()[lev]; ba.coarsen(refratio); const int ncomp = linop.getNComp(); - amrex::MultiFab phi_cp(ba, phi[lev+1]->DistributionMap(), ncomp, 1); + const int ng = (grid_type == utils::enums::GridType::Collocated) ? 1 : 0; + amrex::MultiFab phi_cp(ba, phi[lev+1]->DistributionMap(), ncomp, ng); + if (ng > 0) { + // Set all values outside the domain to zero + phi_cp.setDomainBndry(0.0_rt, geom[lev]); + } // Copy from phi[lev] to phi_cp (in parallel) - const amrex::IntVect& ng = amrex::IntVect::TheUnitVector(); const amrex::Periodicity& crse_period = geom[lev].periodicity(); ablastr::utils::communication::ParallelCopy( @@ -282,8 +304,8 @@ computePhi (amrex::Vector const & rho, 0, 0, 1, - ng, - ng, + amrex::IntVect(0), + amrex::IntVect(ng), do_single_precision_comms, crse_period ); @@ -298,14 +320,14 @@ computePhi (amrex::Vector const & rho, details::PoissonInterpCPtoFP const interp(phi_fp_arr, phi_cp_arr, refratio); - amrex::Box const b = mfi.tilebox(phi[lev + 1]->ixType().toIntVect()); + amrex::Box const& b = mfi.growntilebox(ng); amrex::ParallelFor(b, interp); } } // Run additional operations, such as calculation of the E field for embedded boundaries - if constexpr (!std::is_same::value) { + if constexpr (!std::is_same_v) { if (post_phi_calculation.has_value()) { post_phi_calculation.value()(mlmg, lev); } diff --git a/Source/ablastr/fields/VectorPoissonSolver.H b/Source/ablastr/fields/VectorPoissonSolver.H index d49335723d8..3ef96c30c84 100644 --- a/Source/ablastr/fields/VectorPoissonSolver.H +++ b/Source/ablastr/fields/VectorPoissonSolver.H @@ -241,7 +241,7 @@ computeVectorPotential ( amrex::Vector > co curr[lev][adim]->mult(-1._rt/ablastr::constant::SI::mu0); } // Loop over adim // Run additional operations, such as calculation of the B fields for embedded boundaries - if constexpr (!std::is_same::value) { + if constexpr (!std::is_same_v) { if (post_A_calculation.has_value()) { post_A_calculation.value()(mlmg, lev); } diff --git a/Source/ablastr/math/fft/CMakeLists.txt b/Source/ablastr/math/fft/CMakeLists.txt index f7d689c98e9..913a912e1ee 100644 --- a/Source/ablastr/math/fft/CMakeLists.txt +++ b/Source/ablastr/math/fft/CMakeLists.txt @@ -1,6 +1,6 @@ foreach(D IN LISTS WarpX_DIMS) warpx_set_suffix_dims(SD ${D}) - if(WarpX_PSATD STREQUAL ON) + if(ABLASTR_FFT STREQUAL ON) if(WarpX_COMPUTE STREQUAL CUDA) target_sources(ablastr_${SD} PRIVATE WrapCuFFT.cpp) elseif(WarpX_COMPUTE STREQUAL HIP) diff --git a/Source/ablastr/math/fft/Make.package b/Source/ablastr/math/fft/Make.package index b04062bd9d5..63786cdc006 100644 --- a/Source/ablastr/math/fft/Make.package +++ b/Source/ablastr/math/fft/Make.package @@ -1,4 +1,4 @@ -ifeq ($(USE_PSATD),TRUE) +ifeq ($(USE_FFT),TRUE) ifeq ($(USE_CUDA),TRUE) CEXE_sources += WrapCuFFT.cpp else ifeq ($(USE_HIP),TRUE) diff --git a/Source/ablastr/math/fft/WrapCuFFT.cpp b/Source/ablastr/math/fft/WrapCuFFT.cpp index b6b1706f807..9ceb91457c9 100644 --- a/Source/ablastr/math/fft/WrapCuFFT.cpp +++ b/Source/ablastr/math/fft/WrapCuFFT.cpp @@ -31,7 +31,7 @@ namespace ablastr::math::anyfft Complex * const complex_array, const direction dir, const int dim) { FFTplan fft_plan; - ABLASTR_PROFILE("ablastr::math::anyfft::CreatePlan", false); + ABLASTR_PROFILE("ablastr::math::anyfft::CreatePlan"); // Initialize fft_plan.m_plan with the vendor fft plan. cufftResult result; @@ -71,12 +71,12 @@ namespace ablastr::math::anyfft void DestroyPlan(FFTplan& fft_plan) { - ABLASTR_PROFILE("ablastr::math::anyfft::DestroyPlan", false); + ABLASTR_PROFILE("ablastr::math::anyfft::DestroyPlan"); cufftDestroy( fft_plan.m_plan ); } void Execute(FFTplan& fft_plan){ - ABLASTR_PROFILE("ablastr::math::anyfft::Execute", false); + ABLASTR_PROFILE("ablastr::math::anyfft::Execute"); // make sure that this is done on the same GPU stream as the above copy cudaStream_t stream = amrex::Gpu::Device::cudaStream(); cufftSetStream ( fft_plan.m_plan, stream); diff --git a/Source/ablastr/parallelization/MPIInitHelpers.cpp b/Source/ablastr/parallelization/MPIInitHelpers.cpp index 513035c483c..5ba15a9e64d 100644 --- a/Source/ablastr/parallelization/MPIInitHelpers.cpp +++ b/Source/ablastr/parallelization/MPIInitHelpers.cpp @@ -35,7 +35,7 @@ namespace ablastr::parallelization #ifdef AMREX_USE_MPI # ifdef AMREX_MPI_THREAD_MULTIPLE // i.e. for async_io return MPI_THREAD_MULTIPLE; -# elif AMREX_USE_OMP +# elif defined(AMREX_USE_OMP) return MPI_THREAD_FUNNELED; # else return MPI_THREAD_SINGLE; // equiv. to MPI_Init diff --git a/Source/ablastr/particles/DepositCharge.H b/Source/ablastr/particles/DepositCharge.H index b2dedccd03e..75e3bca170d 100644 --- a/Source/ablastr/particles/DepositCharge.H +++ b/Source/ablastr/particles/DepositCharge.H @@ -34,7 +34,7 @@ namespace ablastr::particles * \param rho MultiFab of the charge density * \param local_rho temporary FArrayBox for deposition with OpenMP * \param particle_shape shape factor in each direction - * \param dx cell spacing at level lev + * \param dinv cell spacing inverses at level lev * \param xyzmin lo corner of the current tile in physical coordinates. * \param n_rz_azimuthal_modes number of azimuthal modes in use, irrelevant outside RZ geometry (default: 0) * \param num_rho_deposition_guards number of ghost cells to use for rho (default: rho.nGrowVect()) @@ -44,7 +44,6 @@ namespace ablastr::particles * \param np_to_deposit number of particles to deposit (default: pti.numParticles()) * \param icomp component in MultiFab to start depositing to * \param nc number of components to deposit - * \param do_device_synchronize call amrex::Gpu::synchronize() for tiny profiler regions (default: true) */ template< typename T_PC > static void @@ -55,16 +54,15 @@ deposit_charge (typename T_PC::ParIterType& pti, amrex::MultiFab* rho, amrex::FArrayBox& local_rho, int const particle_shape, - std::array const & dx, - std::array const & xyzmin, + const amrex::XDim3 dinv, + const amrex::XDim3 xyzmin, int const n_rz_azimuthal_modes = 0, std::optional num_rho_deposition_guards = std::nullopt, std::optional depos_lev = std::nullopt, std::optional rel_ref_ratio = std::nullopt, long const offset = 0, std::optional np_to_deposit = std::nullopt, - int const icomp = 0, int const nc = 1, - bool const do_device_synchronize = true) + int const icomp = 0, int const nc = 1) { // deposition guards amrex::IntVect ng_rho = rho->nGrowVect(); @@ -131,8 +129,8 @@ deposit_charge (typename T_PC::ParIterType& pti, amrex::numParticlesOutOfRange(pti, range) == 0, "Particles shape does not fit within tile (CPU) or guard cells (GPU) used for charge deposition"); - ABLASTR_PROFILE_VAR_NS("ablastr::particles::deposit_charge::ChargeDeposition", blp_ppc_chd, do_device_synchronize); - ABLASTR_PROFILE_VAR_NS("ablastr::particles::deposit_charge::Accumulate", blp_accumulate, do_device_synchronize); + ABLASTR_PROFILE_VAR_NS("ablastr::particles::deposit_charge::ChargeDeposition", blp_ppc_chd); + ABLASTR_PROFILE_VAR_NS("ablastr::particles::deposit_charge::Accumulate", blp_accumulate); // Get tile box where charge is deposited. // The tile box is different when depositing in the buffers (depos_lev(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dinv, xyzmin, lo, charge, n_rz_azimuthal_modes); } else if (nox == 2){ doChargeDepositionShapeN<2>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dinv, xyzmin, lo, charge, n_rz_azimuthal_modes); } else if (nox == 3){ doChargeDepositionShapeN<3>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dinv, xyzmin, lo, charge, n_rz_azimuthal_modes); } else if (nox == 4){ doChargeDepositionShapeN<4>(GetPosition, wp.dataPtr()+offset, ion_lev, - rho_fab, np_to_deposit.value(), dx, xyzmin, lo, charge, + rho_fab, np_to_deposit.value(), dinv, xyzmin, lo, charge, n_rz_azimuthal_modes); } - ABLASTR_PROFILE_VAR_STOP(blp_ppc_chd, do_device_synchronize); + ABLASTR_PROFILE_VAR_STOP(blp_ppc_chd); #ifndef AMREX_USE_GPU // CPU, tiling: atomicAdd local_rho into rho - ABLASTR_PROFILE_VAR_START(blp_accumulate, do_device_synchronize); + ABLASTR_PROFILE_VAR_START(blp_accumulate); (*rho)[pti].lockAdd(local_rho, tb, tb, 0, icomp*nc, nc); - ABLASTR_PROFILE_VAR_STOP(blp_accumulate, do_device_synchronize); + ABLASTR_PROFILE_VAR_STOP(blp_accumulate); #endif } diff --git a/Source/ablastr/profiler/ProfilerWrapper.H b/Source/ablastr/profiler/ProfilerWrapper.H index 6b476d78114..268c1abd971 100644 --- a/Source/ablastr/profiler/ProfilerWrapper.H +++ b/Source/ablastr/profiler/ProfilerWrapper.H @@ -9,54 +9,12 @@ #define ABLASTR_PROFILERWRAPPER_H_ #include -#include - -namespace ablastr::profiler -{ - /** Conditionally synchronizes active GPU operations - * - * @param do_device_synchronize perform amrex::Gpu::synchronize() if true - */ - AMREX_FORCE_INLINE - void - device_synchronize(bool const do_device_synchronize = false) { - if (do_device_synchronize) { - amrex::Gpu::synchronize(); - } - } - - /** An object that conditionally calls device_synchronize() on destruction - * - * Note that objects are destructed in the reverse order of declaration - */ - struct SynchronizeOnDestruct { - SynchronizeOnDestruct(bool const do_device_synchronize = false) - : m_do_device_synchronize(do_device_synchronize) {} - - AMREX_FORCE_INLINE - ~SynchronizeOnDestruct() { - device_synchronize(m_do_device_synchronize); - } - - // default move and copy operations - SynchronizeOnDestruct(const SynchronizeOnDestruct&) = default; - SynchronizeOnDestruct& operator=(const SynchronizeOnDestruct&) = default; - SynchronizeOnDestruct(SynchronizeOnDestruct&&) = default; - SynchronizeOnDestruct& operator=(SynchronizeOnDestruct&& field_data) = default; - - bool m_do_device_synchronize = false; - }; - -} // namespace ablastr::profiler - -// `BL_PROFILE_PASTE(SYNC_SCOPE_, __COUNTER__)` and `SYNC_V_##vname` used to make unique names for -// synchronizeOnDestruct objects, like `SYNC_SCOPE_0` and `SYNC_V_pmain` -#define ABLASTR_PROFILE(fname, sync) ablastr::profiler::device_synchronize(sync); BL_PROFILE(fname); const ablastr::profiler::SynchronizeOnDestruct BL_PROFILE_PASTE(SYNC_SCOPE_, __COUNTER__){sync} -#define ABLASTR_PROFILE_VAR(fname, vname, sync) ablastr::profiler::device_synchronize(sync); BL_PROFILE_VAR(fname, vname); const ablastr::profiler::SynchronizeOnDestruct SYNC_V_##vname{sync} -#define ABLASTR_PROFILE_VAR_NS(fname, vname, sync) BL_PROFILE_VAR_NS(fname, vname); const ablastr::profiler::SynchronizeOnDestruct SYNC_V_##vname{sync} -#define ABLASTR_PROFILE_VAR_START(vname, sync) ablastr::profiler::device_synchronize(sync); BL_PROFILE_VAR_START(vname) -#define ABLASTR_PROFILE_VAR_STOP(vname, sync) ablastr::profiler::device_synchronize(sync); BL_PROFILE_VAR_STOP(vname) -#define ABLASTR_PROFILE_REGION(rname, sync) ablastr::profiler::device_synchronize(sync); BL_PROFILE_REGION(rname); const ablastr::profiler::SynchronizeOnDestruct BL_PROFILE_PASTE(SYNC_R_, __COUNTER__){sync} +#define ABLASTR_PROFILE(fname) BL_PROFILE(fname) +#define ABLASTR_PROFILE_VAR(fname, vname) BL_PROFILE_VAR(fname, vname) +#define ABLASTR_PROFILE_VAR_NS(fname, vname) BL_PROFILE_VAR_NS(fname, vname) +#define ABLASTR_PROFILE_VAR_START(vname) BL_PROFILE_VAR_START(vname) +#define ABLASTR_PROFILE_VAR_STOP(vname) BL_PROFILE_VAR_STOP(vname) +#define ABLASTR_PROFILE_REGION(rname) BL_PROFILE_REGION(rname) #endif // ABLASTR_PROFILERWRAPPER_H_ diff --git a/Source/ablastr/utils/Enums.H b/Source/ablastr/utils/Enums.H new file mode 100644 index 00000000000..1f89bede9e4 --- /dev/null +++ b/Source/ablastr/utils/Enums.H @@ -0,0 +1,35 @@ +/* Copyright 2024 Axel Huebl + * + * This file is part of ABLASTR. + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef ABLASTR_UTILS_ENUMS_H_ +#define ABLASTR_UTILS_ENUMS_H_ + +namespace ablastr::utils::enums +{ + /** Type of grids used in a simulation: + * + * Collocated at the same location (AMReX: all "NODAL"), staggered (Yee-style), or hybrid. + */ + enum struct GridType { + Collocated = 0, + Staggered = 1, + Hybrid = 2 + }; + + /** Mesh-refinement patch + * + * The fine or coarse patch (in terms of spatial resolution) on the same MR level. + * https://warpx.readthedocs.io/en/latest/theory/amr.html + */ + enum struct PatchType { + fine, + coarse + }; + +} // namespace ablastr::utils::enums + +#endif // ABLASTR_UTILS_ENUMS_H_ diff --git a/Source/ablastr/utils/SignalHandling.cpp b/Source/ablastr/utils/SignalHandling.cpp index 4095b9207ce..5eeaeec259f 100644 --- a/Source/ablastr/utils/SignalHandling.cpp +++ b/Source/ablastr/utils/SignalHandling.cpp @@ -104,7 +104,7 @@ SignalHandling::parseSignalNameToNumber (const std::string &str) auto spf = signals_parser.compileHost<0>(); - const int sig = spf(); + const auto sig = int(spf()); ABLASTR_ALWAYS_ASSERT_WITH_MESSAGE(sig < NUM_SIGNALS, "Parsed signal value is outside the supported range of [1, 31]"); diff --git a/Source/main.cpp b/Source/main.cpp index 2a1b828c64f..9273cd3928b 100644 --- a/Source/main.cpp +++ b/Source/main.cpp @@ -8,25 +8,16 @@ */ #include "WarpX.H" -#include "Initialization/WarpXAMReXInit.H" +#include "Initialization/WarpXInit.H" #include "Utils/WarpXProfilerWrapper.H" -#include -#include #include -#include #include - int main(int argc, char* argv[]) { - ablastr::parallelization::mpi_init(argc, argv); - - warpx::initialization::amrex_init(argc, argv); - - ablastr::math::anyfft::setup(); - + warpx::initialization::initialize_external_libraries(argc, argv); { WARPX_PROFILE_VAR("main()", pmain); @@ -34,29 +25,18 @@ int main(int argc, char* argv[]) timer.record_start_time(); auto& warpx = WarpX::GetInstance(); - warpx.InitData(); - warpx.Evolve(); - - amrex::Print() << - ablastr::warn_manager::GetWMInstance().PrintGlobalWarnings("THE END"); + const auto is_warpx_verbose = warpx.Verbose(); + WarpX::Finalize(); timer.record_stop_time(); - if (warpx.Verbose()) - { + if (is_warpx_verbose){ amrex::Print() << "Total Time : " << timer.get_global_duration() << '\n'; } WARPX_PROFILE_VAR_STOP(pmain); - - WarpX::Finalize(); } - - ablastr::math::anyfft::cleanup(); - - amrex::Finalize(); - - ablastr::parallelization::mpi_finalize (); + warpx::initialization::finalize_external_libraries(); } diff --git a/Tools/Linter/runClangTidy.sh b/Tools/Linter/runClangTidy.sh index 39a96cc97d4..046c72d7b27 100755 --- a/Tools/Linter/runClangTidy.sh +++ b/Tools/Linter/runClangTidy.sh @@ -104,7 +104,7 @@ cmake -S ${REPO_DIR} -B ${REPO_DIR}/build_clang_tidy \ -DWarpX_DIMS="1;2;3;RZ" \ -DWarpX_MPI=ON \ -DWarpX_COMPUTE=OMP \ - -DWarpX_PSATD=ON \ + -DWarpX_FFT=ON \ -DWarpX_QED=ON \ -DWarpX_QED_TABLE_GEN=ON \ -DWarpX_OPENPMD=ON \ diff --git a/Tools/QedTablesUtils/Source/QedTableGenerator.cpp b/Tools/QedTablesUtils/Source/QedTableGenerator.cpp index dfbde1df050..1ea62b5c6ed 100644 --- a/Tools/QedTablesUtils/Source/QedTableGenerator.cpp +++ b/Tools/QedTablesUtils/Source/QedTableGenerator.cpp @@ -106,7 +106,7 @@ template void GenerateTableBW (const ParsedArgs& args, const string& outfile_name) { cout << " Generating BW table " << - (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + (is_same_v ? "(double "s : "(single "s) << " precision)\n"s; if (!Contains(args, "--dndt_chi_min") || !Contains(args, "--dndt_chi_max") || !Contains(args, "--dndt_how_many") || !Contains(args, "--pair_chi_min") || @@ -173,7 +173,7 @@ template void GenerateTableQS (const ParsedArgs& args, const string& outfile_name) { cout << " Generating QS table " << - (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + (is_same_v ? "(double "s : "(single "s) << " precision)\n"s; if (!Contains(args, "--dndt_chi_min") || !Contains(args, "--dndt_chi_max") || !Contains(args, "--dndt_how_many") || !Contains(args, "--em_chi_min") || diff --git a/Tools/QedTablesUtils/Source/QedTableReader.cpp b/Tools/QedTablesUtils/Source/QedTableReader.cpp index ae689a87643..ba9d58775f2 100644 --- a/Tools/QedTablesUtils/Source/QedTableReader.cpp +++ b/Tools/QedTablesUtils/Source/QedTableReader.cpp @@ -117,7 +117,7 @@ template void ReadTableBW (const string& input_file, const string& outfile_name) { cout << " Reading BW table " << - (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + (is_same_v ? "(double "s : "(single "s) << " precision)\n"s; auto ifs = ifstream(input_file, std::ios::binary); auto raw_data = vector(istreambuf_iterator(ifs), {}); @@ -161,7 +161,7 @@ void ReadTableQS ( const string& input_file, const string& outfile_name) { cout << " Reading QS table " << - (is_same::value ? "(double "s : "(single "s) << " precision)\n"s; + (is_same_v ? "(double "s : "(single "s) << " precision)\n"s; auto ifs = ifstream(input_file, std::ios::binary); auto raw_data = vector(istreambuf_iterator(ifs), {}); diff --git a/Tools/machines/adastra-cines/adastra_warpx.profile.example b/Tools/machines/adastra-cines/adastra_warpx.profile.example index 0d55e869d6a..3cba4346421 100644 --- a/Tools/machines/adastra-cines/adastra_warpx.profile.example +++ b/Tools/machines/adastra-cines/adastra_warpx.profile.example @@ -11,10 +11,10 @@ module load CCE-GPU-3.0.0 module load amd-mixed/5.2.3 # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SHAREDHOMEDIR}/sw/adastra/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support module load boost/1.83.0-mpi-python3 diff --git a/Tools/machines/adastra-cines/install_dependencies.sh b/Tools/machines/adastra-cines/install_dependencies.sh index b48bf144c2a..242f5fc664c 100755 --- a/Tools/machines/adastra-cines/install_dependencies.sh +++ b/Tools/machines/adastra-cines/install_dependencies.sh @@ -38,14 +38,13 @@ if [ -d $SHAREDHOMEDIR/src/blaspp ] then cd $SHAREDHOMEDIR/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $SHAREDHOMEDIR/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $SHAREDHOMEDIR/src/blaspp fi rm -rf $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build -CXX=$(which CC) cmake -S $SHAREDHOMEDIR/src/blaspp -B $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which CC) cmake -S $SHAREDHOMEDIR/src/blaspp -B $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build --target install --parallel 16 rm -rf $SHAREDHOMEDIR/src/blaspp-adastra-gpu-build @@ -54,14 +53,13 @@ if [ -d $SHAREDHOMEDIR/src/lapackpp ] then cd $SHAREDHOMEDIR/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $SHAREDHOMEDIR/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $SHAREDHOMEDIR/src/lapackpp fi rm -rf $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $SHAREDHOMEDIR/src/lapackpp -B $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $SHAREDHOMEDIR/src/lapackpp -B $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build --target install --parallel 16 rm -rf $SHAREDHOMEDIR/src/lapackpp-adastra-gpu-build diff --git a/Tools/machines/cori-nersc/haswell_warpx.profile.example b/Tools/machines/cori-nersc/haswell_warpx.profile.example index b0ac225cc72..f636b1ab555 100644 --- a/Tools/machines/cori-nersc/haswell_warpx.profile.example +++ b/Tools/machines/cori-nersc/haswell_warpx.profile.example @@ -8,8 +8,8 @@ module load cray-python/3.9.7.1 export PKG_CONFIG_PATH=$FFTW_DIR/pkgconfig:$PKG_CONFIG_PATH export CMAKE_PREFIX_PATH=$HOME/sw/haswell/c-blosc-1.12.1-install:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=$HOME/sw/haswell/adios2-2.7.1-install:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/haswell/blaspp-master-install:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/haswell/lapackpp-master-install:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/haswell/blaspp-2024.05.31-install:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/haswell/lapackpp-2024.05.31-install:$CMAKE_PREFIX_PATH if [ -d "$HOME/sw/haswell/venvs/haswell_warpx" ] then diff --git a/Tools/machines/cori-nersc/knl_warpx.profile.example b/Tools/machines/cori-nersc/knl_warpx.profile.example index 093c7f62783..fdd7e14d594 100644 --- a/Tools/machines/cori-nersc/knl_warpx.profile.example +++ b/Tools/machines/cori-nersc/knl_warpx.profile.example @@ -9,8 +9,8 @@ module load cray-python/3.9.7.1 export PKG_CONFIG_PATH=$FFTW_DIR/pkgconfig:$PKG_CONFIG_PATH export CMAKE_PREFIX_PATH=$HOME/sw/knl/c-blosc-1.12.1-install:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=$HOME/sw/knl/adios2-2.7.1-install:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/knl/blaspp-master-install:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/knl/lapackpp-master-install:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/knl/blaspp-2024.05.31-install:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/knl/lapackpp-2024.05.31-install:$CMAKE_PREFIX_PATH if [ -d "$HOME/sw/knl/venvs/knl_warpx" ] then diff --git a/Tools/machines/crusher-olcf/crusher_warpx.profile.example b/Tools/machines/crusher-olcf/crusher_warpx.profile.example index ffc70a128e2..7b05546b3f3 100644 --- a/Tools/machines/crusher-olcf/crusher_warpx.profile.example +++ b/Tools/machines/crusher-olcf/crusher_warpx.profile.example @@ -21,10 +21,10 @@ module load ninja module load nano # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${HOME}/sw/crusher/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/crusher/gpu/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${HOME}/sw/crusher/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/crusher/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/crusher/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/crusher/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${HOME}/sw/crusher/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${HOME}/sw/crusher/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support module load boost/1.79.0-cxx17 diff --git a/Tools/machines/crusher-olcf/install_dependencies.sh b/Tools/machines/crusher-olcf/install_dependencies.sh index ab5deb6b0a6..6e9f97eddae 100755 --- a/Tools/machines/crusher-olcf/install_dependencies.sh +++ b/Tools/machines/crusher-olcf/install_dependencies.sh @@ -49,14 +49,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-crusher-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-crusher-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-crusher-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build $HOME/src/blaspp-crusher-gpu-build --target install --parallel 16 rm -rf $HOME/src/blaspp-crusher-gpu-build @@ -65,14 +64,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-crusher-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-crusher-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-crusher-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build $HOME/src/lapackpp-crusher-gpu-build --target install --parallel 16 rm -rf $HOME/src/lapackpp-crusher-gpu-build diff --git a/Tools/machines/desktop/spack-macos-openmp.yaml b/Tools/machines/desktop/spack-macos-openmp.yaml index 820cf7069fd..3ea78625b78 100644 --- a/Tools/machines/desktop/spack-macos-openmp.yaml +++ b/Tools/machines/desktop/spack-macos-openmp.yaml @@ -23,6 +23,7 @@ spack: - conduit ~fortran - fftw - hdf5 ~fortran + - heffte ~cuda +fftw - lapackpp ~cuda ~rocm ^blaspp ~cuda +openmp ~rocm - mpi - llvm-openmp diff --git a/Tools/machines/desktop/spack-ubuntu-cuda.yaml b/Tools/machines/desktop/spack-ubuntu-cuda.yaml index 08d0c95ee4b..19b9ae12e24 100644 --- a/Tools/machines/desktop/spack-ubuntu-cuda.yaml +++ b/Tools/machines/desktop/spack-ubuntu-cuda.yaml @@ -25,6 +25,7 @@ spack: - cuda - fftw - hdf5 + - heffte - lapackpp - mpi - pkgconfig diff --git a/Tools/machines/desktop/spack-ubuntu-openmp.yaml b/Tools/machines/desktop/spack-ubuntu-openmp.yaml index b658f1e009d..1eb7d4074a7 100644 --- a/Tools/machines/desktop/spack-ubuntu-openmp.yaml +++ b/Tools/machines/desktop/spack-ubuntu-openmp.yaml @@ -22,6 +22,7 @@ spack: - ecp-data-vis-sdk +adios2 +ascent +hdf5 +sensei - fftw - hdf5 + - heffte ~cuda +fftw - lapackpp ~cuda ~rocm ^blaspp ~cuda +openmp ~rocm - mpi - pkgconfig diff --git a/Tools/machines/desktop/spack-ubuntu-rocm.yaml b/Tools/machines/desktop/spack-ubuntu-rocm.yaml index 45c9b0f776e..7eee1baa13c 100644 --- a/Tools/machines/desktop/spack-ubuntu-rocm.yaml +++ b/Tools/machines/desktop/spack-ubuntu-rocm.yaml @@ -21,6 +21,7 @@ spack: - cmake - ecp-data-vis-sdk +adios2 +ascent +hdf5 +sensei - hdf5 + - heffte - hip - lapackpp - llvm-amdgpu diff --git a/Tools/machines/frontier-olcf/frontier_warpx.profile.example b/Tools/machines/frontier-olcf/frontier_warpx.profile.example index 2f86d047d00..f59f2d3d058 100644 --- a/Tools/machines/frontier-olcf/frontier_warpx.profile.example +++ b/Tools/machines/frontier-olcf/frontier_warpx.profile.example @@ -8,9 +8,9 @@ if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in yo # required dependencies module load cmake/3.23.2 module load craype-accel-amd-gfx90a -module load rocm/5.2.0 # waiting for 5.6 for next bump -module load cray-mpich -module load cce/15.0.0 # must be loaded after rocm +module load rocm/5.7.1 +module load cray-mpich/8.1.28 +module load cce/17.0.0 # must be loaded after rocm # optional: faster builds module load ccache @@ -20,20 +20,20 @@ module load ninja module load nano # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${HOME}/sw/frontier/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/frontier/gpu/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${HOME}/sw/frontier/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/frontier/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/frontier/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/frontier/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${HOME}/sw/frontier/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${HOME}/sw/frontier/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support -module load boost/1.79.0-cxx17 +module load boost/1.79.0 # optional: for openPMD support -module load adios2/2.8.3 -module load hdf5/1.14.0 +module load adios2/2.8.3-mpi +module load hdf5/1.12.1-mpi # optional: for Python bindings or libEnsemble -module load cray-python/3.9.13.1 +module load cray-python/3.11.5 if [ -d "${HOME}/sw/frontier/gpu/venvs/warpx-frontier" ] then diff --git a/Tools/machines/frontier-olcf/install_dependencies.sh b/Tools/machines/frontier-olcf/install_dependencies.sh index 98c30cfca8f..fd1d28e76b5 100755 --- a/Tools/machines/frontier-olcf/install_dependencies.sh +++ b/Tools/machines/frontier-olcf/install_dependencies.sh @@ -49,14 +49,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-frontier-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-frontier-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-frontier-gpu-build -Duse_openmp=OFF -Dgpu_backend=hip -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build $HOME/src/blaspp-frontier-gpu-build --target install --parallel 16 rm -rf $HOME/src/blaspp-frontier-gpu-build @@ -65,18 +64,16 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-frontier-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-frontier-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-frontier-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build $HOME/src/lapackpp-frontier-gpu-build --target install --parallel 16 rm -rf $HOME/src/lapackpp-frontier-gpu-build - # Python ###################################################################### # python3 -m pip install --upgrade pip @@ -111,8 +108,6 @@ CUPY_INSTALL_USE_HIP=1 \ ROCM_HOME=${ROCM_PATH} \ HCC_AMDGPU_TARGET=${AMREX_AMD_ARCH} \ python3 -m pip install -v cupy -# optional: for libEnsemble -python3 -m pip install -r $HOME/src/warpx/Tools/LibEnsemble/requirements.txt # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) #python3 -m pip install --upgrade torch --index-url https://download.pytorch.org/whl/rocm5.4.2 #python3 -m pip install -r $HOME/src/warpx/Tools/optimas/requirements.txt diff --git a/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example b/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example index 98476eecfab..c08255e7962 100644 --- a/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example +++ b/Tools/machines/greatlakes-umich/greatlakes_v100_warpx.profile.example @@ -22,13 +22,13 @@ module load phdf5/1.12.1 SW_DIR="${HOME}/sw/greatlakes/v100" export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc2-2.14.4:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.10.0:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${SW_DIR}/c-blosc2-2.14.4/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.10.0/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH export PATH=${SW_DIR}/adios2-2.10.0/bin:${PATH} diff --git a/Tools/machines/greatlakes-umich/install_v100_dependencies.sh b/Tools/machines/greatlakes-umich/install_v100_dependencies.sh index 9361fd6c64b..30faec52421 100755 --- a/Tools/machines/greatlakes-umich/install_v100_dependencies.sh +++ b/Tools/machines/greatlakes-umich/install_v100_dependencies.sh @@ -80,14 +80,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-v100-build -cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-v100-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-v100-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-v100-build --target install --parallel 8 rm -rf ${build_dir}/blaspp-v100-build @@ -96,14 +95,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-v100-build -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-v100-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-v100-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build ${build_dir}/lapackpp-v100-build --target install --parallel 8 rm -rf ${build_dir}/lapackpp-v100-build diff --git a/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example b/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example index 9f41cc7e337..017613f9d60 100644 --- a/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example +++ b/Tools/machines/hpc3-uci/hpc3_gpu_warpx.profile.example @@ -19,13 +19,13 @@ module load OpenBLAS/0.3.21 module load hdf5/1.13.1/gcc.11.2.0-openmpi.4.1.2 export CMAKE_PREFIX_PATH=${HOME}/sw/hpc3/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${HOME}/sw/hpc3/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/hpc3/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${HOME}/sw/hpc3/gpu/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/hpc3/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${HOME}/sw/hpc3/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${HOME}/sw/hpc3/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH export PATH=${HOME}/sw/hpc3/gpu/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh index 51d4bcfe2c5..b585c2702b6 100755 --- a/Tools/machines/hpc3-uci/install_gpu_dependencies.sh +++ b/Tools/machines/hpc3-uci/install_gpu_dependencies.sh @@ -79,14 +79,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-pm-gpu-build -cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build $HOME/src/blaspp-pm-gpu-build --target install --parallel 8 rm -rf $HOME/src/blaspp-pm-gpu-build @@ -95,14 +94,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-pm-gpu-build -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build $HOME/src/lapackpp-pm-gpu-build --target install --parallel 8 rm -rf $HOME/src/lapackpp-pm-gpu-build diff --git a/Tools/machines/karolina-it4i/install_dependencies.sh b/Tools/machines/karolina-it4i/install_dependencies.sh index 0435b5e2926..cba455f3d29 100755 --- a/Tools/machines/karolina-it4i/install_dependencies.sh +++ b/Tools/machines/karolina-it4i/install_dependencies.sh @@ -53,7 +53,7 @@ python -m pip install --user --upgrade matplotlib #python -m pip install --user --upgrade yt # install or update WarpX dependencies -python -m pip install --user --upgrade picmistandard==0.28.0 +python -m pip install --user --upgrade picmistandard==0.29.0 python -m pip install --user --upgrade lasy # optional: for optimas (based on libEnsemble & ax->botorch->gpytorch->pytorch) diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies.sh b/Tools/machines/lassen-llnl/install_v100_dependencies.sh deleted file mode 100755 index fe8285b7501..00000000000 --- a/Tools/machines/lassen-llnl/install_v100_dependencies.sh +++ /dev/null @@ -1,144 +0,0 @@ -#!/bin/bash -# -# Copyright 2023 The WarpX Community -# -# This file is part of WarpX. -# -# Author: Axel Huebl -# License: BSD-3-Clause-LBNL - -# Exit on first error encountered ############################################# -# -set -eu -o pipefail - - -# Check: ###################################################################### -# -# Was lassen_v100_warpx.profile sourced and configured correctly? -if [ -z ${proj-} ]; then echo "WARNING: The 'proj' variable is not yet set in your lassen_v100_warpx.profile file! Please edit its line 2 to continue!"; exit 1; fi - - -# Remove old dependencies ##################################################### -# -SRC_DIR="/usr/workspace/${USER}/lassen/src" -SW_DIR="/usr/workspace/${USER}/lassen/gpu" -rm -rf ${SW_DIR} -mkdir -p ${SW_DIR} -mkdir -p ${SRC_DIR} - -# remove common user mistakes in python, located in .local instead of a venv -python3 -m pip uninstall -qq -y pywarpx -python3 -m pip uninstall -qq -y warpx -python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true - - -# General extra dependencies ################################################## -# - -# tmpfs build directory: avoids issues often seen with $HOME and is faster -build_dir=$(mktemp -d) - -# c-blosc (I/O compression) -if [ -d ${SRC_DIR}/c-blosc ] -then - cd ${SRC_DIR}/c-blosc - git fetch --prune - git checkout v1.21.1 - cd - -else - git clone -b v1.21.1 https://github.com/Blosc/c-blosc.git ${SRC_DIR}/c-blosc -fi -cmake -S ${SRC_DIR}/c-blosc -B ${build_dir}/c-blosc-lassen-build -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DDEACTIVATE_AVX2=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.1 -cmake --build ${build_dir}/c-blosc-lassen-build --target install --parallel 10 - -# HDF5 -if [ -d ${SRC_DIR}/hdf5 ] -then - cd ${SRC_DIR}/hdf5 - git fetch --prune - git checkout hdf5-1_14_1-2 - cd - -else - git clone -b hdf5-1_14_1-2 https://github.com/HDFGroup/hdf5.git ${SRC_DIR}/hdf5 -fi -cmake -S ${SRC_DIR}/hdf5 -B ${build_dir}/hdf5-lassen-build -DBUILD_TESTING=OFF -DHDF5_ENABLE_PARALLEL=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/hdf5-1.14.1.2 -cmake --build ${build_dir}/hdf5-lassen-build --target install --parallel 10 - -# ADIOS2 -if [ -d ${SRC_DIR}/adios2 ] -then - cd ${SRC_DIR}/adios2 - git fetch --prune - git checkout v2.8.3 - cd - -else - git clone -b v2.8.3 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2 -fi -cmake -S ${SRC_DIR}/adios2 -B ${build_dir}/adios2-lassen-build -DBUILD_TESTING=OFF -DADIOS2_BUILD_EXAMPLES=OFF -DADIOS2_USE_Blosc=ON -DADIOS2_USE_Fortran=OFF -DADIOS2_USE_Python=OFF -DADIOS2_USE_SST=OFF -DADIOS2_USE_ZeroMQ=OFF -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.8.3 -cmake --build ${build_dir}/adios2-lassen-build --target install -j 10 - -# BLAS++ (for PSATD+RZ) -if [ -d ${SRC_DIR}/blaspp ] -then - cd ${SRC_DIR}/blaspp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp -fi -cmake -S ${SRC_DIR}/blaspp -B ${build_dir}/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master -cmake --build ${build_dir}/blaspp-lassen-build --target install --parallel 10 - -# LAPACK++ (for PSATD+RZ) -if [ -d ${SRC_DIR}/lapackpp ] -then - cd ${SRC_DIR}/lapackpp - git fetch --prune - git checkout master - git pull - cd - -else - git clone https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp -fi -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp -B ${build_dir}/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so -cmake --build ${build_dir}/lapackpp-lassen-build --target install --parallel 10 - - -# Python ###################################################################### -# -# sometimes, the Lassen PIP Index is down -export PIP_EXTRA_INDEX_URL="https://pypi.org/simple" - -python3 -m pip install --upgrade --user virtualenv -rm -rf ${SW_DIR}/venvs/warpx-lassen -python3 -m venv ${SW_DIR}/venvs/warpx-lassen -source ${SW_DIR}/venvs/warpx-lassen/bin/activate -python3 -m pip install --upgrade pip -python3 -m pip cache purge -python3 -m pip install --upgrade build -python3 -m pip install --upgrade packaging -python3 -m pip install --upgrade wheel -python3 -m pip install --upgrade setuptools -# Older version for h4py -# https://github.com/h5py/h5py/issues/2268 -python3 -m pip install --upgrade "cython<3" -python3 -m pip install --upgrade numpy -python3 -m pip install --upgrade pandas -python3 -m pip install --upgrade -Ccompile-args="-j10" scipy -python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py -python3 -m pip install --upgrade openpmd-api -CC=mpicc H5PY_SETUP_REQUIRES=0 HDF5_DIR=${SW_DIR}/hdf5-1.14.1.2 HDF5_MPI=ON python3 -m pip install --upgrade h5py --no-cache-dir --no-build-isolation --no-binary h5py -MPLLOCALFREETYPE=1 python3 -m pip install --upgrade matplotlib==3.2.2 # does not try to build freetype itself -echo "matplotlib==3.2.2" > ${build_dir}/constraints.txt -python3 -m pip install --upgrade -c ${build_dir}/constraints.txt yt - -# install or update WarpX dependencies such as picmistandard -python3 -m pip install --upgrade -r ${SRC_DIR}/warpx/requirements.txt - -# for ML dependencies, see install_v100_ml.sh - - -# remove build temporary directory -rm -rf ${build_dir} diff --git a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh b/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh index 916986ee119..1b14159cd22 100644 --- a/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh +++ b/Tools/machines/lassen-llnl/install_v100_dependencies_toss3.sh @@ -82,13 +82,12 @@ if [ -d ${SRC_DIR}/blaspp ] then cd ${SRC_DIR}/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp fi -cmake -S ${SRC_DIR}/blaspp -B ${build_dir}/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake -S ${SRC_DIR}/blaspp -B ${build_dir}/blaspp-lassen-build -Duse_openmp=ON -Dgpu_backend=cuda -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-lassen-build --target install --parallel 10 # LAPACK++ (for PSATD+RZ) @@ -96,13 +95,12 @@ if [ -d ${SRC_DIR}/lapackpp ] then cd ${SRC_DIR}/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp fi -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp -B ${build_dir}/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp -B ${build_dir}/lapackpp-lassen-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 -DLAPACK_LIBRARIES=/usr/lib64/liblapack.so cmake --build ${build_dir}/lapackpp-lassen-build --target install --parallel 10 @@ -121,9 +119,7 @@ python3 -m pip install --upgrade build python3 -m pip install --upgrade packaging python3 -m pip install --upgrade wheel python3 -m pip install --upgrade setuptools -# Older version for h4py -# https://github.com/h5py/h5py/issues/2268 -python3 -m pip install --upgrade "cython<3" +python3 -m pip install --upgrade cython python3 -m pip install --upgrade numpy python3 -m pip install --upgrade pandas CMAKE_PREFIX_PATH=/usr/lib64:${CMAKE_PREFIX_PATH} python3 -m pip install --upgrade -Ccompile-args="-j10" -Csetup-args=-Dblas=BLAS -Csetup-args=-Dlapack=BLAS scipy diff --git a/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example b/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example deleted file mode 100644 index 4d4efbf917d..00000000000 --- a/Tools/machines/lassen-llnl/lassen_v100_warpx.profile.example +++ /dev/null @@ -1,56 +0,0 @@ -# please set your project account -#export proj="" # edit this and comment in - -# required dependencies -module load cmake/3.23.1 -module load gcc/11.2.1 -module load cuda/12.0.0 - -# optional: for QED lookup table generation support -module load boost/1.70.0 - -# optional: for openPMD support -SRC_DIR="/usr/workspace/${USER}/lassen/src" -SW_DIR="/usr/workspace/${USER}/lassen/gpu" -export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.1:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/hdf5-1.14.1.2:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/hdf5-1.14.1.2/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export PATH=${SW_DIR}/hdf5-1.14.1.2/bin:${PATH} -export PATH=${SW_DIR}/adios2-2.8.3/bin:$PATH - -# optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH - -# optional: for Python bindings -module load python/3.8.2 - -if [ -d "${SW_DIR}/venvs/warpx-lassen" ] -then - source ${SW_DIR}/venvs/warpx-lassen/bin/activate -fi - -# optional: an alias to request an interactive node for two hours -alias getNode="bsub -G $proj -W 2:00 -nnodes 1 -Is /bin/bash" -# an alias to run a command on a batch node for up to 30min -# usage: runNode -alias runNode="bsub -q debug -P $proj -W 2:00 -nnodes 1 -I" - -# fix system defaults: do not escape $ with a \ on tab completion -shopt -s direxpand - -# optimize CUDA compilation for V100 -export AMREX_CUDA_ARCH=7.0 -export CUDAARCHS=70 - -# compiler environment hints -export CC=$(which gcc) -export CXX=$(which g++) -export FC=$(which gfortran) -export CUDACXX=$(which nvcc) -export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example b/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example index abad909943c..98e8d6410b3 100644 --- a/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example +++ b/Tools/machines/lassen-llnl/lassen_v100_warpx_toss3.profile.example @@ -22,13 +22,13 @@ export PATH=${SW_DIR}/hdf5-1.14.1.2/bin:${PATH} export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH} # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for Python bindings -module load python/3.8.2 +module load python/3.11.5 if [ -d "${SW_DIR}/venvs/warpx-lassen-toss3" ] then diff --git a/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example b/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example index 472d0785bb2..8db2b44b8a7 100644 --- a/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example +++ b/Tools/machines/lawrencium-lbnl/lawrencium_warpx.profile.example @@ -18,8 +18,8 @@ module load lapack/3.8.0-gcc export CMAKE_PREFIX_PATH=$HOME/sw/v100/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=$HOME/sw/v100/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/v100/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/v100/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/v100/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/v100/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH export PATH=$HOME/sw/v100/adios2-2.8.3/bin:$PATH diff --git a/Tools/machines/leonardo-cineca/install_gpu_dependencies.sh b/Tools/machines/leonardo-cineca/install_gpu_dependencies.sh index bbaf0ab8464..4d89e30cd29 100644 --- a/Tools/machines/leonardo-cineca/install_gpu_dependencies.sh +++ b/Tools/machines/leonardo-cineca/install_gpu_dependencies.sh @@ -48,14 +48,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-gpu-build -CXX=$(which g++) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which g++) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build $HOME/src/blaspp-gpu-build --target install --parallel 16 rm -rf $HOME/src/blaspp-gpu-build @@ -65,14 +64,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build $HOME/src/lapackpp-gpu-build --target install --parallel 16 rm -rf $HOME/src/lapackpp-gpu-build diff --git a/Tools/machines/leonardo-cineca/leonardo_gpu_warpx.profile.example b/Tools/machines/leonardo-cineca/leonardo_gpu_warpx.profile.example index cffe565f9d7..dd8e79ffb37 100644 --- a/Tools/machines/leonardo-cineca/leonardo_gpu_warpx.profile.example +++ b/Tools/machines/leonardo-cineca/leonardo_gpu_warpx.profile.example @@ -16,13 +16,13 @@ module load boost/1.80.0--openmpi--4.1.4--gcc--11.3.0 module load openblas/0.3.21--gcc--11.3.0 export CMAKE_PREFIX_PATH=/leonardo/prod/spack/03/install/0.19/linux-rhel8-icelake/gcc-11.3.0/c-blosc-1.21.1-aifmix6v5lwxgt7rigwoebalrgbcnv26:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=$HOME/sw/adios2-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=$HOME/sw/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=$HOME/sw/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=/leonardo/prod/spack/03/install/0.19/linux-rhel8-icelake/gcc-11.3.0/c-blosc-1.21.1-aifmix6v5lwxgt7rigwoebalrgbcnv26/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=$HOME/sw/adios2-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=$HOME/sw/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=$HOME/sw/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=$HOME/sw/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=$HOME/sw/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH export PATH=$HOME/sw/adios2-master/bin:$PATH diff --git a/Tools/machines/lumi-csc/install_dependencies.sh b/Tools/machines/lumi-csc/install_dependencies.sh index 814e134bad4..2fd31b79bce 100755 --- a/Tools/machines/lumi-csc/install_dependencies.sh +++ b/Tools/machines/lumi-csc/install_dependencies.sh @@ -43,11 +43,10 @@ if [ -d ${SRC_DIR}/blaspp ] then cd ${SRC_DIR}/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp fi rm -rf ${build_dir}/blaspp-lumi-gpu-build CXX=$(which CC) \ @@ -56,7 +55,7 @@ cmake -S ${SRC_DIR}/blaspp \ -Duse_openmp=OFF \ -Dgpu_backend=hip \ -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-lumi-gpu-build --target install --parallel 16 rm -rf ${build_dir}/blaspp-lumi-gpu-build @@ -65,11 +64,10 @@ if [ -d ${SRC_DIR}/lapackpp ] then cd ${SRC_DIR}/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp fi rm -rf ${build_dir}/lapackpp-lumi-gpu-build CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" \ @@ -78,7 +76,7 @@ cmake -S ${SRC_DIR}/lapackpp \ -DCMAKE_CXX_STANDARD=17 \ -Dbuild_tests=OFF \ -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ - -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build ${build_dir}/lapackpp-lumi-gpu-build --target install --parallel 16 rm -rf ${build_dir}/lapackpp-lumi-gpu-build diff --git a/Tools/machines/lumi-csc/lumi_warpx.profile.example b/Tools/machines/lumi-csc/lumi_warpx.profile.example index 33aff1946c2..13fb6b1d81e 100644 --- a/Tools/machines/lumi-csc/lumi_warpx.profile.example +++ b/Tools/machines/lumi-csc/lumi_warpx.profile.example @@ -11,10 +11,10 @@ module load nano # optional: for PSATD in RZ geometry support SW_DIR="${HOME}/sw/lumi/gpu" -export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support module load Boost/1.82.0-cpeCray-23.09 diff --git a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh index f65e43891d0..437300b8303 100755 --- a/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_cpu_dependencies.sh @@ -82,14 +82,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-pm-cpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-cpu-build -Duse_openmp=ON -Dgpu_backend=OFF -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-cpu-build -Duse_openmp=ON -Dgpu_backend=OFF -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-pm-cpu-build --target install --parallel 16 rm -rf ${build_dir}/blaspp-pm-cpu-build @@ -98,17 +97,54 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-pm-cpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-cpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-cpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build ${build_dir}/lapackpp-pm-cpu-build --target install --parallel 16 rm -rf ${build_dir}/lapackpp-pm-cpu-build +# heFFTe +if [ -d $HOME/src/heffte ] +then + cd $HOME/src/heffte + git fetch --prune + git checkout v2.4.0 + cd - +else + git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${HOME}/src/heffte +fi +rm -rf ${HOME}/src/heffte-pm-cpu-build +cmake \ + -S ${HOME}/src/heffte \ + -B ${build_dir}/heffte-pm-cpu-build \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/heffte-2.4.0 \ + -DHeffte_DISABLE_GPU_AWARE_MPI=ON \ + -DHeffte_ENABLE_AVX=ON \ + -DHeffte_ENABLE_AVX512=OFF \ + -DHeffte_ENABLE_FFTW=ON \ + -DHeffte_ENABLE_CUDA=OFF \ + -DHeffte_ENABLE_ROCM=OFF \ + -DHeffte_ENABLE_ONEAPI=OFF \ + -DHeffte_ENABLE_MKL=OFF \ + -DHeffte_ENABLE_DOXYGEN=OFF \ + -DHeffte_SEQUENTIAL_TESTING=OFF \ + -DHeffte_ENABLE_TESTING=OFF \ + -DHeffte_ENABLE_TRACING=OFF \ + -DHeffte_ENABLE_PYTHON=OFF \ + -DHeffte_ENABLE_FORTRAN=OFF \ + -DHeffte_ENABLE_SWIG=OFF \ + -DHeffte_ENABLE_MAGMA=OFF +cmake --build ${build_dir}/heffte-pm-cpu-build --target install --parallel 16 +rm -rf ${build_dir}/heffte-pm-cpu-build + # Python ###################################################################### # diff --git a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh index 9ac5800c6ce..ac95ad9f3a0 100755 --- a/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh +++ b/Tools/machines/perlmutter-nersc/install_gpu_dependencies.sh @@ -82,14 +82,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-pm-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which CC) cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-pm-gpu-build --target install --parallel 16 rm -rf ${build_dir}/blaspp-pm-gpu-build @@ -98,17 +97,54 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-pm-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build ${build_dir}/lapackpp-pm-gpu-build --target install --parallel 16 rm -rf ${build_dir}/lapackpp-pm-gpu-build +# heFFTe +if [ -d $HOME/src/heffte ] +then + cd $HOME/src/heffte + git fetch --prune + git checkout v2.4.0 + cd - +else + git clone -b v2.4.0 https://github.com/icl-utk-edu/heffte.git ${HOME}/src/heffte +fi +rm -rf ${HOME}/src/heffte-pm-gpu-build +cmake \ + -S ${HOME}/src/heffte \ + -B ${build_dir}/heffte-pm-gpu-build \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/heffte-2.4.0 \ + -DHeffte_DISABLE_GPU_AWARE_MPI=OFF \ + -DHeffte_ENABLE_AVX=OFF \ + -DHeffte_ENABLE_AVX512=OFF \ + -DHeffte_ENABLE_FFTW=OFF \ + -DHeffte_ENABLE_CUDA=ON \ + -DHeffte_ENABLE_ROCM=OFF \ + -DHeffte_ENABLE_ONEAPI=OFF \ + -DHeffte_ENABLE_MKL=OFF \ + -DHeffte_ENABLE_DOXYGEN=OFF \ + -DHeffte_SEQUENTIAL_TESTING=OFF \ + -DHeffte_ENABLE_TESTING=OFF \ + -DHeffte_ENABLE_TRACING=OFF \ + -DHeffte_ENABLE_PYTHON=OFF \ + -DHeffte_ENABLE_FORTRAN=OFF \ + -DHeffte_ENABLE_SWIG=OFF \ + -DHeffte_ENABLE_MAGMA=OFF +cmake --build ${build_dir}/heffte-pm-gpu-build --target install --parallel 16 +rm -rf ${build_dir}/heffte-pm-gpu-build + # Python ###################################################################### # diff --git a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example index 1b0ac3182d5..94d598abf5b 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_cpu_warpx.profile.example @@ -17,13 +17,15 @@ export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.05/default/ module load cray-hdf5-parallel/1.12.2.9 export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/heffte-2.4.0:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/heffte-2.4.0/lib64:$LD_LIBRARY_PATH export PATH=${CFS}/${proj}/${USER}/sw/perlmutter/cpu/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example index 759df0b923a..da1d55964d1 100644 --- a/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example +++ b/Tools/machines/perlmutter-nersc/perlmutter_gpu_warpx.profile.example @@ -21,13 +21,15 @@ export BOOST_ROOT=/global/common/software/spackecp/perlmutter/e4s-23.05/default/ module load cray-hdf5-parallel/1.12.2.9 export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/heffte-2.4.0:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/heffte-2.4.0/lib64:$LD_LIBRARY_PATH export PATH=${CFS}/${proj%_g}/${USER}/sw/perlmutter/gpu/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/pitzer-osc/install_cpu_dependencies.sh b/Tools/machines/pitzer-osc/install_cpu_dependencies.sh new file mode 100644 index 00000000000..f0b6ce4b950 --- /dev/null +++ b/Tools/machines/pitzer-osc/install_cpu_dependencies.sh @@ -0,0 +1,156 @@ +#!/bin/bash +# +# Copyright 2024 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Zhongwei Wang +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + +# Check: ###################################################################### +# +# Was pitzer_cpu_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your pitzer_cpu_warpx.profile file! Please edit its line 2 to continue!" + exit 1 +fi + +# Remove old dependencies ##################################################### +# +rm -rf ${SW_DIR} +mkdir -p ${SW_DIR} + +# remove common user mistakes in python, located in .local instead of a venv +python3 -m pip uninstall -qq -y pywarpx +python3 -m pip uninstall -qq -y warpx +python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true + +# General extra dependencies ################################################## +# +SRC_DIR="${HOME}/src" +build_dir=$(mktemp -d) + +# boost (for QED table generation support) +cd ${SRC_DIR} +wget https://archives.boost.io/release/1.82.0/source/boost_1_82_0.tar.gz +tar -xzvf boost_1_82_0.tar.gz +rm -rf boost_1_82_0.tar.gz +cd - + +cd ${SRC_DIR}/boost_1_82_0 +./bootstrap.sh --prefix=$SW_DIR/boost-1.82.0 +./b2 install +cd - + +# BLAS++ (for PSATD+RZ) +if [ -d ${SRC_DIR}/blaspp ]; then + cd ${SRC_DIR}/blaspp + git fetch --prune + git checkout v2024.05.31 + cd - +else + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp +fi +rm -rf ${build_dir}/blaspp-pitzer-cpu-build +CXX=$(which CC) cmake -S ${SRC_DIR}/blaspp \ + -B ${build_dir}/blaspp-pitzer-cpu-build \ + -Duse_openmp=ON \ + -Dgpu_backend=OFF \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 +cmake --build ${build_dir}/blaspp-pitzer-cpu-build --target install --parallel 16 +rm -rf ${build_dir}/blaspp-pitzer-cpu-build + +# LAPACK++ (for PSATD+RZ) +if [ -d ${SRC_DIR}/lapackpp ]; then + cd ${SRC_DIR}/lapackpp + git fetch --prune + git checkout v2024.05.31 + cd - +else + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp +fi +rm -rf ${build_dir}/lapackpp-pitzer-cpu-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp \ + -B ${build_dir}/lapackpp-pitzer-cpu-build \ + -DCMAKE_CXX_STANDARD=17 \ + -Dbuild_tests=OFF \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 +cmake --build ${build_dir}/lapackpp-pitzer-cpu-build --target install --parallel 16 +rm -rf ${build_dir}/lapackpp-pitzer-cpu-build + +# c-blosc (I/O compression, for openPMD) +if [ -d ${SRC_DIR}/c-blosc ]; then + cd ${SRC_DIR}/c-blosc + git fetch --prune + git checkout v1.21.6 + cd - +else + git clone -b v1.21.6 https://github.com/Blosc/c-blosc.git ${SRC_DIR}/c-blosc +fi +rm -rf ${build_dir}/c-blosc-pitzer-build +cmake -S ${SRC_DIR}/c-blosc \ + -B ${build_dir}/c-blosc-pitzer-build \ + -DBUILD_TESTS=OFF \ + -DBUILD_BENCHMARKS=OFF \ + -DDEACTIVATE_AVX2=OFF \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.6 +cmake --build ${build_dir}/c-blosc-pitzer-build --target install --parallel 16 +rm -rf ${build_dir}/c-blosc-pitzer-build + +# ADIOS2 (for openPMD) +if [ -d ${SRC_DIR}/adios2 ]; then + cd ${SRC_DIR}/adios2 + git fetch --prune + git checkout v2.10.1 + cd - +else + git clone -b v2.10.1 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2 +fi +rm -rf ${build_dir}/adios2-pitzer-build +cmake -S ${SRC_DIR}/adios2 \ + -B ${build_dir}/adios2-pitzer-build \ + -DBUILD_TESTING=OFF \ + -DADIOS2_BUILD_EXAMPLES=OFF \ + -DADIOS2_USE_Blosc=ON \ + -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_Python=OFF \ + -DADIOS2_USE_SST=OFF \ + -DADIOS2_USE_ZeroMQ=OFF \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.10.1 +cmake --build ${build_dir}/adios2-pitzer-build --target install -j 16 +rm -rf ${build_dir}/adios2-pitzer-build + +rm -rf ${build_dir} + +# Python ###################################################################### +# +python3 -m pip install --upgrade --user virtualenv +rm -rf ${SW_DIR}/venvs/${VENV_NAME} +python3 -m venv ${SW_DIR}/venvs/${VENV_NAME} +source ${SW_DIR}/venvs/${VENV_NAME}/bin/activate +python3 -m pip install --upgrade pip +python3 -m pip cache purge +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging +python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools +python3 -m pip install --upgrade cython +python3 -m pip install --upgrade numpy +python3 -m pip install --upgrade pandas +python3 -m pip install --upgrade scipy +python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +python3 -m pip install --upgrade openpmd-api +python3 -m pip install --upgrade matplotlib +python3 -m pip install --upgrade yt + +# install or update WarpX dependencies such as picmistandard +python3 -m pip install --upgrade -r ${SRC_DIR}/warpx/requirements.txt + +# ML dependencies +python3 -m pip install --upgrade torch diff --git a/Tools/machines/pitzer-osc/install_v100_dependencies.sh b/Tools/machines/pitzer-osc/install_v100_dependencies.sh new file mode 100644 index 00000000000..5601b4d76c9 --- /dev/null +++ b/Tools/machines/pitzer-osc/install_v100_dependencies.sh @@ -0,0 +1,156 @@ +#!/bin/bash +# +# Copyright 2024 The WarpX Community +# +# This file is part of WarpX. +# +# Author: Zhongwei Wang +# License: BSD-3-Clause-LBNL + +# Exit on first error encountered ############################################# +# +set -eu -o pipefail + +# Check: ###################################################################### +# +# Was pitzer_v100_warpx.profile sourced and configured correctly? +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your pitzer_v100_warpx.profile file! Please edit its line 2 to continue!" + exit 1 +fi + +# Remove old dependencies ##################################################### +# +rm -rf ${SW_DIR} +mkdir -p ${SW_DIR} + +# remove common user mistakes in python, located in .local instead of a venv +python3 -m pip uninstall -qq -y pywarpx +python3 -m pip uninstall -qq -y warpx +python3 -m pip uninstall -qqq -y mpi4py 2>/dev/null || true + +# General extra dependencies ################################################## +# +SRC_DIR="${HOME}/src" +build_dir=$(mktemp -d) + +# boost (for QED table generation support) +cd ${SRC_DIR} +wget https://archives.boost.io/release/1.82.0/source/boost_1_82_0.tar.gz +tar -xzvf boost_1_82_0.tar.gz +rm -rf boost_1_82_0.tar.gz +cd - + +cd ${SRC_DIR}/boost_1_82_0 +./bootstrap.sh --prefix=${SW_DIR}/boost-1.82.0 +./b2 install +cd - + +# BLAS++ (for PSATD+RZ) +if [ -d ${SRC_DIR}/blaspp ]; then + cd ${SRC_DIR}/blaspp + git fetch + git checkout v2024.05.31 + cd - +else + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${SRC_DIR}/blaspp +fi +rm -rf ${build_dir}/blaspp-pitzer-v100-build +CXX=$(which CC) cmake -S ${SRC_DIR}/blaspp \ + -B ${build_dir}/blaspp-pitzer-v100-build \ + -Duse_openmp=ON \ + -Dgpu_backend=cuda \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 +cmake --build ${build_dir}/blaspp-pitzer-v100-build --target install --parallel 16 +rm -rf ${build_dir}/blaspp-pitzer-v100-build + +# LAPACK++ (for PSATD+RZ) +if [ -d ${SRC_DIR}/lapackpp ]; then + cd ${SRC_DIR}/lapackpp + git fetch + git checkout v2024.05.31 + cd - +else + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${SRC_DIR}/lapackpp +fi +rm -rf ${build_dir}/lapackpp-pitzer-v100-build +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${SRC_DIR}/lapackpp \ + -B ${build_dir}/lapackpp-pitzer-v100-build \ + -DCMAKE_CXX_STANDARD=17 \ + -Dbuild_tests=OFF \ + -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 +cmake --build ${build_dir}/lapackpp-pitzer-v100-build --target install --parallel 16 +rm -rf ${build_dir}/lapackpp-pitzer-v100-build + +# c-blosc (I/O compression, for openPMD) +if [ -d ${SRC_DIR}/c-blosc ]; then + cd ${SRC_DIR}/c-blosc + git fetch --prune + git checkout v1.21.6 + cd - +else + git clone -b v1.21.6 https://github.com/Blosc/c-blosc.git ${SRC_DIR}/c-blosc +fi +rm -rf ${build_dir}/c-blosc-pitzer-build +cmake -S ${SRC_DIR}/c-blosc \ + -B ${build_dir}/c-blosc-pitzer-build \ + -DBUILD_TESTS=OFF \ + -DBUILD_BENCHMARKS=OFF \ + -DDEACTIVATE_AVX2=OFF \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/c-blosc-1.21.6 +cmake --build ${build_dir}/c-blosc-pitzer-build --target install --parallel 16 +rm -rf ${build_dir}/c-blosc-pitzer-build + +# ADIOS2 (for openPMD) +if [ -d ${SRC_DIR}/adios2 ]; then + cd ${SRC_DIR}/adios2 + git fetch --prune + git checkout v2.10.1 + cd - +else + git clone -b v2.10.1 https://github.com/ornladios/ADIOS2.git ${SRC_DIR}/adios2 +fi +rm -rf ${build_dir}/adios2-pitzer-build +cmake -S ${SRC_DIR}/adios2 \ + -B ${build_dir}/adios2-pitzer-build \ + -DBUILD_TESTING=OFF \ + -DADIOS2_BUILD_EXAMPLES=OFF \ + -DADIOS2_USE_Blosc=ON \ + -DADIOS2_USE_Fortran=OFF \ + -DADIOS2_USE_Python=OFF \ + -DADIOS2_USE_SST=OFF \ + -DADIOS2_USE_ZeroMQ=OFF \ + -DCMAKE_INSTALL_PREFIX=${SW_DIR}/adios2-2.10.1 +cmake --build ${build_dir}/adios2-pitzer-build --target install -j 16 +rm -rf ${build_dir}/adios2-pitzer-build + +rm -rf ${build_dir} + +# Python ###################################################################### +# +python3 -m pip install --upgrade --user virtualenv +rm -rf ${SW_DIR}/venvs/${VENV_NAME} +python3 -m venv ${SW_DIR}/venvs/${VENV_NAME} +source ${SW_DIR}/venvs/${VENV_NAME}/bin/activate +python3 -m pip install --upgrade pip +python3 -m pip cache purge +python3 -m pip install --upgrade build +python3 -m pip install --upgrade packaging +python3 -m pip install --upgrade wheel +python3 -m pip install --upgrade setuptools +python3 -m pip install --upgrade cython +python3 -m pip install --upgrade numpy +python3 -m pip install --upgrade pandas +python3 -m pip install --upgrade scipy +python3 -m pip install --upgrade mpi4py --no-cache-dir --no-build-isolation --no-binary mpi4py +python3 -m pip install --upgrade openpmd-api +python3 -m pip install --upgrade matplotlib +python3 -m pip install --upgrade yt + +# install or update WarpX dependencies such as picmistandard +python3 -m pip install --upgrade -r ${SRC_DIR}/warpx/requirements.txt + +# ML dependencies +python3 -m pip install --upgrade torch diff --git a/Tools/machines/pitzer-osc/pitzer_cpu.sbatch b/Tools/machines/pitzer-osc/pitzer_cpu.sbatch new file mode 100644 index 00000000000..8097b3842f9 --- /dev/null +++ b/Tools/machines/pitzer-osc/pitzer_cpu.sbatch @@ -0,0 +1,19 @@ +#!/bin/bash +#SBATCH --time=0:20:00 +#SBATCH --nodes=1 --ntasks-per-node=6 +#SBATCH --cpus-per-task=8 +#SBATCH --job-name= +#SBATCH --account= +#SBATCH --output=./logs/%x_%j.out +#SBATCH --error=./logs/%x_%j.err + +# Pitzer cluster has 224 CPU nodes equipped with dual Intel Xeon 6148 (40 cores per node) and 340 CPU nodes with dual Intel Xeon 8268 (48 cores per node). https://www.osc.edu/resources/technical_support/supercomputers/pitzer + +source ${HOME}/pitzer_cpu_warpx.profile +export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} + +# executable & inputs file or python interpreter & PICMI script here +EXE=${HOME}/src/warpx/build/bin/warpx.2d +INPUTS=inputs + +srun --cpu-bind=cores ${EXE} ${INPUTS} >./logs/${SLURM_JOB_NAME}_${SLURM_JOBID}.log 2>&1 diff --git a/Tools/machines/pitzer-osc/pitzer_cpu_warpx.profile.example b/Tools/machines/pitzer-osc/pitzer_cpu_warpx.profile.example new file mode 100644 index 00000000000..d99c16b6cb6 --- /dev/null +++ b/Tools/machines/pitzer-osc/pitzer_cpu_warpx.profile.example @@ -0,0 +1,54 @@ +# please set your project account +export proj="" # change me! + +# remembers the location of this script +export MY_CPU_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your $MY_CPU_PROFILE file! Please edit its line 2 to continue!" + return +fi + +export SW_DIR="${HOME}/sw/osc/pitzer/cpu" + +module purge +module load cmake/3.25.2 +module load gnu/12.3.0 +module load openmpi/4.1.5-hpcx + +# optional: for python binding support +module load miniconda3/24.1.2-py310 +export VENV_NAME="warpx-pitzer-cpu" +if [ -d "${SW_DIR}/venvs/${VENV_NAME}" ]; then + source ${SW_DIR}/venvs/${VENV_NAME}/bin/activate +fi + +# an alias to request an interactive batch node for one hour +# for parallel execution, start on the batch node: srun +alias getNode="salloc -N 1 --ntasks-per-node=2 --cpus-per-task=20 -t 1:00:00 -A $proj" +# an alias to run a command on a batch node for up to 30min +# usage: runNode +alias runNode="srun -N 1 --ntasks-per-node=2 --cpus-per-task=20 -t 1:00:00 -A $proj" + +# optional: for PSATD in RZ geometry support +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH + +# optional: for QED lookup table generation support +# use self-installed boost +export CMAKE_PREFIX_PATH=${SW_DIR}/boost-1.82.0:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/boost-1.82.0/lib:$LD_LIBRARY_PATH + +# optional: for openPMD support (hdf5 and adios2) +module load hdf5/1.12.2 +export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.6:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.10.1:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-1.21.6/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.10.1/lib64:$LD_LIBRARY_PATH +export PATH=${SW_DIR}/adios2-2.10.1/bin:${PATH} + +# compiler environment hints +export CC=$(which gcc) +export CXX=$(which g++) +export FC=$(which gfortran) diff --git a/Tools/machines/pitzer-osc/pitzer_v100.sbatch b/Tools/machines/pitzer-osc/pitzer_v100.sbatch new file mode 100644 index 00000000000..d34868da16c --- /dev/null +++ b/Tools/machines/pitzer-osc/pitzer_v100.sbatch @@ -0,0 +1,26 @@ +#!/bin/bash +#SBATCH --time=0:20:00 +#SBATCH --nodes=1 --ntasks-per-node=2 +#SBATCH --cpus-per-task=24 +#SBATCH --gpus-per-task=1 +#SBATCH --gpu-bind=closest +#SBATCH --job-name= +#SBATCH --account= +#SBATCH --output=./logs/%x_%j.out +#SBATCH --error=./logs/%x_%j.err + +# Pitzer cluster has 32 GPU nodes with dual Intel Xeon 6148 and dual V100 (16GB) GPUs and 42 nodes with dual Intel Xeon 8268 and dual V100 (32GB) GPUs. https://www.osc.edu/resources/technical_support/supercomputers/pitzer + +source ${HOME}/pitzer_v100_warpx.profile +export OMP_NUM_THREADS=${SLURM_CPUS_PER_TASK} + +echo "GPU Information:" +nvidia-smi + +GPU_AWARE_MPI="amrex.use_gpu_aware_mpi=1" + +# executable & inputs file or python interpreter & PICMI script here +EXE=${HOME}/src/warpx/build_v100/bin/warpx.2d +INPUTS=inputs + +srun --cpu-bind=cores ${EXE} ${INPUTS} ${GPU_AWARE_MPI} >./logs/${SLURM_JOB_NAME}_${SLURM_JOBID}.log 2>&1 diff --git a/Tools/machines/pitzer-osc/pitzer_v100_warpx.profile.example b/Tools/machines/pitzer-osc/pitzer_v100_warpx.profile.example new file mode 100644 index 00000000000..061794f5f68 --- /dev/null +++ b/Tools/machines/pitzer-osc/pitzer_v100_warpx.profile.example @@ -0,0 +1,64 @@ +# please set your project account +export proj="" # change me! + +# remembers the location of this script +export MY_V100_PROFILE=$(cd $(dirname $BASH_SOURCE) && pwd)"/"$(basename $BASH_SOURCE) +if [ -z ${proj-} ]; then + echo "WARNING: The 'proj' variable is not yet set in your $MY_V100_PROFILE file! Please edit its line 2 to continue!" + return +fi + +export SW_DIR="${HOME}/sw/osc/pitzer/v100" + +module purge +module load cmake/3.25.2 +module load intel/19.0.5 +module load cuda/11.8.0 +module load openmpi-cuda/4.1.5-hpcx +module load gcc-compatibility/11.2.0 + +# optional: for python binding support +module load miniconda3/24.1.2-py310 +export VENV_NAME="warpx-pitzer-v100" +if [ -d "${SW_DIR}/venvs/${VENV_NAME}" ]; then + source ${SW_DIR}/venvs/${VENV_NAME}/bin/activate +fi + +# an alias to request an interactive batch node for one hour +# for parallel execution, start on the batch node: srun +alias getNode="salloc -N 1 --ntasks-per-node=2 --cpus-per-task=20 --gpus-per-task=v100:1 -t 1:00:00 -A $proj" +# an alias to run a command on a batch node for up to 30min +# usage: runNode +alias runNode="srun -N 1 --ntasks-per-node=2 --cpus-per-task=20 --gpus-per-task=v100:1 -t 1:00:00 -A $proj" + +# optional: for PSATD in RZ geometry support +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH + +# optional: for QED lookup table generation support +# use self-installed boost +export CMAKE_PREFIX_PATH=${SW_DIR}/boost-1.82.0:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/boost-1.82.0/lib:$LD_LIBRARY_PATH + +# optional: for openPMD support (hdf5 and adios2) +# use self-installed hdf5 +module load hdf5/1.12.0 + +export CMAKE_PREFIX_PATH=${SW_DIR}/c-blosc-1.21.6:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.10.1:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/c-blosc-1.21.6/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/adios2-2.10.1/lib64:$LD_LIBRARY_PATH +export PATH=${SW_DIR}/adios2-2.10.1/bin:${PATH} + +# avoid relocation truncation error which result from large executable size +export CUDAFLAGS="--host-linker-script=use-lcs" # https://github.com/ECP-WarpX/WarpX/pull/3673 +export AMREX_CUDA_ARCH=7.0 # 7.0: V100, 8.0: V100, 9.0: H100 https://github.com/ECP-WarpX/WarpX/issues/3214 + +# compiler environment hints +export CC=$(which gcc) +export CXX=$(which g++) +export FC=$(which gfortran) +export CUDACXX=$(which nvcc) +export CUDAHOSTCXX=${CXX} diff --git a/Tools/machines/polaris-alcf/install_gpu_dependencies.sh b/Tools/machines/polaris-alcf/install_gpu_dependencies.sh index dbc66b2ffff..18f94e6fd3b 100755 --- a/Tools/machines/polaris-alcf/install_gpu_dependencies.sh +++ b/Tools/machines/polaris-alcf/install_gpu_dependencies.sh @@ -65,14 +65,13 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi rm -rf $HOME/src/blaspp-pm-gpu-build -CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +CXX=$(which CC) cmake -S $HOME/src/blaspp -B $HOME/src/blaspp-pm-gpu-build -Duse_openmp=OFF -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build $HOME/src/blaspp-pm-gpu-build --target install --parallel 16 rm -rf $HOME/src/blaspp-pm-gpu-build @@ -81,14 +80,13 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi rm -rf $HOME/src/lapackpp-pm-gpu-build -CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXX=$(which CC) CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S $HOME/src/lapackpp -B $HOME/src/lapackpp-pm-gpu-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build $HOME/src/lapackpp-pm-gpu-build --target install --parallel 16 rm -rf $HOME/src/lapackpp-pm-gpu-build diff --git a/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example b/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example index d7635188141..333434c1b97 100644 --- a/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example +++ b/Tools/machines/polaris-alcf/polaris_gpu_warpx.profile.example @@ -19,13 +19,13 @@ module load cmake/3.23.2 module load cray-hdf5-parallel/1.12.2.3 export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/c-blosc-1.21.1:$CMAKE_PREFIX_PATH export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/adios2-2.8.3:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/lapackpp-master:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=/home/${USER}/sw/polaris/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/c-blosc-1.21.1/lib64:$LD_LIBRARY_PATH export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/adios2-2.8.3/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/home/${USER}/sw/polaris/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH export PATH=/home/${USER}/sw/polaris/gpu/adios2-2.8.3/bin:${PATH} diff --git a/Tools/machines/quartz-llnl/install_dependencies.sh b/Tools/machines/quartz-llnl/install_dependencies.sh index 2920b00663b..cfb01769384 100755 --- a/Tools/machines/quartz-llnl/install_dependencies.sh +++ b/Tools/machines/quartz-llnl/install_dependencies.sh @@ -67,13 +67,12 @@ if [ -d ${HOME}/src/blaspp ] then cd ${HOME}/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git ${HOME}/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git ${HOME}/src/blaspp fi -cmake -S ${HOME}/src/blaspp -B ${build_dir}/blaspp-quartz-build -Duse_openmp=ON -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake -S ${HOME}/src/blaspp -B ${build_dir}/blaspp-quartz-build -Duse_openmp=ON -Duse_cmake_find_blas=ON -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-quartz-build --target install --parallel 6 # LAPACK++ (for PSATD+RZ) @@ -81,13 +80,12 @@ if [ -d ${HOME}/src/lapackpp ] then cd ${HOME}/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git ${HOME}/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git ${HOME}/src/lapackpp fi -CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${HOME}/src/lapackpp -B ${build_dir}/lapackpp-quartz-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +CXXFLAGS="-DLAPACK_FORTRAN_ADD_" cmake -S ${HOME}/src/lapackpp -B ${build_dir}/lapackpp-quartz-build -Duse_cmake_find_lapack=ON -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build ${build_dir}/lapackpp-quartz-build --target install --parallel 6 diff --git a/Tools/machines/quartz-llnl/quartz_warpx.profile.example b/Tools/machines/quartz-llnl/quartz_warpx.profile.example index a3646bfd557..f296a0738ff 100644 --- a/Tools/machines/quartz-llnl/quartz_warpx.profile.example +++ b/Tools/machines/quartz-llnl/quartz_warpx.profile.example @@ -21,10 +21,10 @@ export CMAKE_PREFIX_PATH=${SW_DIR}/adios2-2.8.3:$CMAKE_PREFIX_PATH export PATH=${SW_DIR}/adios2-2.8.3/bin:${PATH} # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=${SW_DIR}/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=${SW_DIR}/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=${SW_DIR}/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=${SW_DIR}/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for Python bindings module load python/3.9.12 diff --git a/Tools/machines/summit-olcf/install_gpu_dependencies.sh b/Tools/machines/summit-olcf/install_gpu_dependencies.sh index 433b7ab0580..042e34538d0 100755 --- a/Tools/machines/summit-olcf/install_gpu_dependencies.sh +++ b/Tools/machines/summit-olcf/install_gpu_dependencies.sh @@ -63,13 +63,12 @@ if [ -d $HOME/src/blaspp ] then cd $HOME/src/blaspp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/blaspp.git $HOME/src/blaspp fi -cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-summit-build -Duse_openmp=ON -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-master +cmake -S $HOME/src/blaspp -B ${build_dir}/blaspp-summit-build -Duse_openmp=ON -Dgpu_backend=cuda -DCMAKE_CXX_STANDARD=17 -DCMAKE_INSTALL_PREFIX=${SW_DIR}/blaspp-2024.05.31 cmake --build ${build_dir}/blaspp-summit-build --target install --parallel 10 # LAPACK++ (for PSATD+RZ) @@ -77,13 +76,12 @@ if [ -d $HOME/src/lapackpp ] then cd $HOME/src/lapackpp git fetch --prune - git checkout master - git pull + git checkout v2024.05.31 cd - else - git clone https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp + git clone -b v2024.05.31 https://github.com/icl-utk-edu/lapackpp.git $HOME/src/lapackpp fi -cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-summit-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-master +cmake -S $HOME/src/lapackpp -B ${build_dir}/lapackpp-summit-build -DCMAKE_CXX_STANDARD=17 -Dbuild_tests=OFF -DCMAKE_INSTALL_RPATH_USE_LINK_PATH=ON -DCMAKE_INSTALL_PREFIX=${SW_DIR}/lapackpp-2024.05.31 cmake --build ${build_dir}/lapackpp-summit-build --target install --parallel 10 # remove build temporary directory diff --git a/Tools/machines/summit-olcf/summit_warpx.profile.example b/Tools/machines/summit-olcf/summit_warpx.profile.example index e41521b5815..d70ee9eb42a 100644 --- a/Tools/machines/summit-olcf/summit_warpx.profile.example +++ b/Tools/machines/summit-olcf/summit_warpx.profile.example @@ -18,10 +18,10 @@ module load ccache module load ninja # optional: for PSATD in RZ geometry support -export CMAKE_PREFIX_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/blaspp-master:$CMAKE_PREFIX_PATH -export CMAKE_PREFIX_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/lapackpp-master:$CMAKE_PREFIX_PATH -export LD_LIBRARY_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/blaspp-master/lib64:$LD_LIBRARY_PATH -export LD_LIBRARY_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/lapackpp-master/lib64:$LD_LIBRARY_PATH +export CMAKE_PREFIX_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/blaspp-2024.05.31:$CMAKE_PREFIX_PATH +export CMAKE_PREFIX_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/lapackpp-2024.05.31:$CMAKE_PREFIX_PATH +export LD_LIBRARY_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/blaspp-2024.05.31/lib64:$LD_LIBRARY_PATH +export LD_LIBRARY_PATH=/ccs/proj/$proj/${USER}/sw/summit/gpu/lapackpp-2024.05.31/lib64:$LD_LIBRARY_PATH # optional: for QED lookup table generation support module load boost/1.76.0 diff --git a/WarpXConfig.cmake b/WarpXConfig.cmake index 5a2c0916432..8536d08f67a 100644 --- a/WarpXConfig.cmake +++ b/WarpXConfig.cmake @@ -35,11 +35,12 @@ if(WarpX_OPENPMD) endif() set(WarpX_OPENPMD_FOUND ${WarpX_OPENPMD}) -WarpX_ASCENT "Ascent in situ diagnostics" OFF) -WarpX_PSATD "spectral solver support" OFF) -WarpX_SENSEI "SENSEI in situ diagnostics" OFF) -WarpX_QED "QED support (requires PICSAR)" ON) -WarpX_QED_TABLE_GEN "QED table generation (requires PICSAR and Boost)" OFF) +# TODO +#WarpX_ASCENT "Ascent in situ diagnostics" OFF) +#WarpX_FFT "FFT-based solvers" OFF) +#WarpX_SENSEI "SENSEI in situ diagnostics" OFF) +#WarpX_QED "QED support (requires PICSAR)" ON) +#WarpX_QED_TABLE_GEN "QED table generation (requires PICSAR and Boost)" OFF) @@ -50,7 +51,7 @@ endif() set(WarpX_ADIOS2_FOUND ${WarpX_HAVE_ADIOS2}) # define central WarpX::app WarpX::shared ... targets -ABLASTR here, too? or separate Config.cmake? +#ABLASTR here, too? or separate Config.cmake? include("${CMAKE_CURRENT_LIST_DIR}/WarpXTargets.cmake") # check if components are fulfilled and set WarpX__FOUND vars diff --git a/cmake/WarpXFunctions.cmake b/cmake/WarpXFunctions.cmake index 9b9feb64437..e05b4c3afc4 100644 --- a/cmake/WarpXFunctions.cmake +++ b/cmake/WarpXFunctions.cmake @@ -209,7 +209,7 @@ endmacro() # Enables interprocedural optimization for a list of targets # -function(enable_IPO all_targets_list) +function(warpx_enable_IPO all_targets_list) include(CheckIPOSupported) check_ipo_supported(RESULT is_IPO_available) if(is_IPO_available) @@ -305,7 +305,7 @@ function(set_warpx_binary_name D) set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".OPMD") endif() - if(WarpX_PSATD) + if(WarpX_FFT) set_property(TARGET ${tgt} APPEND_STRING PROPERTY OUTPUT_NAME ".PSATD") endif() @@ -452,7 +452,8 @@ function(warpx_print_summary) endif() message(" PARTICLE PRECISION: ${WarpX_PARTICLE_PRECISION}") message(" PRECISION: ${WarpX_PRECISION}") - message(" PSATD: ${WarpX_PSATD}") + message(" FFT Solvers: ${WarpX_FFT}") + message(" heFFTe: ${WarpX_HEFFTE}") message(" PYTHON: ${WarpX_PYTHON}") if(WarpX_PYTHON) message(" PYTHON IPO: ${WarpX_PYTHON_IPO}") diff --git a/cmake/dependencies/AMReX.cmake b/cmake/dependencies/AMReX.cmake index 349a4fd3f63..f1e5b3f62e2 100644 --- a/cmake/dependencies/AMReX.cmake +++ b/cmake/dependencies/AMReX.cmake @@ -250,7 +250,7 @@ macro(find_amrex) endif() set(COMPONENT_PRECISION ${WarpX_PRECISION} P${WarpX_PARTICLE_PRECISION}) - find_package(AMReX 24.05 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) + find_package(AMReX 24.07 CONFIG REQUIRED COMPONENTS ${COMPONENT_ASCENT} ${COMPONENT_DIMS} ${COMPONENT_EB} PARTICLES ${COMPONENT_PIC} ${COMPONENT_PRECISION} ${COMPONENT_SENSEI} LSOLVERS) # note: TINYP skipped because user-configured and optional # AMReX CMake helper scripts @@ -273,7 +273,7 @@ set(WarpX_amrex_src "" set(WarpX_amrex_repo "https://github.com/AMReX-Codes/amrex.git" CACHE STRING "Repository URI to pull and build AMReX from if(WarpX_amrex_internal)") -set(WarpX_amrex_branch "7ca419ebb90da60fefc01d8c1816846fff8638a5" +set(WarpX_amrex_branch "20e6f2eadf0c297517588ba38973ec7c7084fa31" CACHE STRING "Repository branch for WarpX_amrex_repo if(WarpX_amrex_internal)") diff --git a/cmake/dependencies/pyAMReX.cmake b/cmake/dependencies/pyAMReX.cmake index 79feddf3184..133747aaeae 100644 --- a/cmake/dependencies/pyAMReX.cmake +++ b/cmake/dependencies/pyAMReX.cmake @@ -64,7 +64,7 @@ function(find_pyamrex) endif() elseif(NOT WarpX_pyamrex_internal) # TODO: MPI control - find_package(pyAMReX 24.05 CONFIG REQUIRED) + find_package(pyAMReX 24.07 CONFIG REQUIRED) message(STATUS "pyAMReX: Found version '${pyAMReX_VERSION}'") endif() endfunction() @@ -79,7 +79,7 @@ option(WarpX_pyamrex_internal "Download & build pyAMReX" ON) set(WarpX_pyamrex_repo "https://github.com/AMReX-Codes/pyamrex.git" CACHE STRING "Repository URI to pull and build pyamrex from if(WarpX_pyamrex_internal)") -set(WarpX_pyamrex_branch "1b795c2c9741c8a63a362ceef8bd5e70cb242e92" +set(WarpX_pyamrex_branch "e007e730d48cb5fdbe1e10462d7d0a14e2bc8f8e" CACHE STRING "Repository branch for WarpX_pyamrex_repo if(WarpX_pyamrex_internal)") diff --git a/requirements.txt b/requirements.txt index 60f14e1282d..8e664ae3096 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,10 +1,10 @@ # basic dependencies -numpy~=1.15 +numpy>=1.15 periodictable~=1.5 # PICMI # note: don't forget to update the version in Docs/requirements.txt, too -picmistandard==0.28.0 +picmistandard==0.29.0 # for development against an unreleased PICMI version, use: #picmistandard @ git+https://github.com/picmi-standard/picmi.git#subdirectory=PICMI_Python diff --git a/run_test.sh b/run_test.sh index 099a59702a2..4efa86eb36c 100755 --- a/run_test.sh +++ b/run_test.sh @@ -65,10 +65,11 @@ python3 -m pip install --upgrade pip python3 -m pip install --upgrade build packaging setuptools wheel python3 -m pip install --upgrade cmake python3 -m pip install --upgrade -r warpx/Regression/requirements.txt +python3 -m pip cache purge # Clone AMReX and warpx-data git clone https://github.com/AMReX-Codes/amrex.git -cd amrex && git checkout --detach 7ca419ebb90da60fefc01d8c1816846fff8638a5 && cd - +cd amrex && git checkout --detach 20e6f2eadf0c297517588ba38973ec7c7084fa31 && cd - # warpx-data contains various required data sets git clone --depth 1 https://github.com/ECP-WarpX/warpx-data.git # openPMD-example-datasets contains various required data sets @@ -80,8 +81,6 @@ cd - # Clone the AMReX regression test utility git clone https://github.com/AMReX-Codes/regression_testing.git -# FIXME: https://github.com/AMReX-Codes/regression_testing/issues/136 -cd regression_testing && git checkout 93ddfb11456f47d6555c39388ba1a4ead61fbf4e && cd - # Prepare regression tests mkdir -p rt-WarpX/WarpX-benchmarks @@ -96,10 +95,10 @@ cd ../../regression_testing/ echo "cd $PWD" # run only tests specified in variable tests_arg (single test or multiple tests) if [[ ! -z "${tests_arg}" ]]; then - python3 regtest.py ../rt-WarpX/ci-tests.ini --no_update all "${tests_run}" + python3 regtest.py ../rt-WarpX/ci-tests.ini --skip_comparison --no_update all "${tests_run}" # run all tests (variables tests_arg and tests_run are empty) else - python3 regtest.py ../rt-WarpX/ci-tests.ini --no_update all + python3 regtest.py ../rt-WarpX/ci-tests.ini --skip_comparison --no_update all fi # clean up python virtual environment diff --git a/setup.py b/setup.py index 164f4f1395f..f05d5a71899 100644 --- a/setup.py +++ b/setup.py @@ -96,7 +96,8 @@ def build_extension(self, ext): '-DWarpX_OPENPMD:BOOL=' + WARPX_OPENPMD, '-DWarpX_PRECISION=' + WARPX_PRECISION, '-DWarpX_PARTICLE_PRECISION=' + WARPX_PARTICLE_PRECISION, - '-DWarpX_PSATD:BOOL=' + WARPX_PSATD, + '-DWarpX_FFT:BOOL=' + WARPX_FFT, + '-DWarpX_HEFFTE:BOOL=' + WARPX_HEFFTE, '-DWarpX_PYTHON:BOOL=ON', '-DWarpX_PYTHON_IPO:BOOL=' + WARPX_PYTHON_IPO, '-DWarpX_QED:BOOL=' + WARPX_QED, @@ -205,7 +206,8 @@ def build_extension(self, ext): WARPX_OPENPMD = env.pop('WARPX_OPENPMD', 'ON') WARPX_PRECISION = env.pop('WARPX_PRECISION', 'DOUBLE') WARPX_PARTICLE_PRECISION = env.pop('WARPX_PARTICLE_PRECISION', WARPX_PRECISION) -WARPX_PSATD = env.pop('WARPX_PSATD', 'OFF') +WARPX_FFT = env.pop('WARPX_FFT', 'OFF') +WARPX_HEFFTE = env.pop('WARPX_HEFFTE', 'OFF') WARPX_QED = env.pop('WARPX_QED', 'ON') WARPX_QED_TABLE_GEN = env.pop('WARPX_QED_TABLE_GEN', 'OFF') WARPX_DIMS = env.pop('WARPX_DIMS', '1;2;RZ;3') @@ -278,7 +280,7 @@ def build_extension(self, ext): setup( name='pywarpx', # note PEP-440 syntax: x.y.zaN but x.y.z.devN - version = '24.05', + version = '24.07', packages = ['pywarpx'], package_dir = {'pywarpx': 'Python/pywarpx'}, author='Jean-Luc Vay, David P. Grote, Maxence Thévenet, Rémi Lehe, Andrew Myers, Weiqun Zhang, Axel Huebl, et al.',