Use llama for particle frame and shared memory DataBox layout

include/picongpu/algorithms/Set.hpp

@@ -33,9 +33,9 @@ namespace picongpu
         }
 
         template<typename Dst, typename T_Worker>
-        HDINLINE void operator()(T_Worker const&, Dst& dst) const
+        HDINLINE void operator()(T_Worker const&, Dst&& dst) const
         {
-            dst = value;
+            std::forward<Dst>(dst) = value;
         }
 
     private:

include/picongpu/fields/FieldJ.kernel

@@ -131,7 +131,7 @@ namespace picongpu
                 // The rest uses normal weighting
                 const float_X weighting = particle[weighting_];
                 Velocity velocity;
-                const float3_X vel = velocity(particle[momentum_], attribute::getMass(weighting, particle));
+                const float3_X vel = velocity(static_cast<float3_X>(particle[momentum_]), attribute::getMass(weighting, particle));
                 auto fieldJShiftToParticle = jBox.shift(localCell);
                 ParticleAlgo perParticle;
                 perParticle(worker, fieldJShiftToParticle, pos, vel, charge, m_deltaTime);

include/picongpu/fields/FieldTmp.kernel

@@ -94,7 +94,7 @@ namespace picongpu
             if(!forEachParticle.hasParticles())
                 return;
 
-            auto cachedVal = CachedBox::create<0, typename T_TmpBox::ValueType>(worker, T_BlockDescription{});
+            auto cachedVal = CachedBox::create<0, sharedDataBoxMapping, typename T_TmpBox::ValueType>(worker, T_BlockDescription{});
             Set<typename T_TmpBox::ValueType> set(float_X(0.0));
 
             auto collective = makeThreadCollective<T_BlockDescription>();

include/picongpu/fields/MaxwellSolver/AddCurrentDensity.kernel

@@ -22,6 +22,8 @@
 
 #include "picongpu/simulation_defines.hpp"
 
+#include "picongpu/param/memory.param"
+
 #include <pmacc/dimensions/SuperCellDescription.hpp>
 #include <pmacc/lockstep.hpp>
 #include <pmacc/mappings/threads/ThreadCollective.hpp>
@@ -68,7 +70,9 @@ namespace picongpu::fields::maxwellSolver
 
             constexpr uint32_t cellsPerSuperCell = pmacc::math::CT::volume<SuperCellSize>::type::value;
 
-            auto cachedJ = CachedBox::create<0, typename FieldJ::DataBoxType::ValueType>(worker, BlockArea());
+            auto cachedJ = CachedBox::create<0, sharedDataBoxMapping, typename FieldJ::DataBoxType::ValueType>(
+                worker,
+                BlockArea());
 
             pmacc::math::operation::Assign assign;
             DataSpace<simDim> const block(

include/picongpu/fields/MaxwellSolver/FDTD/FDTDBase.kernel

@@ -179,7 +179,7 @@ namespace picongpu
                         auto srcFieldBlock = srcField.shift(beginCellIdx);
                         auto cacheStencilArea = makeThreadCollective<StencilCfg>();
                         auto cachedSrcField
-                            = CachedBox::create<0u, typename T_SrcBox::ValueType>(worker, StencilCfg{});
+                            = CachedBox::create<0u, sharedDataBoxMapping, typename T_SrcBox::ValueType>(worker, StencilCfg{});
                         cacheStencilArea(worker, assign, cachedSrcField, srcFieldBlock);
 
                         worker.sync();

include/picongpu/fields/currentDeposition/Cache.hpp

@@ -48,15 +48,10 @@ namespace picongpu
                  */
                 template<typename T_BlockDescription, typename T_Worker, typename T_FieldBox>
                 DINLINE static auto create(T_Worker const& worker, T_FieldBox const& fieldBox)
-#if(!BOOST_COMP_CLANG)
-                    -> decltype(CachedBox::create<0u, typename T_FieldBox::ValueType>(
-                        worker,
-                        std::declval<T_BlockDescription>()))
-#endif
                 {
                     using ValueType = typename T_FieldBox::ValueType;
                     /* this memory is used by all virtual blocks */
-                    auto cache = CachedBox::create<0u, ValueType>(worker, T_BlockDescription{});
+                    auto cache = CachedBox::create<0u, sharedDataBoxMapping, ValueType>(worker, T_BlockDescription{});
 
                     Set<ValueType> set(ValueType::create(0.0_X));
                     auto collectiveFill = makeThreadCollective<T_BlockDescription>();
@@ -90,9 +85,6 @@ namespace picongpu
                  */
                 template<typename T_BlockDescription, typename T_Worker, typename T_FieldBox>
                 DINLINE static auto create([[maybe_unused]] T_Worker const& worker, T_FieldBox const& fieldBox)
-#if(!BOOST_COMP_CLANG)
-                    -> T_FieldBox
-#endif
                 {
                     return fieldBox;
                 }

include/picongpu/param/memory.param

@@ -114,4 +114,7 @@ namespace picongpu
      */
     constexpr bool fieldTmpSupportGatherCommunication = true;
 
+    inline constexpr auto particleFrameMapping = pmacc::ParticleFrameMapping::SoA;
+
+    inline constexpr auto sharedDataBoxMapping = pmacc::SharedDataBoxMapping::AoS;
 } // namespace picongpu

include/picongpu/particles/Particles.hpp

@@ -22,6 +22,7 @@
 
 #include "picongpu/fields/Fields.def"
 #include "picongpu/fields/Fields.hpp"
+#include "picongpu/param/memory.param"
 #include "picongpu/particles/boundary/Description.hpp"
 #include "picongpu/particles/boundary/Utility.hpp"
 #include "picongpu/particles/manipulators/manipulators.def"
@@ -91,7 +92,10 @@ namespace picongpu
                       // fallback if the species has not defined the alias boundaryCondition
                       pmacc::HandleGuardRegion<
                           pmacc::particles::policies::ExchangeParticles,
-                          pmacc::particles::policies::DoNothing>>::type>,
+                          pmacc::particles::policies::DoNothing>>::type,
+                  bmpl::vector0<>,
+                  bmpl::vector0<>,
+                  picongpu::particleFrameMapping>,
               MappingDesc,
               DeviceHeap>
         , public ISimulationData
@@ -110,7 +114,10 @@ namespace picongpu
                 // fallback if the species has not defined the alias boundaryCondition
                 pmacc::HandleGuardRegion<
                     pmacc::particles::policies::ExchangeParticles,
-                    pmacc::particles::policies::DoNothing>>::type>;
+                    pmacc::particles::policies::DoNothing>>::type,
+            bmpl::vector0<>,
+            bmpl::vector0<>,
+            picongpu::particleFrameMapping>;
         using ParticlesBaseType = ParticlesBase<SpeciesParticleDescription, picongpu::MappingDesc, DeviceHeap>;
         using FrameType = typename ParticlesBaseType::FrameType;
         using FrameTypeBorder = typename ParticlesBaseType::FrameTypeBorder;

include/picongpu/particles/Particles.kernel

@@ -222,8 +222,8 @@ namespace picongpu
 
             onlyMaster([&]() { mustShiftSupercell = 0; });
 
-            auto cachedB = CachedBox::create<0, typename T_BBox::ValueType>(worker, T_DataDomain());
-            auto cachedE = CachedBox::create<1, typename T_EBox::ValueType>(worker, T_DataDomain());
+            auto cachedB = CachedBox::create<0, sharedDataBoxMapping, typename T_BBox::ValueType>(worker, T_DataDomain());
+            auto cachedE = CachedBox::create<1, sharedDataBoxMapping, typename T_EBox::ValueType>(worker, T_DataDomain());
 
             worker.sync();
 

include/picongpu/particles/Particles.tpp

@@ -44,6 +44,7 @@
 #include <pmacc/traits/Resolve.hpp>
 
 #include <algorithm>
+#include <fstream>
 #include <iostream>
 #include <limits>
 #include <memory>
@@ -214,6 +215,34 @@ namespace picongpu
 
         size_t sizeOfExchanges = 0u;
 
+        // std::cout << "Frame:\n\tSize: " << sizeof(FrameType) << "\n\tLLAMA view offset: " << offsetof(FrameType,
+        // view) << "\n\tLLAMA view size: " << sizeof(FrameType::view) << std::endl;
+
+        {
+            using View = decltype(FrameType::view);
+            using M = typename View::Mapping;
+            auto m = M{};
+            // View view;
+            // std::cout << "view begin " << &view << "\n";
+            // std::cout << "storage begin " << &view.storageBlobs[0][0] << "\n";
+            // for(auto ai : llama::ArrayIndexRange{m.extents()})
+            //{
+            //     llama::forEachLeafCoord<typename M::RecordDim>(
+            //         [&](auto rc)
+            //         {
+            //             auto& e = view(ai)(rc);
+            //             std::cout << "ai " << ai << " rc " << rc << " addr " << (void*) &e << "\n";
+            //         });
+            // }
+            // std::cout << "view end " << (&view + 1) << "\n";
+
+            std::ofstream{"llama_frame.html"} << llama::toHtml(m);
+            std::ofstream{"llama_frame.svg"} << llama::toSvg(m);
+
+            // PMACC_VERIFY(
+            //     reinterpret_cast<std::byte*>(&view) == reinterpret_cast<std::byte*>(&view.storageBlobs[0][0]));
+        }
+
         const uint32_t commTag = pmacc::traits::GetUniqueTypeId<FrameType, uint32_t>::uid();
         log<picLog::MEMORY>("communication tag for species %1%: %2%") % FrameType::getName() % commTag;
 

include/picongpu/particles/bremsstrahlung/Bremsstrahlung.hpp

@@ -87,7 +87,11 @@ namespace picongpu
                 /* shared memory ion density device databoxes */
                 PMACC_ALIGN(
                     cachedIonDensity,
-                    DataBox<SharedBox<ValueTypeIonDensity, typename BlockArea::FullSuperCellSize, 0>>);
+                    DataBox<SharedBox<
+                        ValueTypeIonDensity,
+                        typename BlockArea::FullSuperCellSize,
+                        0,
+                        sharedDataBoxMapping>>);
 
                 PMACC_ALIGN(scaledSpectrumFunctor, ScaledSpectrum::LookupTableFunctor);
                 PMACC_ALIGN(stoppingPowerFunctor, ScaledSpectrum::LookupTableFunctor);

include/picongpu/particles/bremsstrahlung/Bremsstrahlung.tpp

@@ -81,7 +81,7 @@ namespace picongpu
                 const DataSpace<simDim>& blockCell)
             {
                 /* caching of ion density field */
-                cachedIonDensity = CachedBox::create<0, ValueTypeIonDensity>(worker, BlockArea());
+                cachedIonDensity = CachedBox::create<0, sharedDataBoxMapping, ValueTypeIonDensity>(worker, BlockArea());
 
                 /* instance of nvidia assignment operator */
                 pmacc::math::operation::Assign assign;

include/picongpu/particles/ionization/byCollision/ThomasFermi/ThomasFermi_Impl.hpp

@@ -23,6 +23,7 @@
 
 #include "picongpu/fields/CellType.hpp"
 #include "picongpu/fields/FieldTmp.hpp"
+#include "picongpu/param/memory.param"
 #include "picongpu/particles/atomicPhysics/SetToAtomicGroundStateForChargeState.hpp"
 #include "picongpu/particles/ionization/byCollision/ThomasFermi/AlgorithmThomasFermi.hpp"
 #include "picongpu/particles/ionization/byCollision/ThomasFermi/ThomasFermi.def"
@@ -104,8 +105,12 @@ namespace picongpu
                 PMACC_ALIGN(eneBox, FieldTmp::DataBoxType);
 
                 /* shared memory EM-field device databoxes */
-                PMACC_ALIGN(cachedRho, DataBox<SharedBox<ValueType_Rho, typename BlockArea::FullSuperCellSize, 0>>);
-                PMACC_ALIGN(cachedEne, DataBox<SharedBox<ValueType_Ene, typename BlockArea::FullSuperCellSize, 1>>);
+                PMACC_ALIGN(
+                    cachedRho,
+                    DataBox<SharedBox<ValueType_Rho, typename BlockArea::FullSuperCellSize, 0, sharedDataBoxMapping>>);
+                PMACC_ALIGN(
+                    cachedEne,
+                    DataBox<SharedBox<ValueType_Ene, typename BlockArea::FullSuperCellSize, 1, sharedDataBoxMapping>>);
 
             public:
                 /* host constructor initializing member : random number generator */
@@ -185,8 +190,8 @@ namespace picongpu
                 DINLINE void collectiveInit(const T_Worker& worker, const DataSpace<simDim>& blockCell)
                 {
                     /* caching of density and "temperature" fields */
-                    cachedRho = CachedBox::create<0, ValueType_Rho>(worker, BlockArea());
-                    cachedEne = CachedBox::create<1, ValueType_Ene>(worker, BlockArea());
+                    cachedRho = CachedBox::create<0, sharedDataBoxMapping, ValueType_Rho>(worker, BlockArea());
+                    cachedEne = CachedBox::create<1, sharedDataBoxMapping, ValueType_Ene>(worker, BlockArea());
 
                     /* instance of nvidia assignment operator */
                     pmacc::math::operation::Assign assign;

include/picongpu/particles/ionization/byField/ADK/ADK_Impl.hpp

@@ -102,8 +102,12 @@ namespace picongpu
                 PMACC_ALIGN(bBox, FieldB::DataBoxType);
                 PMACC_ALIGN(jBox, FieldJ::DataBoxType);
                 /* shared memory EM-field device databoxes */
-                PMACC_ALIGN(cachedE, DataBox<SharedBox<ValueType_E, typename BlockArea::FullSuperCellSize, 1>>);
-                PMACC_ALIGN(cachedB, DataBox<SharedBox<ValueType_B, typename BlockArea::FullSuperCellSize, 0>>);
+                PMACC_ALIGN(
+                    cachedE,
+                    DataBox<SharedBox<ValueType_E, typename BlockArea::FullSuperCellSize, 1, sharedDataBoxMapping>>);
+                PMACC_ALIGN(
+                    cachedB,
+                    DataBox<SharedBox<ValueType_B, typename BlockArea::FullSuperCellSize, 0, sharedDataBoxMapping>>);
 
             public:
                 /* host constructor initializing member : random number generator */
@@ -137,8 +141,8 @@ namespace picongpu
                     jBox = jBox.shift(blockCell);
 
                     /* caching of E and B fields */
-                    cachedB = CachedBox::create<0, ValueType_B>(worker, BlockArea());
-                    cachedE = CachedBox::create<1, ValueType_E>(worker, BlockArea());
+                    cachedB = CachedBox::create<0, sharedDataBoxMapping, ValueType_B>(worker, BlockArea());
+                    cachedE = CachedBox::create<1, sharedDataBoxMapping, ValueType_E>(worker, BlockArea());
 
                     /* instance of nvidia assignment operator */
                     pmacc::math::operation::Assign assign;

include/picongpu/particles/ionization/byField/BSI/BSI_Impl.hpp

@@ -25,6 +25,7 @@
 #include "picongpu/fields/FieldB.hpp"
 #include "picongpu/fields/FieldE.hpp"
 #include "picongpu/fields/FieldJ.hpp"
+#include "picongpu/param/memory.param"
 #include "picongpu/particles/ParticlesFunctors.hpp"
 #include "picongpu/particles/atomicPhysics/SetToAtomicGroundStateForChargeState.hpp"
 #include "picongpu/particles/ionization/byField/BSI/AlgorithmBSI.hpp"
@@ -93,7 +94,9 @@ namespace picongpu
                 FieldE::DataBoxType eBox;
                 FieldJ::DataBoxType jBox;
                 /* shared memory EM-field device databoxes */
-                PMACC_ALIGN(cachedE, DataBox<SharedBox<ValueType_E, typename BlockArea::FullSuperCellSize, 1>>);
+                PMACC_ALIGN(
+                    cachedE,
+                    DataBox<SharedBox<ValueType_E, typename BlockArea::FullSuperCellSize, 1, sharedDataBoxMapping>>);
 
             public:
                 /* host constructor */
@@ -125,7 +128,7 @@ namespace picongpu
                     jBox = jBox.shift(blockCell);
 
                     /* caching of E field */
-                    cachedE = CachedBox::create<1, ValueType_E>(worker, BlockArea());
+                    cachedE = CachedBox::create<1, sharedDataBoxMapping, ValueType_E>(worker, BlockArea());
 
                     /* instance of nvidia assignment operator */
                     pmacc::math::operation::Assign assign;

include/picongpu/particles/ionization/byField/Keldysh/Keldysh_Impl.hpp

@@ -102,8 +102,12 @@ namespace picongpu
                 PMACC_ALIGN(bBox, FieldB::DataBoxType);
                 PMACC_ALIGN(jBox, FieldJ::DataBoxType);
                 /* shared memory EM-field device databoxes */
-                PMACC_ALIGN(cachedE, DataBox<SharedBox<ValueType_E, typename BlockArea::FullSuperCellSize, 1>>);
-                PMACC_ALIGN(cachedB, DataBox<SharedBox<ValueType_B, typename BlockArea::FullSuperCellSize, 0>>);
+                PMACC_ALIGN(
+                    cachedE,
+                    DataBox<SharedBox<ValueType_E, typename BlockArea::FullSuperCellSize, 1, sharedDataBoxMapping>>);
+                PMACC_ALIGN(
+                    cachedB,
+                    DataBox<SharedBox<ValueType_B, typename BlockArea::FullSuperCellSize, 0, sharedDataBoxMapping>>);
 
             public:
                 /* host constructor initializing member : random number generator */
@@ -137,8 +141,8 @@ namespace picongpu
                     jBox = jBox.shift(blockCell);
 
                     /* caching of E and B fields */
-                    cachedB = CachedBox::create<0, ValueType_B>(worker, BlockArea());
-                    cachedE = CachedBox::create<1, ValueType_E>(worker, BlockArea());
+                    cachedB = CachedBox::create<0, sharedDataBoxMapping, ValueType_B>(worker, BlockArea());
+                    cachedE = CachedBox::create<1, sharedDataBoxMapping, ValueType_E>(worker, BlockArea());
 
                     /* instance of nvidia assignment operator */
                     pmacc::math::operation::Assign assign;

include/picongpu/particles/synchrotronPhotons/PhotonCreator.hpp

@@ -96,8 +96,12 @@ namespace picongpu
                 PMACC_ALIGN(eBox, FieldE::DataBoxType);
                 PMACC_ALIGN(bBox, FieldB::DataBoxType);
                 /* shared memory EM-field device databoxes */
-                PMACC_ALIGN(cachedE, DataBox<SharedBox<ValueType_E, typename BlockArea::FullSuperCellSize, 1>>);
-                PMACC_ALIGN(cachedB, DataBox<SharedBox<ValueType_B, typename BlockArea::FullSuperCellSize, 0>>);
+                PMACC_ALIGN(
+                    cachedE,
+                    DataBox<SharedBox<ValueType_E, typename BlockArea::FullSuperCellSize, 1, sharedDataBoxMapping>>);
+                PMACC_ALIGN(
+                    cachedB,
+                    DataBox<SharedBox<ValueType_B, typename BlockArea::FullSuperCellSize, 0, sharedDataBoxMapping>>);
 
                 PMACC_ALIGN(curF_1, SynchrotronFunctions::SyncFuncCursor);
                 PMACC_ALIGN(curF_2, SynchrotronFunctions::SyncFuncCursor);
@@ -143,8 +147,8 @@ namespace picongpu
                 DINLINE void collectiveInit(const T_Worker& worker, const DataSpace<simDim>& blockCell)
                 {
                     /* caching of E and B fields */
-                    cachedB = CachedBox::create<0, ValueType_B>(worker, BlockArea());
-                    cachedE = CachedBox::create<1, ValueType_E>(worker, BlockArea());
+                    cachedB = CachedBox::create<0, sharedDataBoxMapping, ValueType_B>(worker, BlockArea());
+                    cachedE = CachedBox::create<1, sharedDataBoxMapping, ValueType_E>(worker, BlockArea());
 
                     /* instance of nvidia assignment operator */
                     pmacc::math::operation::Assign assign;

include/pmacc/cuSTL/algorithm/functor/GetComponent.hpp

@@ -42,15 +42,15 @@ namespace pmacc
                 }
 
                 template<typename Array, typename T_Worker>
-                HDINLINE Type& operator()(T_Worker const&, Array& array) const
+                HDINLINE Type& operator()(T_Worker const&, Array&& array) const
                 {
-                    return array[m_component];
+                    return std::forward<Array>(array)[m_component];
                 }
 
                 template<typename Array>
-                HDINLINE Type& operator()(Array& array) const
+                HDINLINE Type& operator()(Array&& array) const
                 {
-                    return array[m_component];
+                    return std::forward<Array>(array)[m_component];
                 }
             };
 

include/pmacc/cuSTL/cursor/Cursor.hpp

@@ -35,6 +35,21 @@ namespace pmacc
 {
     namespace cursor
     {
+        namespace detail
+        {
+            template<typename Accessor, typename = void>
+            struct ValueType
+            {
+                using type = std::remove_reference_t<typename Accessor::Reference>;
+            };
+
+            template<typename Accessor>
+            struct ValueType<Accessor, std::void_t<typename Accessor::ValueType>>
+            {
+                using type = typename Accessor::ValueType;
+            };
+        } // namespace detail
+
         /** A cursor is used to access a single datum and to jump to another one.
          * It is always located at a certain datum. Think of a generalized iterator.
          * @tparam _Accessor Policy functor class that is called inside operator*().
@@ -51,7 +66,7 @@ namespace pmacc
         {
         public:
             using Reference = typename _Accessor::Reference;
-            using ValueType = std::remove_reference_t<Reference>;
+            using ValueType = typename detail::ValueType<_Accessor>::type;
             using Accessor = _Accessor;
             using Navigator = _Navigator;
             using Marker = _Marker;