Merge pull request #64 from DARMA-tasking/63-add-arbitrary-attributes…

…-to-json-validator

#63: Add attributes reading for json reader

src/vt-tv/api/object_work.h

@@ -77,12 +77,14 @@ struct ObjectWork {
     ElementIDType in_id,
     TimeType in_whole_phase_load,
     std::unordered_map<SubphaseType, TimeType> in_subphase_loads,
-    std::unordered_map<std::string, VariantType> in_user_defined = {}
+    std::unordered_map<std::string, VariantType> in_user_defined = {},
+    std::unordered_map<std::string, QOIVariantTypes> in_attributes = {}
   ) : id_(in_id),
       whole_phase_load_(in_whole_phase_load),
       subphase_loads_(std::move(in_subphase_loads)),
       user_defined_(std::move(in_user_defined)),
-      communicator_(id_)
+      communicator_(id_),
+      attributes_(std::move(in_attributes))
   { }
 
   /**
@@ -113,6 +115,13 @@ struct ObjectWork {
    */
   auto const& getUserDefined() const { return user_defined_; }
 
+  /**
+   * \brief Get attribute fields
+   *
+   * \return attribute fields
+   */
+  auto const& getAttributes() const { return attributes_; }
+
   /**
    * \brief set communications for this object
    *
@@ -174,6 +183,7 @@ struct ObjectWork {
     s | subphase_loads_;
     s | user_defined_;
     s | communicator_;
+    s | attributes_;
   }
 
 private:
@@ -185,6 +195,8 @@ struct ObjectWork {
   std::unordered_map<SubphaseType, TimeType> subphase_loads_;
   // User-defined field---used to populate the memory block
   std::unordered_map<std::string, VariantType> user_defined_;
+  /// QOIs to be visualized
+  std::unordered_map<std::string, QOIVariantTypes> attributes_;
 
   /// @todo: add communications
   ObjectCommunicator communicator_;

src/vt-tv/api/rank.h

@@ -65,9 +65,11 @@ struct Rank {
    */
   Rank(
     NodeType in_rank,
-    std::unordered_map<PhaseType, PhaseWork> in_phase_info
+    std::unordered_map<PhaseType, PhaseWork> in_phase_info,
+    std::unordered_map<std::string, QOIVariantTypes> in_attributes = {}
   ) : rank_(in_rank),
-      phase_info_(std::move(in_phase_info))
+      phase_info_(std::move(in_phase_info)),
+      attributes_(std::move(in_attributes))
   { }
 
   /**
@@ -98,6 +100,13 @@ struct Rank {
    */
   double getLoad(PhaseType phase) const { return phase_info_.at(phase).getLoad(); }
 
+  /**
+   * \brief Get attribute fields
+   *
+   * \return attribute fields
+   */
+  auto const& getAttributes() const { return attributes_; }
+
   /**
    * \brief Serializer for data
    *
@@ -107,13 +116,16 @@ struct Rank {
   void serialize(SerializerT& s) {
     s | rank_;
     s | phase_info_;
+    s | attributes_;
   }
 
 private:
   /// The rank ID
   NodeType rank_ = 0;
   /// Work for each phase
   std::unordered_map<PhaseType, PhaseWork> phase_info_;
+  /// QOIs to be visualized
+  std::unordered_map<std::string, QOIVariantTypes> attributes_;
 };
 
 } /* end namespace vt::tv */

src/vt-tv/api/types.h

@@ -45,6 +45,8 @@
 #define INCLUDED_VT_TV_API_TYPES_H
 
 #include <cstdint>
+#include <variant>
+#include <string>
 
 namespace vt::tv {
 
@@ -56,6 +58,9 @@ using UniqueIndexBitType = uint64_t;
 using TimeType = double;
 using CollectionObjGroupIDType = uint64_t;
 
+/// Possible QOIs types
+using QOIVariantTypes = std::variant<int, double, std::string>;
+
 } /* end namespace vt::tv */
 
 #endif /*INCLUDED_VT_TV_API_TYPES_H*/

src/vt-tv/utility/json_reader.cc

@@ -45,6 +45,7 @@
 #include "vt-tv/utility/json_reader.h"
 #include "vt-tv/utility/decompression_input_container.h"
 #include "vt-tv/utility/input_iterator.h"
+#include "vt-tv/utility/qoi_serializer.h"
 
 #include <nlohmann/json.hpp>
 #include <fmt-vt/core.h>
@@ -185,11 +186,22 @@ std::unique_ptr<Info> JSONReader::parseFile() {
                 }
               }
             }
+
+            std::unordered_map<std::string, QOIVariantTypes> readed_metadata;
+            if (task.find("attributes") != task.end()) {
+              auto attributes = task["attributes"];
+              if (attributes.is_object()) {
+                for (auto& [key, value] : attributes.items()) {
+                  readed_metadata[key] = value;
+                }
+              }
+            }
+
             // fmt::print(" Add object {}\n", (ElementIDType)object);
             objects.try_emplace(
               object,
               ObjectWork{
-                object, time, std::move(subphase_loads), std::move(user_defined)
+                object, time, std::move(subphase_loads), std::move(user_defined), std::move(readed_metadata)
               }
             );
           }
@@ -233,7 +245,20 @@ std::unique_ptr<Info> JSONReader::parseFile() {
     }
   }
 
-  Rank r{rank_, std::move(phase_info)};
+  std::unordered_map<std::string, QOIVariantTypes> readed_metadata;
+  if (j.find("metadata") != j.end()) {
+    auto metadata = j["metadata"];
+    if (metadata.find("attributes") != metadata.end()) {
+      auto attributes = metadata["attributes"];
+      if (attributes.is_object()) {
+        for (auto& [key, value] : attributes.items()) {
+          readed_metadata[key] = value;
+        }
+      }
+    }
+  }
+
+  Rank r{rank_, std::move(phase_info), std::move(readed_metadata)};
 
   std::unordered_map<NodeType, Rank> rank_info;
   rank_info.try_emplace(rank_, std::move(r));

src/vt-tv/utility/qoi_serializer.h

@@ -0,0 +1,82 @@
+/*
+//@HEADER
+// *****************************************************************************
+//
+//                            qoi_serializer.h
+//             DARMA/vt-tv => Virtual Transport -- Task Visualizer
+//
+// Copyright 2019 National Technology & Engineering Solutions of Sandia, LLC
+// (NTESS). Under the terms of Contract DE-NA0003525 with NTESS, the U.S.
+// Government retains certain rights in this software.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are met:
+//
+// * Redistributions of source code must retain the above copyright notice,
+//   this list of conditions and the following disclaimer.
+//
+// * Redistributions in binary form must reproduce the above copyright notice,
+//   this list of conditions and the following disclaimer in the documentation
+//   and/or other materials provided with the distribution.
+//
+// * Neither the name of the copyright holder nor the names of its
+//   contributors may be used to endorse or promote products derived from this
+//   software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+// POSSIBILITY OF SUCH DAMAGE.
+//
+// Questions? Contact [email protected]
+//
+// *****************************************************************************
+//@HEADER
+*/
+
+#if !defined INCLUDED_VT_TV_API_QOI_SERIALIZER_H
+#define INCLUDED_VT_TV_API_QOI_SERIALIZER_H
+
+#include "vt-tv/api/types.h"
+
+#include <nlohmann/json.hpp>
+#include <variant>
+
+namespace nlohmann
+{
+  template <>
+  struct adl_serializer<::vt::tv::QOIVariantTypes> {
+    using VariantTypes = ::vt::tv::QOIVariantTypes;
+
+    // Produce compilation error if variant types were modified
+    static_assert(std::is_same_v<int, std::variant_alternative_t<0, VariantTypes>>);
+    static_assert(std::is_same_v<double, std::variant_alternative_t<1, VariantTypes>>);
+    static_assert(std::is_same_v<std::string, std::variant_alternative_t<2, VariantTypes>>);
+
+    static void to_json(json &j, const VariantTypes &value) {
+      std::visit([&](auto const &arg)
+                 { j = arg; },
+                 value);
+    }
+
+    static void from_json(const json &j, VariantTypes &value) {
+      if (j.is_number_integer()) {
+        value = j.get<int>();
+      } else if (j.is_number_float()) {
+        value = j.get<double>();
+      } else if (j.is_string()) {
+        value = j.get<std::string>();
+      }
+    }
+  };
+
+} /* end namespace nlohmann */
+
+#endif /* INCLUDED_VT_TV_API_QOI_SERIALIZER_H */

tests/unit/lb_test_data/reader_test_data.json

@@ -0,0 +1,83 @@
+{
+    "metadata": {
+        "rank": 0,
+        "shared_node": {
+            "id": 0,
+            "num_nodes": 1,
+            "rank": 0,
+            "size": 4
+        },
+        "type": "LBDatafile",
+        "attributes": {
+            "intSample": 1,
+            "doubleSample": 2.213,
+            "stringSample": "abc"
+        }
+    },
+    "phases": [
+        {
+            "communications": [
+                {
+                    "bytes": 112.0,
+                    "from": {
+                        "home": 0,
+                        "id": 0,
+                        "migratable": false,
+                        "type": "object"
+                    },
+                    "messages": 2,
+                    "to": {
+                        "home": 1,
+                        "id": 5,
+                        "migratable": false,
+                        "type": "object"
+                    },
+                    "type": "SendRecv"
+                }
+            ],
+            "id": 0,
+            "tasks": [
+                {
+                    "entity": {
+                        "collection_id": 7,
+                        "home": 0,
+                        "id": 3407875,
+                        "index": [
+                            12
+                        ],
+                        "migratable": true,
+                        "type": "object"
+                    },
+                    "node": 0,
+                    "resource": "cpu",
+                    "subphases": [
+                        {
+                            "id": 0,
+                            "time": 1.1263000033068238e-05
+                        },
+                        {
+                            "id": 1,
+                            "time": 1.1333999964335817e-05
+                        },
+                        {
+                            "id": 2,
+                            "time": 1.1196000059499056e-05
+                        }
+                    ],
+                    "time": 3.379300005690311e-05,
+                    "user_defined": {
+                        "isSample": 1
+                    },
+                    "attributes": {
+                        "intSample": -100,
+                        "doubleSample": 0.0,
+                        "stringSample": ""
+                    }
+                }
+            ],
+            "user_defined": {
+                "isSample": 1
+            }
+        }
+    ]
+}