From 6a7dcab30dc2db980ac52489b3cc2384cb74f3ae Mon Sep 17 00:00:00 2001 From: lambda7xx Date: Sat, 19 Aug 2023 09:30:25 +0000 Subject: [PATCH 1/5] add test for the kernel help --- cmake/cuda.cmake | 7 +- lib/kernels/CMakeLists.txt | 2 + lib/kernels/include/kernels/array_shape.h | 2 + lib/kernels/include/kernels/profiling.h | 4 +- lib/kernels/src/array_shape.cc | 62 ++++++++++++++ lib/kernels/src/perf_metrics.cc | 5 +- lib/kernels/test/CMakeLists.txt | 27 ++++++ lib/kernels/test/src/doctest.h | 1 + lib/kernels/test/src/main.cc | 2 + lib/kernels/test/src/test_accessor.cc | 84 +++++++++++++++++++ lib/kernels/test/src/test_array_shape.cc | 45 ++++++++++ .../test/src/test_datatype_dispatch.cc | 41 +++++++++ lib/kernels/test/src/test_legion_dim.cc | 52 ++++++++++++ lib/kernels/test/src/test_perf_metrics.cc | 49 +++++++++++ lib/kernels/test/src/test_profiling.cc | 49 +++++++++++ lib/op-attrs/include/op-attrs/dim_ordered.h | 2 + 16 files changed, 430 insertions(+), 4 deletions(-) create mode 100644 lib/kernels/test/CMakeLists.txt create mode 100644 lib/kernels/test/src/doctest.h create mode 100644 lib/kernels/test/src/main.cc create mode 100644 lib/kernels/test/src/test_accessor.cc create mode 100644 lib/kernels/test/src/test_array_shape.cc create mode 100644 lib/kernels/test/src/test_datatype_dispatch.cc create mode 100644 lib/kernels/test/src/test_legion_dim.cc create mode 100644 lib/kernels/test/src/test_perf_metrics.cc create mode 100644 lib/kernels/test/src/test_profiling.cc diff --git a/cmake/cuda.cmake b/cmake/cuda.cmake index da358e31dd..94c29e2274 100644 --- a/cmake/cuda.cmake +++ b/cmake/cuda.cmake @@ -83,7 +83,12 @@ if(CUDA_FOUND) add_library(cuda INTERFACE) target_include_directories(cuda SYSTEM INTERFACE "${CUDA_INCLUDE_DIRS}") - + target_link_libraries(cuda INTERFACE + ${CUDADRV_LIBRARIES} + ${CUDA_LIBRARIES} + ${CUDA_CUBLAS_LIBRARIES} + ${CUDA_curand_LIBRARY}) + else() message( FATAL_ERROR "CUDA package not found -> specify search path via CUDA_ROOT variable") endif() diff --git a/lib/kernels/CMakeLists.txt b/lib/kernels/CMakeLists.txt index 59c7d44b60..36b81eb277 100644 --- a/lib/kernels/CMakeLists.txt +++ b/lib/kernels/CMakeLists.txt @@ -35,3 +35,5 @@ set_target_properties( PROPERTIES CUDA_STANDARD 11 ) + +add_subdirectory(test) \ No newline at end of file diff --git a/lib/kernels/include/kernels/array_shape.h b/lib/kernels/include/kernels/array_shape.h index 41c8275b1c..7ffeeb7742 100644 --- a/lib/kernels/include/kernels/array_shape.h +++ b/lib/kernels/include/kernels/array_shape.h @@ -38,6 +38,8 @@ struct ArrayShape { optional at_maybe(std::size_t) const; + bool operator==(ArrayShape const &other) const; // for test case + ArrayShape reversed_dim_order() const; ArrayShape sub_shape(optional start, optional end); diff --git a/lib/kernels/include/kernels/profiling.h b/lib/kernels/include/kernels/profiling.h index 6a06a7b7a5..b1877402ea 100644 --- a/lib/kernels/include/kernels/profiling.h +++ b/lib/kernels/include/kernels/profiling.h @@ -7,7 +7,7 @@ namespace FlexFlow { -struct ProfilingSettings : public use_visitable_cmp { +struct ProfilingSettings { public: ProfilingSettings() = delete; ProfilingSettings(int warmup_iters, int measure_iters); @@ -17,6 +17,8 @@ struct ProfilingSettings : public use_visitable_cmp { int measure_iters; }; +FF_VISITABLE_STRUCT(ProfilingSettings, warmup_iters, measure_iters); + template optional profiling_wrapper(F const &f, bool enable_profiling, Ts &&...ts) { diff --git a/lib/kernels/src/array_shape.cc b/lib/kernels/src/array_shape.cc index 44507c14c4..e756411523 100644 --- a/lib/kernels/src/array_shape.cc +++ b/lib/kernels/src/array_shape.cc @@ -7,7 +7,69 @@ ArrayShape::ArrayShape(size_t *_dims, size_t num_dims) : dims(_dims, _dims + num_dims) {} std::size_t ArrayShape::get_volume() const { + return num_elements(); +} + +std::size_t ArrayShape::get_dim() const { + return num_dims(); +} + +std::size_t ArrayShape::num_elements() const { return product(this->dims); } +std::size_t ArrayShape::num_dims() const { + return this->dims.size(); +} + +std::size_t ArrayShape::operator[](legion_dim_t idx) const { + return dims.at(idx); +} + +std::size_t ArrayShape::at(legion_dim_t idx) const { + return dims.at(idx); +} + +legion_dim_t ArrayShape::last_idx() const { + return legion_dim_t(dims.size() - 1); +} + +legion_dim_t ArrayShape::neg_idx(int idx) const { + assert(idx < 0 && "Idx should be negative for negative indexing"); + return legion_dim_t(dims.size() + idx); +} + +optional ArrayShape::at_maybe(std::size_t idx) const { + if (idx < dims.size()) { + return dims[legion_dim_t(idx)]; + } else { + return {}; + } +} + +ArrayShape ArrayShape::reversed_dim_order() const { + std::vector dims_reversed(dims.rbegin(), dims.rend()); + return ArrayShape(dims_reversed); +} + +ArrayShape ArrayShape::sub_shape(optional start, + optional end) { + size_t s = start.has_value() ? start.value().value() : 0; + size_t e = end.has_value() ? end.value().value() : dims.size(); + std::vector sub_dims(dims.begin() + s, dims.begin() + e); + return ArrayShape(sub_dims); +} + +bool ArrayShape::operator==(ArrayShape const &other) const { + if (this->dims.size() != other.dims.size()) { + return false; + } + + return this->dims == other.dims; +} + +size_t get_volume(ArrayShape const &shape) { + return shape.get_volume(); +} + } // namespace FlexFlow diff --git a/lib/kernels/src/perf_metrics.cc b/lib/kernels/src/perf_metrics.cc index 07bb8de815..ad854dfaf7 100644 --- a/lib/kernels/src/perf_metrics.cc +++ b/lib/kernels/src/perf_metrics.cc @@ -15,8 +15,9 @@ PerfMetrics::PerfMetrics(int _train_all, double _start_time_micro, double _current_time_micro) : train_all(_train_all), train_correct(_train_correct), cce_loss(_cce_loss), - mse_loss(_mse_loss), rmse_loss(_rmse_loss), mae_loss(_mae_loss), - start_time(_start_time_micro), current_time(_current_time_micro) {} + sparse_cce_loss(_sparse_cce_loss), mse_loss(_mse_loss), + rmse_loss(_rmse_loss), mae_loss(_mae_loss), start_time(_start_time_micro), + current_time(_current_time_micro) {} float get_throughput(PerfMetrics const &m) { return m.train_all / (m.current_time - m.start_time); diff --git a/lib/kernels/test/CMakeLists.txt b/lib/kernels/test/CMakeLists.txt new file mode 100644 index 0000000000..3c56a7139b --- /dev/null +++ b/lib/kernels/test/CMakeLists.txt @@ -0,0 +1,27 @@ +set(project_target kernel-test) +project(${project_target} LANGUAGES CUDA) + +file(GLOB_RECURSE SRC + CONFIGURE_DEPENDS + LIST_DIRECTORIES False + src/*.cc) + +add_executable( + ${project_target} + ${SRC}) + +target_link_libraries( + ${project_target} + kernels + cuda + rapidcheck + doctest::doctest) + +set_target_properties( + ${project_target} + PROPERTIES + CUDA_STANDARD 11 +) + +define_ff_vars(${project_target}) +doctest_discover_tests(${project_target}) \ No newline at end of file diff --git a/lib/kernels/test/src/doctest.h b/lib/kernels/test/src/doctest.h new file mode 100644 index 0000000000..7cbb6c633a --- /dev/null +++ b/lib/kernels/test/src/doctest.h @@ -0,0 +1 @@ +#include "doctest/doctest.h" diff --git a/lib/kernels/test/src/main.cc b/lib/kernels/test/src/main.cc new file mode 100644 index 0000000000..9522fa7fdb --- /dev/null +++ b/lib/kernels/test/src/main.cc @@ -0,0 +1,2 @@ +#define DOCTEST_CONFIG_IMPLEMENT_WITH_MAIN +#include "doctest/doctest.h" diff --git a/lib/kernels/test/src/test_accessor.cc b/lib/kernels/test/src/test_accessor.cc new file mode 100644 index 0000000000..4792cc6f82 --- /dev/null +++ b/lib/kernels/test/src/test_accessor.cc @@ -0,0 +1,84 @@ + +#include "doctest.h" +#include "kernels/accessor.h" + +using namespace FlexFlow; + +TEST_CASE("Test GenericTensorAccessorW") { + float dataW = 3.14f; + GenericTensorAccessorW accessorW{ + DataType::FLOAT, ArrayShape(std::vector{}), &dataW}; + + // Test get method + CHECK(*accessorW.get() == doctest::Approx(3.14f)); + + // Test specific type get ptr methods + CHECK(get_float_ptr(accessorW) != nullptr); + CHECK(*get_float_ptr(accessorW) == doctest::Approx(3.14f)); + + // Check runtime error for invalid access + CHECK_THROWS_WITH(accessorW.get(), + "Invalid access data type (FLOAT != DOUBLE)"); +} + +TEST_CASE("Test GenericTensorAccessorR") { + float dataR = 7.89f; + GenericTensorAccessorR accessorR{ + DataType::FLOAT, ArrayShape(std::vector{}), &dataR}; + // Test get method + CHECK(*accessorR.get() == doctest::Approx(7.89f)); + + // Test specific type get ptr methods + CHECK(get_float_ptr(accessorR) != nullptr); + CHECK(*get_float_ptr(accessorR) == doctest::Approx(7.89f)); + + // Check runtime error for invalid access + CHECK_THROWS_WITH(accessorR.get(), + "Invalid access data type (FLOAT != DOUBLE)"); +} + +TEST_CASE("Test get_int32_ptr for GenericTensorAccessorW") { + int32_t dataW = 12345; + GenericTensorAccessorW accessorW{ + DataType::INT32, ArrayShape(std::vector{}), &dataW}; + + // Test get_int32_ptr method + CHECK(get_int32_ptr(accessorW) != nullptr); + CHECK(*get_int32_ptr(accessorW) == 12345); +} + +TEST_CASE("Test get_int64_ptr for GenericTensorAccessorW") { + int64_t dataW = 1234567890LL; + GenericTensorAccessorW accessorW{ + DataType::INT64, ArrayShape(std::vector{}), &dataW}; + // Test get_int64_ptr method + CHECK(get_int64_ptr(accessorW) != nullptr); + CHECK(*get_int64_ptr(accessorW) == 1234567890LL); +} + +TEST_CASE("Test get_float_ptr for GenericTensorAccessorW") { + float dataW = 3.14f; + GenericTensorAccessorW accessorW{ + DataType::FLOAT, ArrayShape(std::vector{}), &dataW}; + // Test get_float_ptr method + CHECK(get_float_ptr(accessorW) != nullptr); + CHECK(*get_float_ptr(accessorW) == doctest::Approx(3.14f)); +} + +TEST_CASE("Test get_double_ptr for GenericTensorAccessorW") { + double dataW = 6.28; + GenericTensorAccessorW accessorW{ + DataType::DOUBLE, ArrayShape(std::vector{}), &dataW}; + // Test get_double_ptr method + CHECK(get_double_ptr(accessorW) != nullptr); + CHECK(*get_double_ptr(accessorW) == doctest::Approx(6.28)); +} + +TEST_CASE("Test get_int32_ptr for GenericTensorAccessorR") { + int32_t dataR = 67890; + GenericTensorAccessorR accessorR{ + DataType::INT32, ArrayShape(std::vector{}), &dataR}; + // Test get_int32_ptr method + CHECK(get_int32_ptr(accessorR) != nullptr); + CHECK(*get_int32_ptr(accessorR) == 67890); +} diff --git a/lib/kernels/test/src/test_array_shape.cc b/lib/kernels/test/src/test_array_shape.cc new file mode 100644 index 0000000000..676c2df9bd --- /dev/null +++ b/lib/kernels/test/src/test_array_shape.cc @@ -0,0 +1,45 @@ +#include "doctest.h" +#include "kernels/array_shape.h" // Assuming this is where your ArrayShape is +#include "kernels/legion_dim.h" + +using namespace FlexFlow; + +TEST_CASE("ArrayShape Initialization and Basic Functions") { + std::vector dims = {2, 3, 4}; + ArrayShape shape(dims); + CHECK(shape.get_dim() == 3); + CHECK(shape.get_volume() == 24); + CHECK(shape.num_elements() == 24); + CHECK(shape.num_dims() == 3); + CHECK(shape[legion_dim_t(1)] == 3); + CHECK(shape.at(legion_dim_t(2)) == 4); +} + +TEST_CASE("Negative Indices and Optional Indexing") { + std::vector dims = {2, 3, 4}; + ArrayShape shape(dims); + + CHECK(shape.neg_idx(-1) == legion_dim_t(2)); + CHECK(shape.neg_idx(-3) == legion_dim_t(0)); + + CHECK(shape.at_maybe(0) == 2); + CHECK(shape.at_maybe(2) == 4); + CHECK(!shape.at_maybe(5).has_value()); +} + +TEST_CASE("Reversed Dim Order and Sub-shape") { + using namespace FlexFlow; + + std::vector dims = {2, 3, 4}; + ArrayShape shape(dims); + + ArrayShape reversed = shape.reversed_dim_order(); + CHECK(reversed[legion_dim_t(0)] == 4); + CHECK(reversed[legion_dim_t(1)] == 3); + CHECK(reversed[legion_dim_t(2)] == 2); + + ArrayShape sub = shape.sub_shape(legion_dim_t(0), legion_dim_t(2)); + CHECK(sub.get_dim() == 2); + CHECK(sub[legion_dim_t(0)] == 2); + CHECK(sub[legion_dim_t(1)] == 3); +} diff --git a/lib/kernels/test/src/test_datatype_dispatch.cc b/lib/kernels/test/src/test_datatype_dispatch.cc new file mode 100644 index 0000000000..b4ef2b4a07 --- /dev/null +++ b/lib/kernels/test/src/test_datatype_dispatch.cc @@ -0,0 +1,41 @@ +#include "doctest.h" +#include "kernels/datatype_dispatch.h" + +using namespace FlexFlow; + +template +struct Function1 { + int operator()(int value) const { + if (DT == DataType::FLOAT) { + return value + 1; + } + if (DT == DataType::DOUBLE) { + return value + 2; + } + return 0; + } +}; + +template +struct Function2 { + int operator()(int value) const { + if (DT1 == DataType::FLOAT && DT2 == DataType::DOUBLE) { + return value + 3; + } + return 0; + } +}; + +TEST_CASE("Testing dispatch function") { + int value = 10; + int result = dispatch(DataType::FLOAT, value); + CHECK(result == 11); +} + +// test DataTypeDispatch1 +TEST_CASE("Testing DataTypeDispatch1") { + DataTypeDispatch1 dispatcher; + int value = 10; + int result = dispatcher(DataType::FLOAT, value); + CHECK(result == 11); +} diff --git a/lib/kernels/test/src/test_legion_dim.cc b/lib/kernels/test/src/test_legion_dim.cc new file mode 100644 index 0000000000..e4048b04bf --- /dev/null +++ b/lib/kernels/test/src/test_legion_dim.cc @@ -0,0 +1,52 @@ +#include "doctest.h" +#include "kernels/legion_dim.h" + +using namespace FlexFlow; + +TEST_CASE("Testing DimOrdered") { + SUBCASE("constructor method") { + DimOrdered fromInitList = {1, 2, 3}; + CHECK(fromInitList.size() == 3); + std::vector vec = {4, 5, 6}; + DimOrdered fromVector(vec); + CHECK(fromVector.size() == 3); + } + + SUBCASE("at") { + DimOrdered dimOrder = {1, 2, 3}; + CHECK(dimOrder[legion_dim_t(0)] == 1); + CHECK(dimOrder[legion_dim_t(1)] == 2); + CHECK(dimOrder[legion_dim_t(2)] == 3); + } + + SUBCASE("comparsion") { + DimOrdered order1 = {1, 2, 3}; + DimOrdered order2 = {1, 2, 4}; + DimOrdered order3 = {1, 2, 3}; + + CHECK(order1 != order2); + CHECK(order1 == order3); + } + + SUBCASE("iterator") { + DimOrdered dimOrder = {1, 2, 3}; + int sum = 0; + for (int value : dimOrder) { + sum += value; + } + CHECK(sum == 6); + } +} + +TEST_CASE("Testing LegionTensorDims") { + + SUBCASE("LegionTensorDims Basic Operation") { + LegionTensorDims tensorDims = {100, 200}; + + // tensorDims[legion_dim_t(1)] = 100; + CHECK(tensorDims[legion_dim_t(0)] == 100); + + // tensorDims[legion_dim_t(2)] = 200; + CHECK(tensorDims[legion_dim_t(1)] == 200); + } +} diff --git a/lib/kernels/test/src/test_perf_metrics.cc b/lib/kernels/test/src/test_perf_metrics.cc new file mode 100644 index 0000000000..731b516588 --- /dev/null +++ b/lib/kernels/test/src/test_perf_metrics.cc @@ -0,0 +1,49 @@ +#include "doctest.h" +#include "kernels/perf_metrics.h" +#include + +using namespace FlexFlow; + +// Helper function to generate random values for PerfMetrics +PerfMetrics randomPerfMetrics() { + std::random_device rd; + std::mt19937 gen(rd()); + + std::uniform_real_distribution<> dis(0.0, 1.0); + + PerfMetrics metrics{0}; + metrics.train_all = dis(gen); + metrics.train_correct = dis(gen); + metrics.cce_loss = dis(gen); + metrics.sparse_cce_loss = dis(gen); + metrics.mse_loss = dis(gen); + metrics.rmse_loss = dis(gen); + metrics.mae_loss = dis(gen); + metrics.start_time = dis(gen); + metrics.current_time = dis(gen); + + return metrics; +} + +TEST_CASE("PerfMetricsTests") { + + SUBCASE("Throughput non-negative") { + auto m = randomPerfMetrics(); + CHECK(get_throughput(m) >= 0); + } + + SUBCASE("Accuracy between 0 and 1") { + auto m = randomPerfMetrics(); + float accuracy = get_accuracy(m); + CHECK(accuracy <= 0.0f); + CHECK(accuracy <= 1.0f); + } + + SUBCASE("Update maintains non-negative values") { + auto lhs = randomPerfMetrics(); + auto rhs = randomPerfMetrics(); + auto result = update(lhs, rhs); + CHECK(result.train_all >= 0); + // Add other assertions for other fields... + } +} diff --git a/lib/kernels/test/src/test_profiling.cc b/lib/kernels/test/src/test_profiling.cc new file mode 100644 index 0000000000..5377212e38 --- /dev/null +++ b/lib/kernels/test/src/test_profiling.cc @@ -0,0 +1,49 @@ +#include "doctest.h" +#include "kernels/perf_metrics.h" + +using namespace FlexFlow; + +TEST_CASE("PerfMetrics Tests") { + + SUBCASE("Constructor and basic properties") { + double start_time = 100.0; + PerfMetrics metrics(start_time); + + CHECK(metrics.start_time == start_time); + CHECK(metrics.current_time == start_time); + CHECK(metrics.train_all == 0); + } + + SUBCASE("Throughput non-negative") { + PerfMetrics m(10, 5, 0.5, 0.5, 0.5, 0.5, 0.5, 100.0, 200.0); + CHECK(get_throughput(m) >= 0); + } + + SUBCASE("Accuracy between 0 and 1") { + PerfMetrics m(10, 5, 0.5, 0.5, 0.5, 0.5, 0.5, 100.0, 200.0); + float accuracy = get_accuracy(m); + CHECK(accuracy >= 0.0f); + CHECK(accuracy <= 1.0f); + } + + SUBCASE("Update maintains non-negative values") { + PerfMetrics lhs(10, 5, 0.5, 0.5, 0.5, 0.5, 0.5, 100.0, 200.0); + PerfMetrics rhs(5, 3, 0.2, 0.2, 0.2, 0.2, 0.2, 200.0, 300.0); + + auto result = update(lhs, rhs); + CHECK(result.train_all == 15); + CHECK(result.train_correct.value() == 8); + } + + SUBCASE("Scale values correctly") { + PerfMetrics pm(10, 5, 0.5, 0.5, 0.5, 0.5, 0.5, 100.0, 200.0); + float scale = 2.0f; + + auto result = apply_scale(pm, scale); + CHECK(result.cce_loss.value() == doctest::Approx(1.0f)); + CHECK(result.sparse_cce_loss.value() == doctest::Approx(1.0f)); + CHECK(result.mse_loss.value() == doctest::Approx(1.0f)); + CHECK(result.rmse_loss.value() == doctest::Approx(1.0f)); + CHECK(result.mae_loss.value() == doctest::Approx(1.0f)); + } +} diff --git a/lib/op-attrs/include/op-attrs/dim_ordered.h b/lib/op-attrs/include/op-attrs/dim_ordered.h index b726d0687f..fd97359a88 100644 --- a/lib/op-attrs/include/op-attrs/dim_ordered.h +++ b/lib/op-attrs/include/op-attrs/dim_ordered.h @@ -4,6 +4,8 @@ #include "op-attrs/ff_dim.h" #include "utils/stack_vector.h" +#define MAX_TENSOR_DIM 5 + namespace FlexFlow { template From 6703ef296902c481a1b3014461b8c901a11a9825 Mon Sep 17 00:00:00 2001 From: lambda7xx Date: Sat, 19 Aug 2023 09:35:27 +0000 Subject: [PATCH 2/5] refine the test_datatype_dispatch --- lib/kernels/test/src/test_datatype_dispatch.cc | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/lib/kernels/test/src/test_datatype_dispatch.cc b/lib/kernels/test/src/test_datatype_dispatch.cc index b4ef2b4a07..9f4e81a68a 100644 --- a/lib/kernels/test/src/test_datatype_dispatch.cc +++ b/lib/kernels/test/src/test_datatype_dispatch.cc @@ -16,16 +16,6 @@ struct Function1 { } }; -template -struct Function2 { - int operator()(int value) const { - if (DT1 == DataType::FLOAT && DT2 == DataType::DOUBLE) { - return value + 3; - } - return 0; - } -}; - TEST_CASE("Testing dispatch function") { int value = 10; int result = dispatch(DataType::FLOAT, value); From e370e54868a5e2be758c0e022475975731ab4221 Mon Sep 17 00:00:00 2001 From: lambda7xx Date: Sat, 19 Aug 2023 09:36:16 +0000 Subject: [PATCH 3/5] fix the error of test_perf_metrics --- lib/kernels/test/src/test_perf_metrics.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/kernels/test/src/test_perf_metrics.cc b/lib/kernels/test/src/test_perf_metrics.cc index 731b516588..c60174b349 100644 --- a/lib/kernels/test/src/test_perf_metrics.cc +++ b/lib/kernels/test/src/test_perf_metrics.cc @@ -35,7 +35,7 @@ TEST_CASE("PerfMetricsTests") { SUBCASE("Accuracy between 0 and 1") { auto m = randomPerfMetrics(); float accuracy = get_accuracy(m); - CHECK(accuracy <= 0.0f); + CHECK(accuracy >= 0.0f); CHECK(accuracy <= 1.0f); } From 22a9781ad51d7f447b52c0fd1d2f9623b336ecd6 Mon Sep 17 00:00:00 2001 From: lambda7xx Date: Mon, 21 Aug 2023 06:47:47 +0000 Subject: [PATCH 4/5] forma t --- lib/CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index f5d4c788af..46afb3a095 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -1,6 +1,6 @@ -add_subdirectory(pcg) -add_subdirectory(compiler) -# add_subdirectory(runtime) +#add_subdirectory(pcg) +#add_subdirectory(compiler) +add_subdirectory(runtime) add_subdirectory(op-attrs) add_subdirectory(kernels) add_subdirectory(utils) From b2d6bfa593d9c7a7f3a80c3de0e787d1ec9b2f59 Mon Sep 17 00:00:00 2001 From: lambda7xx Date: Wed, 20 Mar 2024 03:09:49 -0400 Subject: [PATCH 5/5] modify 992 --- cmake/cuda.cmake | 5 -- deps/any | 1 + deps/googletest | 1 + deps/invoke | 1 + deps/optional | 1 + deps/pybind11 | 1 + deps/variant | 1 + lib/CMakeLists.txt | 1 - lib/kernels/include/kernels/array_shape.h | 2 - lib/kernels/include/kernels/profiling.h | 2 +- lib/kernels/src/array_shape.cc | 32 +++++------- lib/kernels/test/CMakeLists.txt | 2 +- lib/kernels/test/src/test_array_shape.cc | 1 + .../test/src/test_datatype_dispatch.cc | 14 ++++++ lib/kernels/test/src/test_legion_dim.cc | 3 ++ lib/kernels/test/src/test_perf_metrics.cc | 49 ++++++++++++++++++- lib/kernels/test/src/test_profiling.cc | 49 ------------------- lib/op-attrs/include/op-attrs/dim_ordered.h | 2 - 18 files changed, 86 insertions(+), 82 deletions(-) create mode 160000 deps/any create mode 160000 deps/googletest create mode 160000 deps/invoke create mode 160000 deps/optional create mode 160000 deps/pybind11 create mode 160000 deps/variant delete mode 100644 lib/kernels/test/src/test_profiling.cc diff --git a/cmake/cuda.cmake b/cmake/cuda.cmake index 94c29e2274..26f72df448 100644 --- a/cmake/cuda.cmake +++ b/cmake/cuda.cmake @@ -83,11 +83,6 @@ if(CUDA_FOUND) add_library(cuda INTERFACE) target_include_directories(cuda SYSTEM INTERFACE "${CUDA_INCLUDE_DIRS}") - target_link_libraries(cuda INTERFACE - ${CUDADRV_LIBRARIES} - ${CUDA_LIBRARIES} - ${CUDA_CUBLAS_LIBRARIES} - ${CUDA_curand_LIBRARY}) else() message( FATAL_ERROR "CUDA package not found -> specify search path via CUDA_ROOT variable") diff --git a/deps/any b/deps/any new file mode 160000 index 0000000000..e88b1bfc16 --- /dev/null +++ b/deps/any @@ -0,0 +1 @@ +Subproject commit e88b1bfc160fa9b01e6174dd29c812eeeece3be9 diff --git a/deps/googletest b/deps/googletest new file mode 160000 index 0000000000..2fe3bd994b --- /dev/null +++ b/deps/googletest @@ -0,0 +1 @@ +Subproject commit 2fe3bd994b3189899d93f1d5a881e725e046fdc2 diff --git a/deps/invoke b/deps/invoke new file mode 160000 index 0000000000..2c1eabc2e2 --- /dev/null +++ b/deps/invoke @@ -0,0 +1 @@ +Subproject commit 2c1eabc2e20ab02961f95c704ff0c0818671ddd1 diff --git a/deps/optional b/deps/optional new file mode 160000 index 0000000000..c28fcf74d2 --- /dev/null +++ b/deps/optional @@ -0,0 +1 @@ +Subproject commit c28fcf74d207fc667c4ed3dbae4c251ea551c8c1 diff --git a/deps/pybind11 b/deps/pybind11 new file mode 160000 index 0000000000..8de7772cc7 --- /dev/null +++ b/deps/pybind11 @@ -0,0 +1 @@ +Subproject commit 8de7772cc72daca8e947b79b83fea46214931604 diff --git a/deps/variant b/deps/variant new file mode 160000 index 0000000000..23cb94f027 --- /dev/null +++ b/deps/variant @@ -0,0 +1 @@ +Subproject commit 23cb94f027d4ef33bf48133acc2695c7e5c6f1e7 diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 62bd8a1668..f7c166f0dd 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -1,4 +1,3 @@ - add_subdirectory(pcg) add_subdirectory(compiler) add_subdirectory(runtime) diff --git a/lib/kernels/include/kernels/array_shape.h b/lib/kernels/include/kernels/array_shape.h index ec0f2ddfac..0a0124e13c 100644 --- a/lib/kernels/include/kernels/array_shape.h +++ b/lib/kernels/include/kernels/array_shape.h @@ -37,8 +37,6 @@ struct ArrayShape { std::optional at_maybe(std::size_t) const; - bool operator==(ArrayShape const &other) const; // for test case - ArrayShape reversed_dim_order() const; ArrayShape sub_shape(std::optional start, std::optional end); diff --git a/lib/kernels/include/kernels/profiling.h b/lib/kernels/include/kernels/profiling.h index b1877402ea..210e35934f 100644 --- a/lib/kernels/include/kernels/profiling.h +++ b/lib/kernels/include/kernels/profiling.h @@ -14,7 +14,7 @@ struct ProfilingSettings { public: int warmup_iters; - int measure_iters; + req measure_iters; }; FF_VISITABLE_STRUCT(ProfilingSettings, warmup_iters, measure_iters); diff --git a/lib/kernels/src/array_shape.cc b/lib/kernels/src/array_shape.cc index e756411523..cf51616641 100644 --- a/lib/kernels/src/array_shape.cc +++ b/lib/kernels/src/array_shape.cc @@ -7,15 +7,15 @@ ArrayShape::ArrayShape(size_t *_dims, size_t num_dims) : dims(_dims, _dims + num_dims) {} std::size_t ArrayShape::get_volume() const { - return num_elements(); + return this->num_elements(); } std::size_t ArrayShape::get_dim() const { - return num_dims(); + return this->num_dims(); } std::size_t ArrayShape::num_elements() const { - return product(this->dims); + return this->product(this->dims); } std::size_t ArrayShape::num_dims() const { @@ -23,51 +23,43 @@ std::size_t ArrayShape::num_dims() const { } std::size_t ArrayShape::operator[](legion_dim_t idx) const { - return dims.at(idx); + return this->dims.at(idx); } std::size_t ArrayShape::at(legion_dim_t idx) const { - return dims.at(idx); + return this->dims.at(idx); } legion_dim_t ArrayShape::last_idx() const { - return legion_dim_t(dims.size() - 1); + return legion_dim_t(this->dims.size() - 1); } legion_dim_t ArrayShape::neg_idx(int idx) const { assert(idx < 0 && "Idx should be negative for negative indexing"); - return legion_dim_t(dims.size() + idx); + return legion_dim_t(this->dims.size() + idx); } optional ArrayShape::at_maybe(std::size_t idx) const { - if (idx < dims.size()) { - return dims[legion_dim_t(idx)]; + if (idx < this->dims.size()) { + return this->dims[legion_dim_t(idx)]; } else { return {}; } } ArrayShape ArrayShape::reversed_dim_order() const { - std::vector dims_reversed(dims.rbegin(), dims.rend()); + std::vector dims_reversed(this->dims.rbegin(), this->dims.rend()); return ArrayShape(dims_reversed); } ArrayShape ArrayShape::sub_shape(optional start, optional end) { size_t s = start.has_value() ? start.value().value() : 0; - size_t e = end.has_value() ? end.value().value() : dims.size(); - std::vector sub_dims(dims.begin() + s, dims.begin() + e); + size_t e = end.has_value() ? end.value().value() : this->dims.size(); + std::vector sub_dims(this->dims.begin() + s, this->dims.begin() + e); return ArrayShape(sub_dims); } -bool ArrayShape::operator==(ArrayShape const &other) const { - if (this->dims.size() != other.dims.size()) { - return false; - } - - return this->dims == other.dims; -} - size_t get_volume(ArrayShape const &shape) { return shape.get_volume(); } diff --git a/lib/kernels/test/CMakeLists.txt b/lib/kernels/test/CMakeLists.txt index 3c56a7139b..b6085b2ebd 100644 --- a/lib/kernels/test/CMakeLists.txt +++ b/lib/kernels/test/CMakeLists.txt @@ -15,7 +15,7 @@ target_link_libraries( kernels cuda rapidcheck - doctest::doctest) + octest) set_target_properties( ${project_target} diff --git a/lib/kernels/test/src/test_array_shape.cc b/lib/kernels/test/src/test_array_shape.cc index 676c2df9bd..dc15e13f0d 100644 --- a/lib/kernels/test/src/test_array_shape.cc +++ b/lib/kernels/test/src/test_array_shape.cc @@ -13,6 +13,7 @@ TEST_CASE("ArrayShape Initialization and Basic Functions") { CHECK(shape.num_dims() == 3); CHECK(shape[legion_dim_t(1)] == 3); CHECK(shape.at(legion_dim_t(2)) == 4); + CHECK(shape.at(ff_dim_t(2)) == 4); } TEST_CASE("Negative Indices and Optional Indexing") { diff --git a/lib/kernels/test/src/test_datatype_dispatch.cc b/lib/kernels/test/src/test_datatype_dispatch.cc index 9f4e81a68a..a60617e8e7 100644 --- a/lib/kernels/test/src/test_datatype_dispatch.cc +++ b/lib/kernels/test/src/test_datatype_dispatch.cc @@ -29,3 +29,17 @@ TEST_CASE("Testing DataTypeDispatch1") { int result = dispatcher(DataType::FLOAT, value); CHECK(result == 11); } + +TEST_CASE("Testing dispatch function double") { + int value = 10; + int result = dispatch(DataType::DOUBLE, value); + CHECK(result == 11); +} + + +TEST_CASE("Testing DataTypeDispatch1 Double") { + DataTypeDispatch1 dispatcher; + int value = 10; + int result = dispatcher(DataType::DOUBLE, value); + CHECK(result == 11); +} \ No newline at end of file diff --git a/lib/kernels/test/src/test_legion_dim.cc b/lib/kernels/test/src/test_legion_dim.cc index e4048b04bf..1b8aa6f3b6 100644 --- a/lib/kernels/test/src/test_legion_dim.cc +++ b/lib/kernels/test/src/test_legion_dim.cc @@ -17,6 +17,9 @@ TEST_CASE("Testing DimOrdered") { CHECK(dimOrder[legion_dim_t(0)] == 1); CHECK(dimOrder[legion_dim_t(1)] == 2); CHECK(dimOrder[legion_dim_t(2)] == 3); + CHECK(dimOrder[ff_dim_t(0)] == 1); + CHECK(dimOrder[ff_dim_t(1)] == 2); + CHECK(dimOrder[ff_dim_t(2)] == 3); } SUBCASE("comparsion") { diff --git a/lib/kernels/test/src/test_perf_metrics.cc b/lib/kernels/test/src/test_perf_metrics.cc index c60174b349..f6f3d41c08 100644 --- a/lib/kernels/test/src/test_perf_metrics.cc +++ b/lib/kernels/test/src/test_perf_metrics.cc @@ -25,7 +25,7 @@ PerfMetrics randomPerfMetrics() { return metrics; } -TEST_CASE("PerfMetricsTests") { +TEST_CASE("PerfMetricsTests1 " ) { SUBCASE("Throughput non-negative") { auto m = randomPerfMetrics(); @@ -47,3 +47,50 @@ TEST_CASE("PerfMetricsTests") { // Add other assertions for other fields... } } + + +TEST_CASE("PerfMetrics Tests") { + + SUBCASE("Constructor and basic properties") { + double start_time = 100.0; + PerfMetrics metrics(start_time); + + CHECK(metrics.start_time == start_time); + CHECK(metrics.current_time == start_time); + CHECK(metrics.train_all == 0); + } + + SUBCASE("Throughput non-negative") { + PerfMetrics m(10, 5, 0.5, 0.5, 0.5, 0.5, 0.5, 100.0, 200.0); + CHECK(get_throughput(m) >= 0); + } + + SUBCASE("Accuracy between 0 and 1") { + PerfMetrics m(10, 5, 0.5, 0.5, 0.5, 0.5, 0.5, 100.0, 200.0); + float accuracy = get_accuracy(m); + CHECK(accuracy >= 0.0f); + CHECK(accuracy <= 1.0f); + } + + SUBCASE("Update maintains non-negative values") { + PerfMetrics lhs(10, 5, 0.5, 0.5, 0.5, 0.5, 0.5, 100.0, 200.0); + PerfMetrics rhs(5, 3, 0.2, 0.2, 0.2, 0.2, 0.2, 200.0, 300.0); + + auto result = update(lhs, rhs); + CHECK(result.train_all == 15); + CHECK(result.train_correct.value() == 8); + } + + SUBCASE("Scale values correctly") { + PerfMetrics pm(10, 5, 0.5, 0.5, 0.5, 0.5, 0.5, 100.0, 200.0); + float scale = 2.0f; + + auto result = apply_scale(pm, scale); + CHECK(result.cce_loss.value() == doctest::Approx(1.0f)); + CHECK(result.sparse_cce_loss.value() == doctest::Approx(1.0f)); + CHECK(result.mse_loss.value() == doctest::Approx(1.0f)); + CHECK(result.rmse_loss.value() == doctest::Approx(1.0f)); + CHECK(result.mae_loss.value() == doctest::Approx(1.0f)); + } +} + diff --git a/lib/kernels/test/src/test_profiling.cc b/lib/kernels/test/src/test_profiling.cc deleted file mode 100644 index 5377212e38..0000000000 --- a/lib/kernels/test/src/test_profiling.cc +++ /dev/null @@ -1,49 +0,0 @@ -#include "doctest.h" -#include "kernels/perf_metrics.h" - -using namespace FlexFlow; - -TEST_CASE("PerfMetrics Tests") { - - SUBCASE("Constructor and basic properties") { - double start_time = 100.0; - PerfMetrics metrics(start_time); - - CHECK(metrics.start_time == start_time); - CHECK(metrics.current_time == start_time); - CHECK(metrics.train_all == 0); - } - - SUBCASE("Throughput non-negative") { - PerfMetrics m(10, 5, 0.5, 0.5, 0.5, 0.5, 0.5, 100.0, 200.0); - CHECK(get_throughput(m) >= 0); - } - - SUBCASE("Accuracy between 0 and 1") { - PerfMetrics m(10, 5, 0.5, 0.5, 0.5, 0.5, 0.5, 100.0, 200.0); - float accuracy = get_accuracy(m); - CHECK(accuracy >= 0.0f); - CHECK(accuracy <= 1.0f); - } - - SUBCASE("Update maintains non-negative values") { - PerfMetrics lhs(10, 5, 0.5, 0.5, 0.5, 0.5, 0.5, 100.0, 200.0); - PerfMetrics rhs(5, 3, 0.2, 0.2, 0.2, 0.2, 0.2, 200.0, 300.0); - - auto result = update(lhs, rhs); - CHECK(result.train_all == 15); - CHECK(result.train_correct.value() == 8); - } - - SUBCASE("Scale values correctly") { - PerfMetrics pm(10, 5, 0.5, 0.5, 0.5, 0.5, 0.5, 100.0, 200.0); - float scale = 2.0f; - - auto result = apply_scale(pm, scale); - CHECK(result.cce_loss.value() == doctest::Approx(1.0f)); - CHECK(result.sparse_cce_loss.value() == doctest::Approx(1.0f)); - CHECK(result.mse_loss.value() == doctest::Approx(1.0f)); - CHECK(result.rmse_loss.value() == doctest::Approx(1.0f)); - CHECK(result.mae_loss.value() == doctest::Approx(1.0f)); - } -} diff --git a/lib/op-attrs/include/op-attrs/dim_ordered.h b/lib/op-attrs/include/op-attrs/dim_ordered.h index fd97359a88..b726d0687f 100644 --- a/lib/op-attrs/include/op-attrs/dim_ordered.h +++ b/lib/op-attrs/include/op-attrs/dim_ordered.h @@ -4,8 +4,6 @@ #include "op-attrs/ff_dim.h" #include "utils/stack_vector.h" -#define MAX_TENSOR_DIM 5 - namespace FlexFlow { template