Samsung · chenyx113 · Jan 23, 2025
diff --git a/tools/onnx-subgraph/.Readme.md.swp b/tools/onnx-subgraph/.Readme.md.swp
diff --git a/tools/onnx-subgraph/CMakeLists.txt b/tools/onnx-subgraph/CMakeLists.txt
@@ -0,0 +1,81 @@
+# cmake version dependency
+cmake_minimum_required(VERSION 3.10)
+SET(CMAKE_BUILD_TYPE "Debug")
+SET(CMAKE_CXX_FLAGS_DEBUG "$ENV{CXXFLAGS} -O0 -Wall -g2 -ggdb")
+SET(CMAKE_CXX_FLAGS_RELEASE "$ENV{CXXFLAGS} -O3 -Wall")
+SET(CMAKE_CXX_STANDARD 17)
+
+project(onnx-subgraph-parser)
+
+find_package(Protobuf REQUIRED)
+set(PROTO_FILES onnx.proto)
+protobuf_generate_cpp(PROTO_SRCS PROTO_HDRS ${PROTO_FILES})
+
+
+include_directories(${CMAKE_CURRENT_BINARY_DIR})
+
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
+include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include/json)
+
+find_package(Python3 COMPONENTS Interpreter Development REQUIRED)
+include_directories(${Python3_INCLUDE_DIRS})
+
+
+file(GLOB SOURCES "src/lib/*.cpp" "src/lib/*.cpp" )
+
+add_library(onnx-subgraph-parser STATIC ${SOURCES} ${PROTO_SRCS} ${PROTO_FILES})
+target_link_libraries(onnx-subgraph-parser protobuf)
+
+
+add_executable(onnx-subgraph src/main.cpp)
+target_link_libraries(onnx-subgraph onnx-subgraph-parser ${Python3_LIBRARIES})
+
+
+ set(ONNX_SUGRAPH_FILES                                                          
+    extract_onnx_lib.py
+    extract_onnx.py
+    single_vs_multiple_onnx.py
+    quant.py
+    model_inference.py
+    model_inference_multiple_output.py
+    onnx_subgraph_ut.py
+    test_model_download.sh
+    config.json
+    config-sample-1.json
+    config-sample-2.json
+ )                                                                            
+
+ foreach(ONNX_SUGRAPH IN ITEMS ${ONNX_SUGRAPH_FILES})                               
+
+   set(ONNX_SUGRAPH_FILE ${ONNX_SUGRAPH})                                           
+   set(ONNX_SUGRAPH_SRC "${CMAKE_CURRENT_SOURCE_DIR}/${ONNX_SUGRAPH_FILE}")         
+   set(ONNX_SUGRAPH_BIN "${CMAKE_CURRENT_BINARY_DIR}/scripts/${ONNX_SUGRAPH_FILE}")         
+   set(ONNX_SUGRAPH_TARGET "${ONNX_SUGRAPH}_target")                                
+
+   add_custom_command(OUTPUT ${ONNX_SUGRAPH_BIN}                                 
+     COMMAND ${CMAKE_COMMAND} -E copy "${ONNX_SUGRAPH_SRC}" "${ONNX_SUGRAPH_BIN}"   
+     DEPENDS ${ONNX_SUGRAPH_SRC}                                                 
+     COMMENT "Generate ${ONNX_SUGRAPH_BIN}"                                      
+   )                                                                          
+
+   add_custom_target(${ONNX_SUGRAPH_TARGET} ALL DEPENDS ${ONNX_SUGRAPH_BIN})        
+
+   install(FILES ${ONNX_SUGRAPH_BIN}                                             
+           PERMISSIONS OWNER_WRITE OWNER_READ OWNER_EXECUTE                   
+                       GROUP_READ GROUP_EXECUTE                               
+                       WORLD_READ WORLD_EXECUTE                               
+           DESTINATION bin)                                                   
+
+ endforeach(ONNX_SUGRAPH)     
+
+set(Python_ADDITIONAL_VERSIONS 3.8)
+find_package(PythonInterp 3.8 QUIET)
+if(NOT ${PYTHONINTERP_FOUND}) 
+	message(FATAL_ERROR "Build onnx-subgraph_unittest: FAIL (Python3 is missing)")
+endif() 
+
+  add_test(
+ 		 NAME onnx-subgraph_unittest
+ 		 COMMAND ${PYTHON_EXECUTABLE} -m unittest
+ 		 WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+  )
diff --git a/tools/onnx-subgraph/Readme.md b/tools/onnx-subgraph/Readme.md
@@ -0,0 +1,85 @@
+# onnx_autosubgraph
+onnx-subgraph tool provides  model auto partitionioning of onnx model to several sub models by operator, performance and model size limitations, 
+with the order and input / output names of sub models
+
+# How to build the onnx-subgraph
+## OS environment dependence
+     1. ubuntu >=20.04
+     2. GCC >= 9.4.0
+     3. cmake >= 3.10
+     4. python >= 3.8
+     5. apt-get install libprotobuf-dev protobuf-compiler
+
+## Python packages dependence
+    onnx                         1.16.0
+    onnxruntime                  1.18.1
+    onnxsim                      0.4.36
+    torch                        2.3.1
+    scikit-image
+    scikit-learn
+    pandas
+    tqdm
+
+## building the onnx-subgraph
+    1. cd onnx-subgraph
+    2. mkdir build & cd build
+    3. cmake .. & make
+    4. we can get following output at ./build
+          ├── onnx-subgraph
+          └── scripts
+              ├── config.json
+              ├── config-sample-1.json
+              ├── config-sample-2.json
+              ├── extract_onnx_lib.py
+              ├── extract_onnx.py
+              ├── model_inference_multiple_output.py
+              ├── model_inference.py
+              ├── onnx_subgraph_ut.py
+              ├── quant.py
+              ├── single_vs_multiple_onnx.py
+              └── test_model_download.sh
+# How to use the onnx-subgraph
+## Pre-steps
+### Download the test AI models
+    1. bash scripts/test_model_download.sh, then "resnet-test.onnx" will be got in ./build
+    2. you can change to any other onnx files as your needs, or edit the download link in "scripts/test_model_download.sh"
+### Prepare the config.json
+    1. edit the config.json
+       . you can edit operators in "NPU_supported_ops" and "CPU_supported_ops";
+       . you can edit performance data in "performance_data" as the real HW status, 
+       . you can edit "max_subgraph_size" in case of "NPU_supported_ops" is []
+    2. you can also check more examples in "config-sample-1.json" and "config-sample-2.json"
+
+
+## Parse the onnx model
+     ./onnx-subgraph --onnx=resnet-test.onnx
+       after parsing done, subgraphs_ios.txt will be generated at current path
+
+## Split the onnx model to subgraphs
+    1. edit the config path and model file path at ./scripts/extract_onnx.py 
+       e.g.: extract_onnx_lib.split_onnx_ios('./subgraphs_ios.txt','./resnet-test.onnx') 
+    2. python scripts/extract_onnx.py, after extraction done, the subgraphs will be saved at './subgraphs'
+       subgraphs
+       ├── CPU
+       │   ├── CPUsubgraph0.onnx
+       │   └── CPUsubgraph1.onnx
+       └── NPU
+           ├── NPUsubgraph0.onnx
+           └── NPUsubgraph1.onnx
+
+### Verify the subgraphs inference with original model file
+    1. edit the model path, subgraph path and config path in ./scripts/single_vs_multiple_onnx.py
+             single_onnx_model_path = './resnet-test.onnx'
+             model_path = './subgraphs/'
+             subgraphsiostxt_path = './subgraphs_ios.txt'
+    2. edit the input shape and name of onnx model in ./scripts/single_vs_multiple_onnx.py
+             default_input_data = {
+                 "x": np.random.rand(1, 3, 256, 256).astype(np.float32),
+             }
+    3. compare the MSE of original inference result and subgraphs inference result
+       python ./scripts/single_vs_multiple_onnx.py
+       output:
+            Single model inference completed!
+            Multiple subgraph inference completed!
+            Comparing inference results between single ONNX model and multiple subgraphs...
+            Output '316' MSE: 5.125894080395578e-14
diff --git a/tools/onnx-subgraph/config-sample-1.json b/tools/onnx-subgraph/config-sample-1.json
@@ -0,0 +1,10 @@
+{
+    "NPU_supported_ops": [],
+    "CPU_supported_ops": ["Sub", "Pow", "ReduceMean", "Add", "Sqrt", "Div","Transpose", "Gather", "MatMul", "Mul", "Softmax", "Erf", "Gemm", "Conv", "Reshape",
+    "Sin", "Where", "ConstantOfShape", "Cast", "Sigmoid", "Cos", "Expand", "Slice", "Unsqueeze"],
+    "performance_data": [],
+    "hardware_limits": {
+        "max_subgraph_size": 10240.0,
+        "max_subgraphs": 5
+    }
+}
diff --git a/tools/onnx-subgraph/config-sample-2.json b/tools/onnx-subgraph/config-sample-2.json
@@ -0,0 +1,15 @@
+{
+    "NPU_supported_ops": ["Conv", "Reshape", "Transpose", "Add", "ReduceMean", "Sub", "Div", "Mul", "Sigmoid","MatMul"],
+    "CPU_supported_ops": ["Sub", "Pow", "ReduceMean", "Add", "Sqrt", "Div","Transpose", "Gather", "MatMul", "Mul", "Softmax", "Erf", "Gemm", "Conv", "Reshape",
+    "Sin", "Where", "ConstantOfShape", "Cast", "Sigmoid", "Cos", "Expand", "Slice", "Unsqueeze"],
+    "performance_data": [
+        {"name":"Conv","CPU_time": 0.1, "NPU_time": 0.05},
+        {"name":"Mul", "CPU_time": 0.15, "NPU_time": 0.07}
+        {"name":"Add", "CPU_time": 0.15, "NPU_time": 0.07}
+        {"name":"Sub", "CPU_time": 0.15, "NPU_time": 0.07}
+    ],
+    "hardware_limits": {
+        "max_subgraph_size": 60024.0,
+        "max_subgraphs": 5
+    }
+}
diff --git a/tools/onnx-subgraph/config.json b/tools/onnx-subgraph/config.json
@@ -0,0 +1,13 @@
+{
+    "NPU_supported_ops": ["Conv", "Reshape", "Transpose", "Add", "ReduceMean", "Sub", "Div", "Mul", "Sigmoid","MatMul"],
+    "CPU_supported_ops": ["Sub", "Pow", "ReduceMean", "Add", "Sqrt", "Div","Transpose", "Gather", "MatMul", "Mul", "Softmax", "Erf", "Gemm", "Conv", "Reshape",
+    "Sin", "Where", "ConstantOfShape", "Cast", "Sigmoid", "Cos", "Expand", "Slice", "Unsqueeze"],
+    "performance_data": [
+        {"name":"Conv","CPU_time": 0.1, "NPU_time": 0.05},
+		{"name":"Mul", "CPU_time": 0.15, "NPU_time": 0.07}
+    ],
+    "hardware_limits": {
+        "max_subgraph_size": 60024.0,
+        "max_subgraphs": 5
+    }
+}
diff --git a/tools/onnx-subgraph/extract_onnx.py b/tools/onnx-subgraph/extract_onnx.py
@@ -0,0 +1,21 @@
+# Copyright (c) 2024 Samsung Electronics Co., Ltd. All Rights Reserved
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import extract_onnx_lib
+import torch
+import onnx
+import re
+print("python executed")
+extract_onnx_lib.split_onnx_ios('./subgraphs_ios.txt','./resnet-test.onnx')
+