Refactor and simplify logic in sdksCommon cmake script to not rely on…

… c2j model (#3251)
aws · Jan 22, 2025 · 3e72626 · 3e72626
1 parent 1676e3d
commit 3e72626
Showing 2 changed files with 36 additions and 109 deletions.
diff --git a/cmake/sdks.cmake b/cmake/sdks.cmake
@@ -214,7 +214,6 @@ function(add_sdks)
         endif()
 
         add_subdirectory("${SDK_DIR}")
-        message(STATUS "exporting ${SDK_TARGET}")
         LIST(APPEND EXPORTS "${SDK_TARGET}")
         unset(SDK_TARGET)
     endforeach()

diff --git a/cmake/sdksCommon.cmake b/cmake/sdksCommon.cmake
@@ -1,22 +1,11 @@
 # Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0.
-#
 
-# helper function that that gives primitive map functionality by treating a colon as the key-value separator, while the list semi-colon separates pairs
-# to use, pass the list-map in as a third parameter (see helper functions below)
-function(get_map_element KEY VALUE_VAR)
-    foreach(ELEMENT_PAIR ${ARGN})
-        STRING(REGEX REPLACE "([^:]+):.*" "\\1" KEY_RESULT ${ELEMENT_PAIR})
-        if(${KEY_RESULT} STREQUAL ${KEY} )
-            STRING(REGEX REPLACE "[^:]+:(.*)" "\\1" VALUE_RESULT ${ELEMENT_PAIR})
-            set(${VALUE_VAR} "${VALUE_RESULT}" PARENT_SCOPE)
-            return()
-        endif()
-    endforeach()
-    set(${VALUE_VAR} "" PARENT_SCOPE)
-endfunction(get_map_element)
-
-function(get_multimap_element KEY VALUE_VAR)
+# Return all values for a key from a flatmap
+# I.e. given a flatmap list of items such as "key1:val1,val2,val3;key2:val2;key1:valDoNotLoseMe" and "key1" as arguments
+# when called such as "get_flatmap_value(${TEST_NAME} TEMP_VAR ${TEST_DEPENDENCY_LIST})"
+# will return "val1,val2,val3,valDoNotLoseMe"
+function(get_flatmap_value KEY VALUE_VAR)
     set(${VALUE_VAR} "" PARENT_SCOPE)
     set(RETURN_LIST "")
     foreach(ELEMENT_PAIR ${ARGN})
@@ -27,39 +16,30 @@ function(get_multimap_element KEY VALUE_VAR)
         endif()
     endforeach()
     set(${VALUE_VAR} "${RETURN_LIST}" PARENT_SCOPE)
-endfunction(get_multimap_element)
-
-# a bunch of key-value retrieval functions for the list-maps we defined above
-function(get_c2j_date_for_service SERVICE_NAME C2J_DATE_VAR)
-    get_map_element(${SERVICE_NAME} TEMP_VAR ${C2J_LIST})
-    set(${C2J_DATE_VAR} "${TEMP_VAR}" PARENT_SCOPE)
-endfunction()
-
-function(get_c2j_name_for_service SERVICE_NAME C2J_NAME_VAR)
-    get_map_element(${SERVICE_NAME} TEMP_VAR ${C2J_SPECIAL_NAME_LIST})
-    if(TEMP_VAR)
-        set(${C2J_NAME_VAR} "${TEMP_VAR}" PARENT_SCOPE)
-    else()
-        set(${C2J_NAME_VAR} "${SERVICE_NAME}" PARENT_SCOPE)
-    endif()
-endfunction()
+endfunction(get_flatmap_value)
 
 function(get_test_projects_for_service SERVICE_NAME TEST_PROJECT_NAME_VAR)
-    get_multimap_element(${SERVICE_NAME} TEMP_VAR ${SDK_TEST_PROJECT_LIST})
+    get_flatmap_value(${SERVICE_NAME} TEMP_VAR ${SDK_TEST_PROJECT_LIST})
     set(${TEST_PROJECT_NAME_VAR} "${TEMP_VAR}" PARENT_SCOPE)
 endfunction()
 
 function(get_dependencies_for_sdk PROJECT_NAME DEPENDENCY_LIST_VAR)
-    get_map_element(${PROJECT_NAME} TEMP_VAR ${SDK_DEPENDENCY_LIST})
+    get_flatmap_value(${PROJECT_NAME} TEMP_VAR ${SDK_DEPENDENCY_LIST})
+    if (TEMP_VAR)
+        string(REPLACE "," ";" TEMP_LIST_VAR ${TEMP_VAR})
+        list(FIND TEMP_LIST_VAR "core" _index)
+        if (${_index} GREATER -1) # old cmake search in a list syntax
+            # core is already there
+            set(${DEPENDENCY_LIST_VAR} "${TEMP_VAR}" PARENT_SCOPE)
+            return()
+        endif()
+    endif()
     # "core" is the default dependency for every sdk.
-    # Since we removed the hand-written C2J_LIST and instead auto generating it based on models,
-    # and location of models may not exist or incorrect when SDK is installed and then source has been deleted by customers.
-    # we end up getting an incomplete C2J_LIST when customers call find_package(AWSSDK). But C2J_LIST is only used in customers code for dependencies completing.
     set(${DEPENDENCY_LIST_VAR} "${TEMP_VAR},core" PARENT_SCOPE)
 endfunction()
 
 function(get_dependencies_for_test TEST_NAME DEPENDENCY_LIST_VAR)
-    get_map_element(${TEST_NAME} TEMP_VAR ${TEST_DEPENDENCY_LIST})
+    get_flatmap_value(${TEST_NAME} TEMP_VAR ${TEST_DEPENDENCY_LIST})
     set(${DEPENDENCY_LIST_VAR} "${TEMP_VAR}" PARENT_SCOPE)
 endfunction()
 
@@ -80,69 +60,21 @@ function(get_sdks_depending_on SDK_NAME DEPENDING_SDKS_VAR)
     set(${DEPENDING_SDKS_VAR} "${TEMP_SDK_LIST}" PARENT_SCOPE)
 endfunction()
 
-# function that automatically picks up models from <sdkrootdir>/code-generation/api-descriptions/ directory and build
-# C2J_LIST needed for generation, services have multiple models will use the latest model (decided by model files' date)
-# services have the name format abc.def.ghi will be renamed to ghi-def-abc (dot will not be accepted as Windows directory name )
-# and put into C2J_SPECIAL_NAME_LIST, but rumtime.lex will be renamed to lex based on historical reason.
+# Function that reads <repo>/src/generated and builds a list of all available generated service clients for a build,
+# The resulting $SERVICE_CLIENT_LIST is as simple as "cognito-identity,s3,dynamodb,etc,..."
 function(build_sdk_list)
-    file(GLOB ALL_MODEL_FILES "${CMAKE_CURRENT_SOURCE_DIR}/tools/code-generation/api-descriptions/*-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9].normal.json")
-    foreach(model IN LISTS ALL_MODEL_FILES)
-        get_filename_component(modelName "${model}" NAME)
-        STRING(REGEX MATCH "([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9])" date "${modelName}")
-        STRING(REGEX REPLACE "-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9].normal.json" "" svc "${modelName}")
-        #special svc name conversion, e.g: runtime.lex->lex; abc.def.ghi->ghi-def-abc
-        if ("${svc}" STREQUAL "runtime.lex")
-            LIST(APPEND C2J_SPECIAL_NAME_LIST "lex:runtime.lex")
-            set(svc "lex")
-        elseif("${svc}" STREQUAL "runtime.lex.v2")
-            LIST(APPEND C2J_SPECIAL_NAME_LIST "lexv2-runtime:runtime.lex.v2")
-            set(svc "lexv2-runtime")
-        elseif("${svc}" STREQUAL "models.lex.v2")
-            LIST(APPEND C2J_SPECIAL_NAME_LIST "lexv2-models:models.lex.v2")
-            set(svc "lexv2-models")
-        elseif ("${svc}" STREQUAL "transfer")
-            LIST(APPEND C2J_SPECIAL_NAME_LIST "awstransfer:transfer")
-            set(svc "awstransfer")
-        elseif ("${svc}" STREQUAL "transcribe-streaming")
-            LIST(APPEND C2J_SPECIAL_NAME_LIST "transcribestreaming:transcribe-streaming")
-            set(svc "transcribestreaming")
-        elseif ("${svc}" STREQUAL "streams.dynamodb")
-            LIST(APPEND C2J_SPECIAL_NAME_LIST "dynamodbstreams:streams.dynamodb")
-            set (svc "dynamodbstreams")
-        elseif("${svc}" MATCHES "\\.")
-            string(REPLACE "." ";" nameParts ${svc})
-            LIST(REVERSE nameParts)
-            string(REPLACE ";" "-" tmpSvc "${nameParts}")
-            LIST(APPEND C2J_SPECIAL_NAME_LIST "${tmpSvc}:${svc}")
-            set(svc "${tmpSvc}")
-        elseif("${svc}" STREQUAL "ec2")
-            if(PLATFORM_ANDROID AND CMAKE_HOST_WIN32)  # ec2 isn't building for android on windows atm due to an internal compiler error, TODO: investigate further
-                continue()
-            endif()
-        elseif("${svc}" STREQUAL "s3")
-            LIST(APPEND C2J_SPECIAL_NAME_LIST "s3-crt:s3")
-            message(STATUS "Add s3-crt:s3 to C2J_SPECIAL_NAME_LIST")
-        endif()
-        get_map_element(${svc} existingDate ${C2J_LIST})
-        if ("${existingDate}" STREQUAL "")
-            LIST(APPEND C2J_LIST "${svc}:${date}")
-            if ("${svc}" STREQUAL "s3")
-                LIST(APPEND C2J_LIST "s3-crt:${date}")
-            endif()
-        elseif(${existingDate} STRLESS ${date})
-            LIST(REMOVE_ITEM C2J_LIST "${svc}:${existingDate}")
-            LIST(APPEND C2J_LIST "${svc}:${date}")
-            if ("${svc}" STREQUAL "s3")
-                LIST(REMOVE_ITEM C2J_LIST "s3-crt:${existingDate}")
-                LIST(APPEND C2J_LIST "s3-crt:${date}")
-            endif()
-        endif()
+    set(LIST_DIRECTORIES true)
+    file(GLOB ALL_GENERATED_CLIENTS $LIST_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/generated/src/aws-cpp-sdk-*")
+
+    foreach(clientDir IN LISTS ALL_GENERATED_CLIENTS)
+        get_filename_component(clientDirName "${clientDir}" NAME)
+        STRING(REGEX REPLACE "^aws-cpp-sdk-" "" serviceName "${clientDirName}")
+        LIST(APPEND SERVICE_CLIENT_LIST "${serviceName}")
     endforeach()
-    set(C2J_LIST "${C2J_LIST}" PARENT_SCOPE)
-    set(C2J_SPECIAL_NAME_LIST "${C2J_SPECIAL_NAME_LIST}" PARENT_SCOPE)
+    set(SERVICE_CLIENT_LIST "${SERVICE_CLIENT_LIST}" PARENT_SCOPE)
 endfunction()
 
-
+# A list of so-called "hand-written SDK high-level libraries/components"
 set(HIGH_LEVEL_SDK_LIST "")
 list(APPEND HIGH_LEVEL_SDK_LIST "access-management")
 list(APPEND HIGH_LEVEL_SDK_LIST "identity-management")
@@ -151,6 +83,7 @@ list(APPEND HIGH_LEVEL_SDK_LIST "transfer")
 list(APPEND HIGH_LEVEL_SDK_LIST "s3-encryption")
 list(APPEND HIGH_LEVEL_SDK_LIST "text-to-speech")
 
+# A flatmap list of sdk_component:sdk_component_tests,sdk_component_another_tests
 set(SDK_TEST_PROJECT_LIST "")
 list(APPEND SDK_TEST_PROJECT_LIST "cloudfront:tests/aws-cpp-sdk-cloudfront-integration-tests")
 list(APPEND SDK_TEST_PROJECT_LIST "cognito-identity:tests/aws-cpp-sdk-cognitoidentity-integration-tests")
@@ -184,11 +117,9 @@ build_sdk_list()
 
 if(EXISTS "${CMAKE_SOURCE_DIR}/generated")
     if(EXISTS "${CMAKE_SOURCE_DIR}/generated/tests")
-        foreach(GENERATED_C2J_TEST IN LISTS C2J_LIST)
-            STRING(REGEX REPLACE "([^:]+):.*" "\\1" GENERATED_C2J_TEST_RESULT ${GENERATED_C2J_TEST})
-
-            if(EXISTS "${CMAKE_SOURCE_DIR}/generated/tests/${GENERATED_C2J_TEST_RESULT}-gen-tests")
-                list(APPEND SDK_TEST_PROJECT_LIST "${GENERATED_C2J_TEST_RESULT}:generated/tests/${GENERATED_C2J_TEST_RESULT}-gen-tests")
+        foreach(SERVICE_NAME IN LISTS SERVICE_CLIENT_LIST)
+            if(EXISTS "${CMAKE_SOURCE_DIR}/generated/tests/${SERVICE_NAME}-gen-tests")
+                list(APPEND SDK_TEST_PROJECT_LIST "${SERVICE_NAME}:generated/tests/${SERVICE_NAME}-gen-tests")
             endif()
         endforeach()
     endif()
@@ -213,10 +144,7 @@ list(APPEND TEST_DEPENDENCY_LIST "text-to-speech:polly,core")
 list(APPEND TEST_DEPENDENCY_LIST "transfer:s3,core")
 list(APPEND TEST_DEPENDENCY_LIST "logs:access-management,cognito-identity,iam,core")
 
-# make a list of the generated clients
-set(GENERATED_SERVICE_LIST "")
-foreach(GENERATED_C2J_SERVICE IN LISTS C2J_LIST)
-    STRING(REGEX REPLACE "([^:]+):.*" "\\1" SERVICE_RESULT ${GENERATED_C2J_SERVICE})
-    list(APPEND GENERATED_SERVICE_LIST ${SERVICE_RESULT})
-    list(APPEND SDK_DEPENDENCY_LIST "${SERVICE_RESULT}:core")
+set(GENERATED_SERVICE_LIST ${SERVICE_CLIENT_LIST})
+foreach(SERVICE_NAME IN LISTS SERVICE_CLIENT_LIST)
+    list(APPEND SDK_DEPENDENCY_LIST "${SERVICE_NAME}:core")
 endforeach()