Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactor and simplify logic in sdksCommon cmake script #3251

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
+36 −109
Open

Refactor and simplify logic in sdksCommon cmake script #3251

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion cmake/sdks.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -214,7 +214,6 @@ function(add_sdks)
endif()

add_subdirectory("${SDK_DIR}")
message(STATUS "exporting ${SDK_TARGET}")
LIST(APPEND EXPORTS "${SDK_TARGET}")
unset(SDK_TARGET)
endforeach()
Expand Down
144 changes: 36 additions & 108 deletions cmake/sdksCommon.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,22 +1,11 @@
# Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
# SPDX-License-Identifier: Apache-2.0.
#

# helper function that that gives primitive map functionality by treating a colon as the key-value separator, while the list semi-colon separates pairs
# to use, pass the list-map in as a third parameter (see helper functions below)
function(get_map_element KEY VALUE_VAR)
foreach(ELEMENT_PAIR ${ARGN})
STRING(REGEX REPLACE "([^:]+):.*" "\\1" KEY_RESULT ${ELEMENT_PAIR})
if(${KEY_RESULT} STREQUAL ${KEY} )
STRING(REGEX REPLACE "[^:]+:(.*)" "\\1" VALUE_RESULT ${ELEMENT_PAIR})
set(${VALUE_VAR} "${VALUE_RESULT}" PARENT_SCOPE)
return()
endif()
endforeach()
set(${VALUE_VAR} "" PARENT_SCOPE)
endfunction(get_map_element)

function(get_multimap_element KEY VALUE_VAR)
# Return all values for a key from a flatmap
# I.e. given a flatmap list of items such as "key1:val1,val2,val3;key2:val2;key1:valDoNotLoseMe" and "key1" as arguments
# when called such as "get_flatmap_value(${TEST_NAME} TEMP_VAR ${TEST_DEPENDENCY_LIST})"
# will return "val1,val2,val3,valDoNotLoseMe"
function(get_flatmap_value KEY VALUE_VAR)
set(${VALUE_VAR} "" PARENT_SCOPE)
set(RETURN_LIST "")
foreach(ELEMENT_PAIR ${ARGN})
Expand All @@ -27,39 +16,30 @@ function(get_multimap_element KEY VALUE_VAR)
endif()
endforeach()
set(${VALUE_VAR} "${RETURN_LIST}" PARENT_SCOPE)
endfunction(get_multimap_element)

# a bunch of key-value retrieval functions for the list-maps we defined above
function(get_c2j_date_for_service SERVICE_NAME C2J_DATE_VAR)
get_map_element(${SERVICE_NAME} TEMP_VAR ${C2J_LIST})
set(${C2J_DATE_VAR} "${TEMP_VAR}" PARENT_SCOPE)
endfunction()

function(get_c2j_name_for_service SERVICE_NAME C2J_NAME_VAR)
get_map_element(${SERVICE_NAME} TEMP_VAR ${C2J_SPECIAL_NAME_LIST})
if(TEMP_VAR)
set(${C2J_NAME_VAR} "${TEMP_VAR}" PARENT_SCOPE)
else()
set(${C2J_NAME_VAR} "${SERVICE_NAME}" PARENT_SCOPE)
endif()
endfunction()
endfunction(get_flatmap_value)

function(get_test_projects_for_service SERVICE_NAME TEST_PROJECT_NAME_VAR)
get_multimap_element(${SERVICE_NAME} TEMP_VAR ${SDK_TEST_PROJECT_LIST})
get_flatmap_value(${SERVICE_NAME} TEMP_VAR ${SDK_TEST_PROJECT_LIST})
set(${TEST_PROJECT_NAME_VAR} "${TEMP_VAR}" PARENT_SCOPE)
endfunction()

function(get_dependencies_for_sdk PROJECT_NAME DEPENDENCY_LIST_VAR)
get_map_element(${PROJECT_NAME} TEMP_VAR ${SDK_DEPENDENCY_LIST})
get_flatmap_value(${PROJECT_NAME} TEMP_VAR ${SDK_DEPENDENCY_LIST})
if (TEMP_VAR)
string(REPLACE "," ";" TEMP_LIST_VAR ${TEMP_VAR})
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

one thing I see is because get_flatmap_value returns a cmake list, it already separates by semicolon, so we do not need this line I think. It doesn't hurt but will simplify even more on removal

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I saw this and even tried to get rid of this replacements as well, but
while it is indeed tries to return a semicolon-separated cmake list,
thanks to some cmake deep "magic", it is returned as a comma-separated list, and then it starts to fail in other places of the project in very strange ways (like when both comma and semicolon just disappear and the whole list is concatenated into a single word).

We can address it later (if it is even possible to fix)

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Interesting. Thanks for letting me know

list(FIND TEMP_LIST_VAR "core" _index)
if (${_index} GREATER -1) # old cmake search in a list syntax
# core is already there
set(${DEPENDENCY_LIST_VAR} "${TEMP_VAR}" PARENT_SCOPE)
return()
endif()
endif()
# "core" is the default dependency for every sdk.
# Since we removed the hand-written C2J_LIST and instead auto generating it based on models,
# and location of models may not exist or incorrect when SDK is installed and then source has been deleted by customers.
# we end up getting an incomplete C2J_LIST when customers call find_package(AWSSDK). But C2J_LIST is only used in customers code for dependencies completing.
set(${DEPENDENCY_LIST_VAR} "${TEMP_VAR},core" PARENT_SCOPE)
endfunction()

function(get_dependencies_for_test TEST_NAME DEPENDENCY_LIST_VAR)
get_map_element(${TEST_NAME} TEMP_VAR ${TEST_DEPENDENCY_LIST})
get_flatmap_value(${TEST_NAME} TEMP_VAR ${TEST_DEPENDENCY_LIST})
set(${DEPENDENCY_LIST_VAR} "${TEMP_VAR}" PARENT_SCOPE)
endfunction()

Expand All @@ -80,69 +60,21 @@ function(get_sdks_depending_on SDK_NAME DEPENDING_SDKS_VAR)
set(${DEPENDING_SDKS_VAR} "${TEMP_SDK_LIST}" PARENT_SCOPE)
endfunction()

# function that automatically picks up models from <sdkrootdir>/code-generation/api-descriptions/ directory and build
# C2J_LIST needed for generation, services have multiple models will use the latest model (decided by model files' date)
# services have the name format abc.def.ghi will be renamed to ghi-def-abc (dot will not be accepted as Windows directory name )
# and put into C2J_SPECIAL_NAME_LIST, but rumtime.lex will be renamed to lex based on historical reason.
# Function that reads <repo>/src/generated and builds a list of all available generated service clients for a build,
# The resulting $SERVICE_CLIENT_LIST is as simple as "cognito-identity,s3,dynamodb,etc,..."
function(build_sdk_list)
file(GLOB ALL_MODEL_FILES "${CMAKE_CURRENT_SOURCE_DIR}/tools/code-generation/api-descriptions/*-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9].normal.json")
foreach(model IN LISTS ALL_MODEL_FILES)
get_filename_component(modelName "${model}" NAME)
STRING(REGEX MATCH "([0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9])" date "${modelName}")
STRING(REGEX REPLACE "-[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9].normal.json" "" svc "${modelName}")
#special svc name conversion, e.g: runtime.lex->lex; abc.def.ghi->ghi-def-abc
if ("${svc}" STREQUAL "runtime.lex")
LIST(APPEND C2J_SPECIAL_NAME_LIST "lex:runtime.lex")
set(svc "lex")
elseif("${svc}" STREQUAL "runtime.lex.v2")
LIST(APPEND C2J_SPECIAL_NAME_LIST "lexv2-runtime:runtime.lex.v2")
set(svc "lexv2-runtime")
elseif("${svc}" STREQUAL "models.lex.v2")
LIST(APPEND C2J_SPECIAL_NAME_LIST "lexv2-models:models.lex.v2")
set(svc "lexv2-models")
elseif ("${svc}" STREQUAL "transfer")
LIST(APPEND C2J_SPECIAL_NAME_LIST "awstransfer:transfer")
set(svc "awstransfer")
elseif ("${svc}" STREQUAL "transcribe-streaming")
LIST(APPEND C2J_SPECIAL_NAME_LIST "transcribestreaming:transcribe-streaming")
set(svc "transcribestreaming")
elseif ("${svc}" STREQUAL "streams.dynamodb")
LIST(APPEND C2J_SPECIAL_NAME_LIST "dynamodbstreams:streams.dynamodb")
set (svc "dynamodbstreams")
elseif("${svc}" MATCHES "\\.")
string(REPLACE "." ";" nameParts ${svc})
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These special name mapping is also captured in https://github.com/aws/aws-sdk-cpp/blob/main/tools/code-generation/smithy/codegen/smithy2c2j_service_map.json
Please just ensure the original naming is retained otherwise smoke tests wont build because they are built using the cmake library name which mimics the folder name
example: https://github.com/aws/aws-sdk-cpp/blob/main/generated/smoke-tests/apigateway/CMakeLists.txt#L4

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For reference, here is how the cmake is generated:

aws-sdk-cpp/tools/code-generation/smithy/codegen/cpp-smoke-tests-codegen/src/main/java/com/amazonaws/util/awsclientsmithygenerator/generators/SmokeTestsCMakeWriter.java

Line 27 in 94627af

"aws-cpp-sdk-" + folderNamespace + "\n" +

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

please read the PR description first, I've actually documented this:

  1. c2j file renaming logic is extracted and duplicated within the code generation script run_code_generation.py, so additional logic of datetime extraction, c2j filename renaming and etc is not actually required in the cmake script.

I've run an additional test today, just in case:

# Build main branch and install
# git branch: main
rm -rf buildMainClean && cmake -B buildMainClean -S . -DCMAKE_INSTALL_PREFIX=/home/srryabin/sdk-install-main -DENABLE_SMOKE_TESTS=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=toolchains/clang-c++20.cmake &> configureMain.log && cmake --build buildMainClean -j 32 --config Debug --target install &> buildMain.log

# Build this branch and install
# git branch: sr/cmakeListClients
rm -rf buildRefactor && cmake -B buildRefactor -S . -DCMAKE_INSTALL_PREFIX=/home/srryabin/sdk-install-refactor -DENABLE_SMOKE_TESTS=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_TOOLCHAIN_FILE=toolchains/clang-c++20.cmake &> configureRefactor.log && cmake --build buildRefactor -j 32 --config Debug --target install &> buildRefactor.log

# Compare the build output, the only difference is the output from the test testing race conditions on the logging (which is pretty okay)
tree buildRefactor > cmakerefac_tree.log
tree buildMainClean > cmakeclean_tree.log
diff cmakeclean_tree.log cmakerefac_tree.log

1c1
< buildMainClean
---
> buildRefactor
24739a24740
>     │   ├── aws_sdk_2025-01-21-18.log
24847c24848,24849
<     │   └── testRaceOnLogFile2025-01-21-17.log
---
>     │   ├── testRaceOnLogFile2025-01-21-17.log
>     │   └── testRaceOnLogFile2025-01-21-18.log
25487c25489
< 4748 directories, 20736 files
---
> 4748 directories, 20738 files

63 smoke tests app are built in both cases.

Smoke test for a legacy renamed service "apigateway" also builds and runs fine on my dev machine:

~/workspace/github/aws-sdk-cpp]
my-super-duper-dev-machine $ ./buildRefactor/generated/smoke-tests/apigateway/apigateway-smoke-tests
[==========] Running 1 test from 1 test suite.
[----------] Global test environment set-up.
[----------] 1 test from APIGatewaySmokeTestSuite
[ RUN      ] APIGatewaySmokeTestSuite.GetDomainNamesSuccess
[       OK ] APIGatewaySmokeTestSuite.GetDomainNamesSuccess (61 ms)
[----------] 1 test from APIGatewaySmokeTestSuite (61 ms total)

[----------] Global test environment tear-down
[==========] 1 test from 1 test suite ran. (96 ms total)
[  PASSED  ] 1 test.

-DBUILD_ONLY logic was also tested.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for verifying.

LIST(REVERSE nameParts)
string(REPLACE ";" "-" tmpSvc "${nameParts}")
LIST(APPEND C2J_SPECIAL_NAME_LIST "${tmpSvc}:${svc}")
set(svc "${tmpSvc}")
elseif("${svc}" STREQUAL "ec2")
if(PLATFORM_ANDROID AND CMAKE_HOST_WIN32) # ec2 isn't building for android on windows atm due to an internal compiler error, TODO: investigate further
continue()
endif()
elseif("${svc}" STREQUAL "s3")
LIST(APPEND C2J_SPECIAL_NAME_LIST "s3-crt:s3")
message(STATUS "Add s3-crt:s3 to C2J_SPECIAL_NAME_LIST")
endif()
get_map_element(${svc} existingDate ${C2J_LIST})
if ("${existingDate}" STREQUAL "")
LIST(APPEND C2J_LIST "${svc}:${date}")
if ("${svc}" STREQUAL "s3")
LIST(APPEND C2J_LIST "s3-crt:${date}")
endif()
elseif(${existingDate} STRLESS ${date})
LIST(REMOVE_ITEM C2J_LIST "${svc}:${existingDate}")
LIST(APPEND C2J_LIST "${svc}:${date}")
if ("${svc}" STREQUAL "s3")
LIST(REMOVE_ITEM C2J_LIST "s3-crt:${existingDate}")
LIST(APPEND C2J_LIST "s3-crt:${date}")
endif()
endif()
set(LIST_DIRECTORIES true)
file(GLOB ALL_GENERATED_CLIENTS $LIST_DIRECTORIES "${CMAKE_CURRENT_SOURCE_DIR}/generated/src/aws-cpp-sdk-*")

foreach(clientDir IN LISTS ALL_GENERATED_CLIENTS)
get_filename_component(clientDirName "${clientDir}" NAME)
STRING(REGEX REPLACE "^aws-cpp-sdk-" "" serviceName "${clientDirName}")
LIST(APPEND SERVICE_CLIENT_LIST "${serviceName}")
endforeach()
set(C2J_LIST "${C2J_LIST}" PARENT_SCOPE)
set(C2J_SPECIAL_NAME_LIST "${C2J_SPECIAL_NAME_LIST}" PARENT_SCOPE)
set(SERVICE_CLIENT_LIST "${SERVICE_CLIENT_LIST}" PARENT_SCOPE)
endfunction()


# A list of so-called "hand-written SDK high-level libraries/components"
set(HIGH_LEVEL_SDK_LIST "")
list(APPEND HIGH_LEVEL_SDK_LIST "access-management")
list(APPEND HIGH_LEVEL_SDK_LIST "identity-management")
Expand All @@ -151,6 +83,7 @@ list(APPEND HIGH_LEVEL_SDK_LIST "transfer")
list(APPEND HIGH_LEVEL_SDK_LIST "s3-encryption")
list(APPEND HIGH_LEVEL_SDK_LIST "text-to-speech")

# A flatmap list of sdk_component:sdk_component_tests,sdk_component_another_tests
set(SDK_TEST_PROJECT_LIST "")
list(APPEND SDK_TEST_PROJECT_LIST "cloudfront:tests/aws-cpp-sdk-cloudfront-integration-tests")
list(APPEND SDK_TEST_PROJECT_LIST "cognito-identity:tests/aws-cpp-sdk-cognitoidentity-integration-tests")
Expand Down Expand Up @@ -184,11 +117,9 @@ build_sdk_list()

if(EXISTS "${CMAKE_SOURCE_DIR}/generated")
if(EXISTS "${CMAKE_SOURCE_DIR}/generated/tests")
foreach(GENERATED_C2J_TEST IN LISTS C2J_LIST)
STRING(REGEX REPLACE "([^:]+):.*" "\\1" GENERATED_C2J_TEST_RESULT ${GENERATED_C2J_TEST})

if(EXISTS "${CMAKE_SOURCE_DIR}/generated/tests/${GENERATED_C2J_TEST_RESULT}-gen-tests")
list(APPEND SDK_TEST_PROJECT_LIST "${GENERATED_C2J_TEST_RESULT}:generated/tests/${GENERATED_C2J_TEST_RESULT}-gen-tests")
foreach(SERVICE_NAME IN LISTS SERVICE_CLIENT_LIST)
if(EXISTS "${CMAKE_SOURCE_DIR}/generated/tests/${SERVICE_NAME}-gen-tests")
list(APPEND SDK_TEST_PROJECT_LIST "${SERVICE_NAME}:generated/tests/${SERVICE_NAME}-gen-tests")
endif()
endforeach()
endif()
Expand All @@ -213,10 +144,7 @@ list(APPEND TEST_DEPENDENCY_LIST "text-to-speech:polly,core")
list(APPEND TEST_DEPENDENCY_LIST "transfer:s3,core")
list(APPEND TEST_DEPENDENCY_LIST "logs:access-management,cognito-identity,iam,core")

# make a list of the generated clients
set(GENERATED_SERVICE_LIST "")
foreach(GENERATED_C2J_SERVICE IN LISTS C2J_LIST)
STRING(REGEX REPLACE "([^:]+):.*" "\\1" SERVICE_RESULT ${GENERATED_C2J_SERVICE})
list(APPEND GENERATED_SERVICE_LIST ${SERVICE_RESULT})
list(APPEND SDK_DEPENDENCY_LIST "${SERVICE_RESULT}:core")
set(GENERATED_SERVICE_LIST ${SERVICE_CLIENT_LIST})
foreach(SERVICE_NAME IN LISTS SERVICE_CLIENT_LIST)
list(APPEND SDK_DEPENDENCY_LIST "${SERVICE_NAME}:core")
endforeach()
Loading