Merge branch 'main' into feat/opensearch-extended-support

assets/build_lambda_layer.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# This script builids a lambda layer. Outpits relative path of layer zip.
+# This script builds a lambda layer. Outputs relative path of layer zip.
 export CID_VERSION=$(python3 -c "from cid import _version;print(_version.__version__)")
 rm -rf build
 

assets/publish_lambda_layer.sh

@@ -3,6 +3,9 @@
 
 # First build layer
 layer=$(./assets/build_lambda_layer.sh)
+# Source the version
+export CID_VERSION=$(python3 -c "from cid import _version;print(_version.__version__)")
+
 
 # Then publish on s3
 export AWS_REGION=us-east-1
@@ -23,7 +26,7 @@ aws cloudformation list-stack-instances \
     if [ $? -ne 0 ]; then
       echo "Error: $output"
     else
-      echo "Uploaded successfuly"
+      echo "Uploaded successfully"
     fi
   done
 
@@ -35,6 +38,8 @@ rm -vf ./$layer
 if aws s3 ls "s3://aws-managed-cost-intelligence-dashboards" >/dev/null 2>&1; then
   echo "Updating cid-cfn.yml"
   aws s3 sync ./cfn-templates/ s3://aws-managed-cost-intelligence-dashboards/cfn/
+  # Publish additional copy into respective version folder
+  aws s3 sync ./cfn-templates/ "s3://aws-managed-cost-intelligence-dashboards/cfn/${CID_VERSION}/"
 
   echo "Syncing dashboards"
   aws s3 sync ./dashboards s3://aws-managed-cost-intelligence-dashboards/hub/

cfn-templates/cid-admin-policies.yaml

@@ -439,6 +439,7 @@ Resources:
           Action:
           - quicksight:CreateDataSet
           - quicksight:DeleteDataSet
+          - quicksight:UpdateDataSet
           - quicksight:PassDataSet
           - quicksight:DescribeDataSet
           - quicksight:DescribeDataSetPermissions

cfn-templates/cid-cfn.yml

@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: '2010-09-09'
-Description: Deployment of Cloud Intelligence Dashboards v4.0.5
+Description: Deployment of Cloud Intelligence Dashboards v4.0.8
 Metadata:
   AWS::CloudFormation::Interface:
     ParameterGroups:
@@ -531,11 +531,14 @@ Resources:
     Properties:
       Name: !Sub 'CID${Suffix}'
       Description: !Sub 'Used for CloudIntelligenceDashboards${Suffix}'
+      State: ENABLED
+      RecursiveDeleteOption: true
       WorkGroupConfiguration:
         EnforceWorkGroupConfiguration: true
         ResultConfiguration:
           EncryptionConfiguration:
             EncryptionOption: SSE_S3
+          ExpectedBucketOwner: !Ref AWS::AccountId
           OutputLocation: !If [ NeedAthenaQueryResultsBucket, !Sub 's3://${MyAthenaQueryResultsBucket}/', !Sub 's3://${AthenaQueryResultsBucket}/' ]
 
   #Legacy version. Replaced by CustomResourceFunctionInit but we cannot remove it completely as it was removing workgroup on deletion of the custom resource.
@@ -582,7 +585,7 @@ Resources:
     Properties:
       FunctionName: !Sub "CidCustomResourceFunctionInit-DoNotRun${Suffix}"
       Role: !GetAtt 'InitLambdaExecutionRole.Arn'
-      Description: "Do what CFN cannot: start crawler, delete bucket with objects and delete an non empty workgroup"
+      Description: "Do what CFN cannot: start crawler and delete bucket with objects"
       Runtime: python3.11
       Architectures: [ x86_64 ] #Compatible with arm64 but it is not supported in all regions
       MemorySize: 128
@@ -602,7 +605,6 @@ Resources:
           from cid.utils import set_parameters
 
           BUCKET = os.environ['BUCKET']
-          WORKGROUP = os.environ['WORKGROUP']
           CRAWLER = os.environ['CRAWLER']
           QUICKSIGHT_USER = os.environ['QUICKSIGHT_USER']
           QUICKSIGHT_ROLE = os.environ.get('QUICKSIGHT_ROLE')
@@ -677,7 +679,6 @@ Resources:
 
           def on_delete():
               # Delete bucket (CF cannot delete if they are non-empty)
-              # and delete WorkGroup (CF cannot do that)
               s3 = boto3.resource('s3')
               log = []
 
@@ -699,24 +700,6 @@ Resources:
                   except Exception as exc:
                       log.append(f'ERROR: {BUCKET} Error: {exc}')
 
-              if WORKGROUP:
-                  try:
-                      response = boto3.client('athena').delete_work_group(
-                          WorkGroup=WORKGROUP,
-                          RecursiveDeleteOption=True
-                      )
-                      print(f'DEBUG: WorkGroup {WORKGROUP} deleted. {response}')
-                      log.append(f'INFO: WorkGroup {WORKGROUP} deleted.')
-                  except botocore.exceptions.ClientError as exc:
-                      status = exc.response["ResponseMetadata"]["HTTPStatusCode"]
-                      errcode = exc.response["Error"]["Code"]
-                      if status == 404:
-                          log.append(f'INFO:  WorkGroup {WORKGROUP} - {errcode}')
-                      else:
-                          log.append(f'ERROR: WorkGroup {WORKGROUP} - {errcode}')
-                  except Exception as exc:
-                      log.append(f'ERROR: WorkGroup {WORKGROUP} Error: {exc}')
-
               if QUICKSIGHT_ROLE:
                   try:
                       role_name = QUICKSIGHT_ROLE.split('/')[-1]
@@ -739,7 +722,6 @@ Resources:
       Environment:
         Variables:
           BUCKET: !If [NeedAthenaQueryResultsBucket, !Ref MyAthenaQueryResultsBucket, '']
-          WORKGROUP: !If [NeedAthenaWorkgroup, !Ref MyAthenaWorkGroup, '']
           CRAWLER: !If [NeedCURTable, !Ref MyGlueCURCrawler, '']
           QUICKSIGHT_USER: !Ref QuickSightUser
           QUICKSIGHT_ROLE: !If [ NeedQuickSightDataSourceRole, !Ref QuickSightDataSourceRole, !Ref 'AWS::NoValue' ]
@@ -790,20 +772,6 @@ Resources:
                 - !Ref 'AWS::NoValue'
       PermissionsBoundary: !If [NeedPermissionsBoundary, !Ref PermissionsBoundary, !Ref 'AWS::NoValue']
 
-  InitLambdaExecutionRoleWorkGroupPolicy:
-    Type: AWS::IAM::Policy
-    Condition: NeedAthenaWorkgroup
-    Properties:
-      PolicyName: AthenaWorkGroupDeletion
-      PolicyDocument:
-        Version: 2012-10-17
-        Statement:
-          - Effect: Allow
-            Action: athena:DeleteWorkGroup
-            Resource: !Sub 'arn:${AWS::Partition}:athena:${AWS::Region}:${AWS::AccountId}:workgroup/${MyAthenaWorkGroup}'
-      Roles:
-        - !Ref InitLambdaExecutionRole
-
   InitLambdaExecutionRoleBucketPolicy:
     Type: AWS::IAM::Policy
     Condition: NeedAthenaQueryResultsBucket
@@ -855,8 +823,6 @@ Resources:
           Value: !If [NeedAthenaWorkgroup, !Ref MyAthenaWorkGroup, '']
         - Key: IgnoreConditionalDependsOnDatabase
           Value: !If [NeedCURTable, !Ref MyGlueCURCrawler, '']
-        - Key: IgnoreConditionalDependsOnPolicy1
-          Value: !If [NeedAthenaWorkgroup, !Ref InitLambdaExecutionRoleWorkGroupPolicy, '']
         - Key: IgnoreConditionalDependsOnPolicy2
           Value: !If [NeedAthenaQueryResultsBucket, !Ref InitLambdaExecutionRoleBucketPolicy, '']
         - Key: IgnoreConditionalDependsOnPolicy3
@@ -1795,6 +1761,7 @@ Resources:
           quicksight_user: !Ref QuickSightUser
           account_map_source: 'dummy' #initial
           share_with_account: !Ref ShareDashboard
+          account_map_database_name: !If [NeedDatabase, !Ref CidDatabase, !Ref DatabaseName ]
     Metadata:
       cfn_nag:
         rules_to_suppress:
@@ -1813,7 +1780,7 @@ Resources:
           - LambdaLayerBucketPrefixIsManaged
           - !FindInMap [RegionMap, !Ref 'AWS::Region', BucketName]
           - !Sub '${LambdaLayerBucketPrefix}-${AWS::Region}' # Region added for backward compatibility
-        S3Key: 'cid-resource-lambda-layer/cid-4.0.5.zip' #replace version here if needed
+        S3Key: 'cid-resource-lambda-layer/cid-4.0.8.zip' #replace version here if needed
       CompatibleRuntimes:
         - python3.10
         - python3.11
@@ -1934,7 +1901,3 @@ Outputs:
     Description: Technical Value - CidExecArn
     Value: !GetAtt CidExec.Arn
     Export: { Name: !Sub 'cid${Suffix}-CidExecArn'}
-  CidDatabase:
-    Description: Technical Value - CidDatabase
-    Value: !Ref CidDatabase
-    Export: { Name: !Sub 'cid${Suffix}-CidDatabase'}

cfn-templates/data-exports-aggregation.yaml

@@ -1,5 +1,5 @@
 AWSTemplateFormatVersion: "2010-09-09"
-Description: AWS Billing Data Export Aggregation v0.1.4
+Description: AWS Billing Data Export Aggregation v0.1.5
 Metadata:
 
   AWS::CloudFormation::Interface:
@@ -632,12 +632,10 @@ Resources:
                 Action:
                   - iam:GetRole
                   - iam:CreateServiceLinkedRole
-                  - iam:DeleteServiceLinkedRole
-                  - iam:GetServiceLinkedRoleDeletionStatus
                 Resource: !Sub 'arn:${AWS::Partition}:iam::${AWS::AccountId}:role/aws-service-role/bcm-data-exports.amazonaws.com/AWSServiceRoleForBCMDataExports'
               - Effect: Allow
                 Action:
-                  - cost-optimization-hub:GetPreferences
+                  - cost-optimization-hub:ListEnrollmentStatuses
                 Resource: '*' # Cannot restrict this
 
     Metadata:
@@ -657,57 +655,100 @@ Resources:
       Handler: index.handler
       MemorySize: 128
       Runtime: python3.12
-      Timeout: 15
+      Timeout: 90
       Role: !GetAtt LambdaServiceLinkedRoleExecutionRole.Arn
       Code:
         ZipFile: |
           import json
+          import logging
           import time
-          import boto3
-          import cfnresponse
+
+          import boto3  # type: ignore
+          from botocore.exceptions import ClientError  # type: ignore
+          from botocore.config import Config  # type: ignore
+          import cfnresponse  # type: ignore
+
+          CUSTOM_RESOURCE_PHYSICAL_ID = "CreateServiceLinkedRoleBcmDataExports"
+
+          logger = logging.getLogger()
+          logger.setLevel(logging.INFO)
+
+          config = Config(retries={"max_attempts": 5, "mode": "standard"})
+          iam_client = boto3.client("iam", region_name="us-east-1", config=config)
+          coh_client = boto3.client("cost-optimization-hub", region_name="us-east-1", config=config)  # noqa: E501
+
 
           def handler(event, context):
-              print(json.dumps(event))
-              coh = boto3.client('cost-optimization-hub', region_name='us-east-1')
-              iam = boto3.client('iam')
+              logger.info(f"Event: {json.dumps(event, default=str)}")
+
               try:
-                  if event['RequestType'] in ['Create', 'Update']:
+                  if event["RequestType"] == "Delete":
+                      cfnresponse.send(event, context, cfnresponse.SUCCESS,
+                                      {}, physicalResourceId=CUSTOM_RESOURCE_PHYSICAL_ID)
+                      return
 
-                      print("Make sure CO hub is activated")
-                      try:
-                          coh.get_preferences()
-                      except Exception as e:
-                          if 'AWS account is not enrolled for recommendations' in str(e):
-                              raise Exception('AWS account is not enrolled for recommendations. Please activate Cost Optimization Hub.')
-                          raise
-
-                      print("Creating service linked role")
-                      iam.create_service_linked_role(
-                          AWSServiceName='bcm-data-exports.amazonaws.com',
-                          Description='Service-linked role for bcm-data-exports.amazonaws.com'
-                      )
-
-                      print("Waiting for the role to be created")
-                      for i in range(60):
-                          try:
-                              iam.get_role(RoleName='AWSServiceRoleForBCMDataExports')
-                              print("Role is created")
-                              break
-                          except iam.exceptions.NoSuchEntityException:
-                              time.sleep(1)
-
-                      print("Additional wait to make sure the role is available")
-                      time.sleep(30)
-
-                  cfnresponse.send(event, context, cfnresponse.SUCCESS, {})
+                  account_id = context.invoked_function_arn.split(":")[4]
+                  validate_coh_enrollment(account_id)
+                  create_service_linked_role(
+                      service_name="bcm-data-exports.amazonaws.com",
+                      description="Service-linked role for bcm-data-exports.amazonaws.com"
+                  )
+
+                  cfnresponse.send(event, context, cfnresponse.SUCCESS,
+                                  {}, physicalResourceId=CUSTOM_RESOURCE_PHYSICAL_ID)
 
               except Exception as e:
-                  if 'has been taken in this account' in str(e):
-                      print('the role AWSServiceRoleForBCMDataExports already exist')
-                      cfnresponse.send(event, context, cfnresponse.SUCCESS, {})
+                  logger.exception(e)
+                  cfnresponse.send(event, context, cfnresponse.FAILED,
+                                  {}, physicalResourceId=CUSTOM_RESOURCE_PHYSICAL_ID, reason=str(e))
+
+
+          def validate_coh_enrollment(account_id: str):
+              """
+              Returns `None` if account is enrolled in Cost Optimization Hub; otherwise, raises `AssertionError`
+              """
+              logger.info("Checking if account is enrolled in COH...")
+
+              paginator = coh_client.get_paginator("list_enrollment_statuses")
+              page_iterator = paginator.paginate(accountId=account_id)
+              items = [item
+                      for page in page_iterator
+                      for item in page["items"]
+                      if item["status"] == "Active"]
+
+              if items:
+                  logger.info("Account is enrolled in COH")
+                  return
+
+              raise AssertionError(
+                  "Account is NOT enrolled in COH. Please try again after enabling Cost Optimization Hub."
+              )
+
+
+          def create_service_linked_role(service_name: str, description: str):
+              try:
+                  logger.info(f"Creating a service-linked role for {service_name}...")
+                  role_name = iam_client.create_service_linked_role(
+                      AWSServiceName=service_name,
+                      Description=description
+                  )["Role"]["RoleName"]
+
+                  logger.info(f"Waiting for the service-linked role to be available...")
+                  waiter = iam_client.get_waiter("role_exists")
+                  waiter.wait(
+                      RoleName=role_name,
+                      WaiterConfig={"Delay": 1, "MaxAttempts": 30}
+                  )
+                  time.sleep(10)  # Additional wait time, just in case
+
+                  logger.info(
+                      f"Successfully created a service-linked role for {service_name}: {role_name}")
+              except ClientError as e:
+                  if e.response["Error"]["Code"] == "InvalidInput":
+                      logger.info(
+                          f"Service-linked role for {service_name} already exists")
                   else:
-                      print(e)
-                      cfnresponse.send(event, context, cfnresponse.FAILED, {}, reason=str(e))
+                      raise
     Metadata:
       cfn_nag:
         rules_to_suppress:
@@ -721,6 +762,7 @@ Resources:
     Type: 'AWS::CloudFormation::CustomResource'
     Properties:
       ServiceToken: !GetAtt CreateServiceLinkedRoleFunction.Arn
+      ServiceTimeout: "90"
 
 ###########################################################################
 # Lambda DataExport Creator: used to create DataExport from outside us-east-1 or cn-northwest-1

changes/CHANGELOG-amazon-connect.md

@@ -0,0 +1,10 @@
+# What's new in Amazon Connect Dashboard
+
+## Amazon Connect Dashboard - v1.1.0
+* Removed link to deprecated feedback form
+
+## Amazon Connect Dashboard - v1.0.1
+* Minor bugfixes
+
+## Amazon Connect Dashboard - v1.0.0
+* Initial release

changes/CHANGELOG-cid.md

@@ -1,6 +1,9 @@
 # What's new in the Cost Intelligence Dashboard (CID) 
 
-## CID - 3.3
+## CID - 3.5
+* Removed link to deprecated feedback form
+
+## CID - 3.4
 * All sheets
 	* Added Payer Accounts control
 

changes/CHANGELOG-cod.md

@@ -1,5 +1,8 @@
 # What's new in the Compute Optimizer Dashboard (COD)
 
+## Compute Optimizer Dashboard - v3.1.0
+* Removed link to deprecated feedback form
+
 ## Compute Optimizer Dashboard - v3.0.0
 * Added RDS Compute Optimizer sheets
 

changes/CHANGELOG-kpi.md

@@ -1,5 +1,8 @@
 # What's new in the KPI Dashboard
 
+## KPI - 2.1.0
+* Removed link to deprecated feedback form
+
 ## KPI - 2.0.0
 > [!IMPORTANT] 
 > Update to this version requires cid-cmd v0.2.23. Please update cid-cmd first before updating the dashboard. During the update QuickSight datasets and Athena views will be updated, please make a copy if you've made any customizations. To update run these commands in your CloudShell (recommended) or other terminal:

changes/CHANGELOG-support-cases-radar.md

@@ -0,0 +1,4 @@
+# What's new in Support Cases Radar
+
+## v1.0.0
+* Initial Release

cid/_version.py

@@ -1,2 +1,2 @@
-__version__ = '4.0.5'
+__version__ = '4.0.8'