start Datadog migration

ServerlessOpsIO · Sep 1, 2024 · 6e45c84 · 6e45c84
1 parent 2296a13
commit 6e45c84
Show file tree

Hide file tree

Showing 4 changed files with 211 additions and 61 deletions.
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -66,19 +66,28 @@ jobs:
               $_f;
           done
 
-      - name: Build artifact
-        run: sam build --parallel --template template.yaml
-
       - name: Upload SAM artifact
-        id: upload-sam
+        id: upload-sam-template
         shell: bash
         run: |
           sam package \
+            --template template.yaml \
             --s3-bucket aws-sam-cli-sourcebucket-346402060170-us-east-1  \
             --s3-prefix ${{ env.GITHUB_REPOSITORY_OWNER_PART_SLUG_URL }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG_URL }}/${{ env.GITHUB_REF_SLUG_URL }} \
             --region us-east-1 \
             --output-template-file packaged-template.yaml
 
+      - name: Upload SAM artifact
+        id: upload-sam-stackset-template
+        shell: bash
+        run: |
+          sam package \
+            --template stacksets.yaml \
+            --s3-bucket aws-sam-cli-sourcebucket-346402060170-us-east-1  \
+            --s3-prefix ${{ env.GITHUB_REPOSITORY_OWNER_PART_SLUG_URL }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG_URL }}/${{ env.GITHUB_REF_SLUG_URL }} \
+            --region us-east-1 \
+            --output-template-file packaged-stacksets.yaml
+
       - name: 'Upload pipeline artifact'
         id: upload-artifact
         uses: actions/upload-artifact@v4
@@ -93,7 +102,8 @@ jobs:
           if-no-files-found: error
           retention-days: 30
 
-  deploy:
+  deploy_management:
+    if: false
     needs:
       - build
     runs-on: ubuntu-latest
@@ -205,4 +215,118 @@ jobs:
             --role-arn arn:aws:iam::349603509961:role/CfnExecIamRole \
             --on-failure DELETE \
             $AWS_CFN_PARAMS \
+            $AWS_CFN_TAGS
+
+  deploy_stackset:
+    needs:
+      - build
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write
+      contents: read
+
+    steps:
+      - name: Set extra GitHub environment variables
+        id: github-env-vars
+        uses: rlespinasse/github-slug-action@v4
+
+      - name: Download artifact
+        id: download-artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: ${{ env.GITHUB_REPOSITORY_SLUG }}-${{ env.GITHUB_REF_SLUG_URL }}-${{ github.run_number }}-${{ github.sha }}
+
+      - name: Setup Python
+        id: install-python
+        uses: actions/setup-python@v3
+        with:
+          python-version: 3.12
+          cache: pipenv
+
+      - name: Install pipenv
+        id: install-pipenv
+        shell: bash
+        run: python -m pip install --upgrade pipenv
+
+
+      - name: Install dependencies
+        id: install-python-deps
+        shell: bash
+        run: pipenv install --dev
+
+      - name: Assume build account AWS credentials
+        id: build-account-credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::346402060170:role/GitHubActionsBuildRole
+          role-session-name: ${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG_URL }}-${{ github.run_number }}-${{ github.job }}
+          role-duration-seconds: 3600   # 60 minutes; needs to be less than our current max duration
+          aws-region: us-east-1
+
+      - name: Assume deploy account AWS credentials
+        id: deploy-account-credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: arn:aws:iam::481665128499:role/GitHubActionsCfnDeployRole
+          role-session-name: ${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG_URL }}-${{ github.run_number }}-${{ github.job }}
+          role-duration-seconds: 3600   # 60 minutes; needs to be less than our current max duration
+          aws-region: us-east-1
+          role-chaining: true
+
+      - name: Install AWS SAM
+        uses: aws-actions/setup-sam@v2
+
+      - name: Process AWS Tags
+        id: processaws-tags
+        shell: bash
+        run: >-
+          AWS_CFN_TAGS=$(
+            cat cfn-tags.json |
+            jq -r '
+              to_entries |
+              map("\(.key)=\(.value|tostring)") |
+              . += [
+                "org:branch=${{ env.GITHUB_EVENT_REF_SLUG_URL }}",
+                "org:stackName=${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG_URL }}-${{ env.GITHUB_EVENT_REF_SLUG_URL }}"
+              ] |
+              join(" ")
+            '
+          ) &&
+          echo "AWS_CFN_TAGS=--tags ${AWS_CFN_TAGS}">> $GITHUB_ENV
+
+      - name: Process AWS Cloudformation Parameters
+        id: process-aws-parameters
+        shell: bash
+        env:
+          GHA_SECRETS: ${{ tojson(secrets) }}
+          GHA_ENV: ${{ toJson(env) }}
+        run: >-
+         AWS_CFN_PARAMS=$(
+            jq \
+              -n \
+              --argjson secrets "$GHA_SECRETS" \
+              --argjson env "$GHA_ENV" \
+              -f cfn-parameters.json |
+            jq \
+              -r '
+                to_entries |
+                map("\(.key)=\(.value|tostring)") |
+                join(" ")
+              '
+          ) &&
+          echo "AWS_CFN_PARAMS=--parameter-overrides ${AWS_CFN_PARAMS}" >> $GITHUB_ENV
+
+      - name: Deploy via SAM
+        id: deploy-sam
+        shell: bash
+        run: |
+          sam deploy \
+            --template-file packaged-stacksets.yaml \
+            --stack-name ${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG_URL }}-${{ env.GITHUB_EVENT_REF_SLUG_URL }} \
+            --s3-bucket aws-sam-cli-sourcebucket-346402060170-us-east-1 \
+            --s3-prefix ${{ env.GITHUB_REPOSITORY_OWNER_PART_SLUG_URL }}/${{ env.GITHUB_REPOSITORY_NAME_PART_SLUG_URL }}/${{ env.GITHUB_REF_SLUG_URL }} \
+            --region us-east-1 \
+            --role-arn arn:aws:iam::481665128499:role/CfnExecIamRole \
+            --on-failure DELETE \
+            $AWS_CFN_PARAMS \
             $AWS_CFN_TAGS
diff --git a/cfn-parameters.json b/cfn-parameters.json
@@ -2,6 +2,7 @@
     "AwsOrgId": $secrets.AWS_ORG_ID,
     "AwsOrgRootId": $secrets.AWS_ORG_ROOT_ID,
     "AwsOamSinkArn": $secrets.AWS_OAM_SINK_ARN,
+    "ManagementOu": $secrets.AWS_ORG_MANAGEMENT_OU,
     "ObservabilityOu": $secrets.AWS_ORG_OBSERVABILITY_OU,
     "TargetRegions": "us-east-1",
     "DatadogApiKey": $secrets.DATADOG_API_KEY,

diff --git a/delegate-template.yaml b/delegate-template.yaml
@@ -0,0 +1,75 @@
+AWSTemplateFormatVersion: '2010-09-09'
+Transform: AWS::Serverless-2016-10-31
+Description: AWS account observability stack
+
+Parameters:
+  AwsOrgId:
+    Type: String
+  AwsOrgRootId:
+    Type: String
+  TargetRegions:
+    Type: String
+  ManagementOu:
+    Type: String
+  ObservabilityOu:
+    Type: String
+  DatadogSite:
+    Type: String
+  DatadogApiKey:
+    Type: String
+    NoEcho: 'true'
+  DatadogAppKey:
+    Type: String
+    NoEcho: 'true'
+  DataDogLogsDestinationArn:
+    Type: String
+    NoEcho: 'true'
+  AwsOamSinkArn:
+    Type: String
+
+Resources:
+  DatadogShippingStackSet:
+    Type: AWS::Serverless::Application
+    Properties:
+      Location: "./stacksets/datadog-shipping/stackset.yaml"
+      Parameters:
+        AwsOrgId: !Ref AwsOrgId
+        ObservabilityOu: !Ref ObservabilityOu
+        TargetRegions: !Ref TargetRegions
+        DatadogSite: !Ref DatadogSite
+        DatadogApiKey: !Ref DatadogApiKey
+
+#  DatadogIntegrationStackSet:
+#    Type: AWS::Serverless::Application
+#    Properties:
+#      Location: "./stacksets/datadog-integration/stackset.yaml"
+#      Parameters:
+#        DeploymentOrgId: !Ref AwsOrgRootId
+#        TargetRegions: !Ref TargetRegions
+#        DatadogApiKey: !Ref DatadogApiKey
+#        DatadogAppKey: !Ref DatadogAppKey
+#        DatadogSite: !Ref DatadogSite
+#        DisableMetricCollection: 'true'
+
+#  LogShippingStackSet:
+#    Type: AWS::Serverless::Application
+#    DependsOn:
+#      - DatadogShippingStackSet
+#    Properties:
+#      Location: "./stacksets/logging/stackset.yaml"
+#      Parameters:
+#        DestinationArn: !Ref DataDogLogsDestinationArn
+#        DeploymentOrgId: !Ref AwsOrgRootId
+#        TargetRegions: !Ref TargetRegions
+
+#  CwCrossAccountShareStackSet:
+#    Type: AWS::Serverless::Application
+#    Properties:
+#      Location: "./stacksets/metrics/stackset.yaml"
+#      Parameters:
+#        DeploymentOrgId: !Ref AwsOrgRootId
+#        TargetRegions: !Ref TargetRegions
+#        AwsOamSinkArn: !Ref AwsOamSinkArn
+#        CrossAccountSharingPolicy: View-Access-for-all-services
+#        MonitoringAccountId: !Select [ "4", !Split [ ":", !Ref AwsOamSinkArn ] ]
+#        ObservabilityOuPath: !Sub "${AwsOrgId}/${AwsOrgRootId}/${ManagementOu}/${ObservabilityOu}"
diff --git a/template.yaml b/template.yaml
@@ -7,7 +7,7 @@ Parameters:
     Type: String
   AwsOrgRootId:
     Type: String
-  TargetRegions:
+  ManagementOu:
     Type: String
   ObservabilityOu:
     Type: String
@@ -26,19 +26,7 @@ Parameters:
     Type: String
 
 Resources:
-  DatadogIntegrationStackSet:
-    Type: AWS::Serverless::Application
-    Properties:
-      Location: "./stacksets/datadog-integration/stackset.yaml"
-      Parameters:
-        DeploymentOrgId: !Ref AwsOrgRootId
-        TargetRegions: !Ref TargetRegions
-        DatadogApiKey: !Ref DatadogApiKey
-        DatadogAppKey: !Ref DatadogAppKey
-        DatadogSite: !Ref DatadogSite
-        DisableMetricCollection: 'true'
-
-  DatadogIntegrationManagementStack:
+  DatadogIntegrationStack:
     Type: AWS::Serverless::Application
     Properties:
       Location: "https://datadog-cloudformation-template.s3.amazonaws.com/aws/main_organizations.yaml"
@@ -48,63 +36,25 @@ Resources:
         DatadogSite: !Ref DatadogSite
         DisableMetricCollection: 'true'
 
-  DatadogShippingStackSet:
-    Type: AWS::Serverless::Application
-    Properties:
-      Location: "./stacksets/datadog-shipping/stackset.yaml"
-      Parameters:
-        AwsOrgId: !Ref AwsOrgId
-        ObservabilityOu: !Ref ObservabilityOu
-        TargetRegions: !Ref TargetRegions
-        DatadogSite: !Ref DatadogSite
-        DatadogApiKey: !Ref DatadogApiKey
-
-  LogShippingStackSet:
-    Type: AWS::Serverless::Application
-    DependsOn:
-      - DatadogShippingStackSet
-    Properties:
-      Location: "./stacksets/logging/stackset.yaml"
-      Parameters:
-        DestinationArn: !Ref DataDogLogsDestinationArn
-        DeploymentOrgId: !Ref AwsOrgRootId
-        TargetRegions: !Ref TargetRegions
-
-  LogShippingManagementStack:
+  LogShippingStack:
     Type: AWS::Serverless::Application
-    DependsOn:
-      - DatadogShippingStackSet
     Properties:
       Location: "./stacksets/logging/template.yaml"
       Parameters:
         DestinationArn: !Ref DataDogLogsDestinationArn
 
-  MetricsSinkManagementStack:
+  MetricsSinkStack:
     Type: AWS::Serverless::Application
-    DependsOn:
-      - DatadogShippingStackSet
     Properties:
       Location: "./stacksets/metrics/oam-link-template.yaml"
       Parameters:
         AwsOamSinkArn: !Ref AwsOamSinkArn
 
-  CwCrossAccountShareStackSet:
-    Type: AWS::Serverless::Application
-    Properties:
-      Location: "./stacksets/metrics/stackset.yaml"
-      Parameters:
-        DeploymentOrgId: !Ref AwsOrgRootId
-        TargetRegions: !Ref TargetRegions
-        AwsOamSinkArn: !Ref AwsOamSinkArn
-        CrossAccountSharingPolicy: View-Access-for-all-services
-        MonitoringAccountId: !Select [ "4", !Split [ ":", !Ref AwsOamSinkArn ] ]
-        ObservabilityOuPath: !Sub "${AwsOrgId}/${AwsOrgRootId}/${ObservabilityOu}"
-
-  CwCrossAccountShareManagementStack:
+  CwCrossAccountShareStack:
     Type: AWS::Serverless::Application
     Properties:
       Location: "./stacksets/metrics/cw-cross-account-sharing-template.yaml"
       Parameters:
-        ObservabilityOuPath: !Sub "${AwsOrgId}/${AwsOrgRootId}/${ObservabilityOu}"
+        ObservabilityOuPath: !Sub "${AwsOrgId}/${AwsOrgRootId}/${ManagementOu}/${ObservabilityOu}"
         Policy: View-Access-for-all-services
         MonitoringAccountId: !Select [ "4", !Split [ ":", !Ref AwsOamSinkArn ] ]