PRMP-639 (#10)

Add terraform and github workflow for new ODS lookup

.github/workflows/PR-terraform-plan.yml

@@ -0,0 +1,28 @@
+name: "PR-terraform-plan"
+on:
+  pull_request:
+    branches:
+      - master
+    paths:
+      - 'stacks/**'
+
+permissions:
+  pull-requests: write
+  id-token: write # This is required for requesting the JWT
+  contents: read # This is required for actions/checkout
+
+jobs:
+  terraform_plan:
+    strategy:
+      fail-fast: false
+      max-parallel: 1
+      matrix:
+        environment: [ dev, prod-plan ]
+        terraform_stack: [container-repositories, base-networking, ecs-cluster]
+    uses: ./.github/workflows/base-terraform-plan-and-apply.yml
+    with:
+      environment: ${{ matrix.environment }}
+      terraform_stack: ${{ matrix.terraform_stack }}
+    secrets: inherit
+
+

.github/workflows/base-gp-registrations-mi.yml

@@ -0,0 +1,171 @@
+name: base-gp-registrations-mi
+on:
+  workflow_call:
+    inputs:
+      environment:
+        description: "Which Environment settings to use."
+        required: true
+        type: string
+        default: "dev"
+      is_deployment:
+        description: "Is workflow run on deployment"
+        type: boolean
+        default: false
+
+jobs:
+  terraform_process:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ vars.AWS_REGION }}
+          role-to-assume:  ${{(inputs.is_deployment && secrets.AWS_ASSUME_ROLE) || secrets.AWS_ASSUME_ROLE_READ_ONLY}}
+          role-skip-session-tagging: true
+
+      - name: Publish Docker Image to Prod
+        id: push-image-to-prod
+        if: inputs.environment == 'prod' && inputs.is_deployment
+
+        run: |
+          aws ecr get-login-password --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin ${{ secrets.AWS_ECR_DEV_REPOSITORY }}
+          IMAGE_TAG=$(aws ecr describe-images --registy-id ${{ secrets.AWS_DEV_ACCOUNT_ID }} --repository-name ${{ secrets.ECR_REPOSITORY_NAME }} --query 'sort_by(imageDetails,& imagePushedAt)[-1].imageTags[0]')
+          source_repo=${{ secrets.AWS_ECR_DEV_REPOSITORY }}/${{ secrets.ECR_REPOSITORY_DEV_NAME }}:${IMAGE_TAG//\"}
+          destination_repo=${{ secrets.AWS_ECR_PROD_REPOSITORY}}/${{ secrets.ECR_REPOSITORY_NAME }}:${IMAGE_TAG//\"}
+          docker pull $source_repo
+          docker tag $source_repo $destination_repo
+          aws ecr get-login-password --region ${{ vars.AWS_REGION }} | docker login --username AWS --password-stdin ${{ secrets.AWS_ECR_PROD_REPOSITORY }}
+          docker push $destination_repo
+          echo "image-tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT"
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: latest
+
+      - name: Terraform Format
+        id: fmt
+        run: terraform fmt -check
+        working-directory: ./stacks/gp-registrations-mi/terraform
+
+      - name: Terraform Init
+        id: init
+        run: | 
+          terraform init -no-color -backend-config="key=${{ secrets.AWS_STATE_S3_KEY }}" \
+          -backend-config="bucket=${{ secrets.AWS_STATE_BUCKET }}" \
+          -backend-config="dynamodb_table=${{ secrets.AWS_STATE_LOCK_TABLE }}"
+        working-directory: ./stacks/gp-registrations-mi/terraform
+        shell: bash
+
+      - name: Terraform Validate
+        id: validate
+        run: terraform validate -no-color
+        working-directory: ./stacks/gp-registrations-mi/terraform
+
+      - name: Build Lambdas
+        run: | 
+            ./tasks_github_actions.sh build-lambdas
+
+      - name: Set up Python
+        if: github.ref == 'refs/heads/master' && inputs.is_deployment
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+
+      - name: Make virtual environment
+        if: github.ref == 'refs/heads/master' && inputs.is_deployment
+        run: |
+            python3 -m venv ./venv
+            ./venv/bin/pip3 install --upgrade pip requests
+
+      - name: Get ODS CSV Files
+        if: github.ref == 'refs/heads/master' && inputs.is_deployment
+        run: |
+          PYTHONPATH=$PYTHONPATH:. ./venv/bin/python3 stacks/gp-registrations-mi/scripts/get_latest_ods_csv.py ${{ secrets.TRUD_API_KEY }} ${{ vars.TRUD_API_URL }}
+
+      - name: Setup Terraform variables
+        id: vars-prod
+        if: inputs.environment == 'prod' && inputs.is_deployment
+        run: |-
+          cat > pipeline.auto.tfvars <<EOF
+          gp_registrations_mi_image_tag = ${{ steps.push-image-to-prod.outputs.image-tag }}
+          environment = "${{ vars.AWS_ENVIRONMENT }}"
+          EOF
+        working-directory: ./stacks/gp-registrations-mi/terraform
+
+      - name: Setup Terraform variables
+        id: vars-dev
+        if: inputs.environment != 'prod'
+        run: |-
+          IMAGE_TAG=$(aws ecr describe-images --repository-name ${{ secrets.ECR_REPOSITORY_NAME }} --query 'sort_by(imageDetails,& imagePushedAt)[-1].imageTags[0]')
+          cat > pipeline.auto.tfvars <<EOF
+          gp_registrations_mi_image_tag = $IMAGE_TAG
+          environment = "${{ vars.AWS_ENVIRONMENT }}"
+          EOF
+        working-directory: ./stacks/gp-registrations-mi/terraform
+
+      - name: Terraform Plan
+        id: plan
+        run: |
+          terraform plan -no-color -input=false -var-file="../vars/${{ vars.AWS_ENVIRONMENT }}.tfvars" -out "${{ vars.AWS_ENVIRONMENT }}.tfplan"
+          terraform show -no-color ${{ vars.AWS_ENVIRONMENT }}.tfplan > ${{ vars.AWS_ENVIRONMENT }}.tfplan.txt
+          echo "summary=$(grep -E 'Plan: [0-9]+ to add, [0-9]+ to change, [0-9]+ to destroy\.|No changes\. Your infrastructure matches the configuration\.' ${{ vars.AWS_ENVIRONMENT }}.tfplan.txt | sed 's/.*No changes\. Your infrastructure matches the configuration/Plan: no changes/g' | sed 's/.*Plan: //g' | sed 's/\..*//g')" >> $GITHUB_OUTPUT
+        working-directory: ./stacks/gp-registrations-mi/terraform
+        shell: bash
+
+      - name: Add PR comment
+        uses: actions/github-script@v7
+        if: github.event_name == 'pull_request' && (success() || failure())
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            // 1. Retrieve existing bot comments for the PR
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+            const botComment = comments.find(comment => {
+              return comment.user.type === 'Bot' && comment.body.includes('Report for gp-registrations-mi environment: ${{ inputs.environment }}')
+            });
+
+            // 2. Prepare format of the comment
+            const output = `### Report for gp-registrations-mi environment: ${{ inputs.environment }}
+
+            #### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\`
+
+
+            #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\`
+
+
+            #### Terraform Validation 🤖\`${{ steps.validate.outcome }}\`
+
+
+            #### Terraform Plan 📖\`${{ steps.plan.outcome }}\`
+
+            Plan results: ${{ steps.plan.outputs.summary }}`;
+
+            // 3. If we have a comment, update it, otherwise create a new one
+            if (botComment) {
+              github.rest.issues.deleteComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: botComment.id,
+              })
+            }
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: output
+            });
+
+      - name: Terraform Apply
+        if: github.ref == 'refs/heads/master' && inputs.is_deployment
+        run: terraform apply -auto-approve -input=false ${{ vars.AWS_ENVIRONMENT }}.tfplan
+        working-directory: ./stacks/gp-registrations-mi/terraform

.github/workflows/base-terraform-plan-and-apply.yml

@@ -0,0 +1,122 @@
+name: base-terraform-plan-and-apply
+on:
+  workflow_call:
+    inputs:
+      environment:
+        description: "Which Environment settings to use"
+        required: true
+        type: string
+        default: "dev"
+      is_deployment:
+        description: "Is workflow run on deployment"
+        type: boolean
+        default: false
+      terraform_stack:
+        description: "Which terraform stack directory to run"
+        type: string
+        required: true
+
+jobs:
+  terraform_process:
+    runs-on: ubuntu-latest
+    environment: ${{ inputs.environment }}
+    defaults:
+      run:
+        working-directory: ./stacks/${{ inputs.terraform_stack }}/terraform
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: ${{ vars.AWS_REGION }}
+          role-to-assume:  ${{inputs.is_deployment && secrets.AWS_ASSUME_ROLE || secrets.AWS_ASSUME_ROLE_READ_ONLY}}
+          role-skip-session-tagging: true
+
+      - name: Setup Terraform
+        uses: hashicorp/setup-terraform@v3
+        with:
+          terraform_version: latest
+
+      - name: Terraform Format
+        id: fmt
+        run: terraform fmt -check
+
+      - name: Terraform Init
+        id: init
+        run: | 
+          terraform init -no-color -backend-config="key=gp-registrations-mi/${{ inputs.terraform_stack }}/terraform.tfstate" \
+          -backend-config="bucket=${{ secrets.AWS_STATE_BUCKET }}" \
+          -backend-config="dynamodb_table=${{ secrets.AWS_STATE_LOCK_TABLE }}"
+        shell: bash
+
+      - name: Terraform Validate
+        id: validate
+        run: terraform validate -no-color
+
+      - name: Setup Terraform variables
+        id: vars
+        run: |-
+          cat > pipeline.auto.tfvars <<EOF
+          environment = "${{ vars.AWS_ENVIRONMENT }}"
+          EOF
+
+      - name: Terraform Plan
+        id: plan
+        run: |
+          terraform plan -no-color -input=false -var-file="../vars/${{ vars.AWS_ENVIRONMENT }}.tfvars" -out "${{ vars.AWS_ENVIRONMENT }}.tfplan"
+          terraform show -no-color ${{ vars.AWS_ENVIRONMENT }}.tfplan > ${{ vars.AWS_ENVIRONMENT }}.tfplan.txt
+          echo "summary=$(grep -E 'Plan: [0-9]+ to add, [0-9]+ to change, [0-9]+ to destroy\.|No changes\. Your infrastructure matches the configuration\.' ${{ vars.AWS_ENVIRONMENT }}.tfplan.txt | sed 's/.*No changes\. Your infrastructure matches the configuration/Plan: no changes/g' | sed 's/.*Plan: //g' | sed 's/\..*//g')" >> $GITHUB_OUTPUT
+        shell: bash
+
+      - name: Add PR comment
+        uses: actions/github-script@v7
+        if: github.event_name == 'pull_request' && (success() || failure())
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          script: |
+            // 1. Retrieve existing bot comments for the PR
+            const { data: comments } = await github.rest.issues.listComments({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: context.issue.number,
+            });
+            const botComment = comments.find(comment => {
+              return comment.user.type === 'Bot' && comment.body.includes('Report for ${{inputs.terraform_stack}} environment: ${{ inputs.environment }}')
+            });
+
+            // 2. Prepare format of the comment
+            const output = `### Report for ${{inputs.terraform_stack}} environment: ${{ inputs.environment }}
+
+            #### Terraform Format and Style 🖌\`${{ steps.fmt.outcome }}\`
+
+
+            #### Terraform Initialization ⚙️\`${{ steps.init.outcome }}\`
+
+
+            #### Terraform Validation 🤖\`${{ steps.validate.outcome }}\`
+
+
+            #### Terraform Plan 📖\`${{ steps.plan.outcome }}\`
+
+            Plan results: ${{ steps.plan.outputs.summary }}`;
+
+            // 3. If we have a comment, update it, otherwise create a new one
+            if (botComment) {
+              github.rest.issues.deleteComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: botComment.id,
+              })
+            }
+            github.rest.issues.createComment({
+              issue_number: context.issue.number,
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: output
+            });
+
+      - name: Terraform Apply
+        if: github.ref == 'refs/heads/master' && inputs.is_deployment
+        run: terraform apply -auto-approve -input=false ${{ vars.AWS_ENVIRONMENT }}.tfplan

.github/workflows/deploy-gp-registrations-mi.yml

@@ -0,0 +1,25 @@
+name: deploy-gp-registrations-mi
+on:
+  workflow_dispatch:
+      inputs:
+        environment:
+          default: "dev"
+          description: "Which environment should this run against"
+          required: true
+          type: choice
+          options:
+            - dev
+            - prod
+
+permissions:
+  pull-requests: write
+  id-token: write # This is required for requesting the JWT
+  contents: read # This is required for actions/checkout
+
+jobs:
+  terraform_process:
+    uses: ./.github/workflows/base-gp-registrations-mi.yml
+    with:
+      environment: ${{ inputs.environment }}
+      is_deployment: true
+    secrets: inherit

.github/workflows/deploy-terraform-by-stack.yml

@@ -0,0 +1,37 @@
+name: deploy-terraform-by-stack
+on:
+  workflow_dispatch:
+      inputs:
+        environment:
+          default: "dev"
+          description: "Which environment should this run against"
+          required: true
+          type: choice
+          options:
+            - dev
+            - prod
+        terraform_stack:
+          description: "Which terraform stack directory to run"
+          type: choice
+          required: true
+          options:
+            - container-repositories
+            - base-networking
+            - ecs-cluster
+
+permissions:
+  pull-requests: write
+  id-token: write # This is required for requesting the JWT
+  contents: read # This is required for actions/checkout
+
+jobs:
+  terraform_process:
+    uses: ./.github/workflows/base-terraform-plan-and-apply.yml
+    with:
+      environment: ${{ inputs.environment }}
+      is_deployment: true
+      terraform_stack: ${{ inputs.terraform_stack }}
+    secrets: inherit
+
+
+