Feature-3533 Adding new Istio support (#64)

* Updating modules. * Disabling kubeconfig. * Disabling kubeconfig * Disabling deletion_protection * Removing extra output. * Removing provider version declaration. * Adding tfenv file. * key_algorithm is not require. * Replacing list function. * Applying fmt * Updating pre-commit * Updating CI pipeline. * Using pre-commit executor. * Removing version config. * Updating cluster version * Adding db_deletion_protection variable. * Updating test case * Removing fix deployment_id. * Adding env TFENV_AUTO_INSTALL for auto install. * Fixing http provider. * Update .circleci/config.yml Co-authored-by: Daniel Hoherd <[email protected]> * Update .circleci/config.yml Co-authored-by: Daniel Hoherd <[email protected]> * Removing kubeconfig configurations. Co-authored-by: Daniel Hoherd <[email protected]>
astronomer · Aug 8, 2022 · 826f200 · 826f200
1 parent 3b03097
commit 826f200
Show file tree

Hide file tree

Showing 17 changed files with 171 additions and 135 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -1,27 +1,71 @@
 ---
 version: 2.1
+
 orbs:
   slack: circleci/[email protected]
+
+executors:
+  pre-commit-executor:
+    docker:
+      - image: quay.io/astronomer/ci-pre-commit:2022-08
+        environment:
+          TFENV_AUTO_INSTALL: true
+  terraform-executor:
+    docker:
+      - image: quay.io/astronomer/ci-terraform:2022-08
+        environment:
+          TFENV_AUTO_INSTALL: true
+
+workflows:
+  test:
+    jobs:
+      - run_pre_commit
+  terraform_pipeline:
+    jobs:
+      - terraform_lint:
+          context:
+            - slack
+      - terraform_apply:
+          requires:
+            - terraform_lint
+          context:
+            - slack
+            - gcp-dev
+      - terraform_destroy:
+          requires:
+            - terraform_apply
+          context:
+            - slack
+            - gcp-dev
+      #- git_tag:
+      #    context:
+      #      - github-repo
+      #      - slack
+      #    requires:
+      #      - terraform_destroy
+      #    filters:
+      #      branches:
+      #        only:
+      #          - master
+
 jobs:
   run_pre_commit:
-    docker:
-      - image: quay.io/astronomer/ci-pre-commit:2021-09
+    executor: pre-commit-executor
     steps:
       - checkout
-      - run: pre-commit run --all-files
+      - pre-commit-commands
+
   terraform_lint:
-    docker:
-      - image: quay.io/astronomer/ci-terraform:2021-09
+    executor: pre-commit-executor
     steps:
       - checkout
-      - run: pipeline/lint.sh
+      - pre-commit-commands
       - slack/notify:
           event: fail
           template: basic_fail_1
 
   terraform_apply:
-    docker:
-      - image: quay.io/astronomer/ci-terraform:2021-09
+    executor: terraform-executor
     steps:
       - checkout
       - run: EXAMPLE=from_scratch pipeline/run_terraform.sh
@@ -30,8 +74,7 @@ jobs:
           template: basic_fail_1
 
   terraform_destroy:
-    docker:
-      - image: quay.io/astronomer/ci-terraform:2021-09
+    executor: terraform-executor
     steps:
       - checkout
       - run: DESTROY=1 EXAMPLE=from_scratch pipeline/run_terraform.sh
@@ -40,8 +83,7 @@ jobs:
           template: basic_fail_1
 
   git_tag:
-    docker:
-      - image: quay.io/astronomer/ci-terraform:2021-09
+    executor: terraform-executor
     steps:
       - checkout
       - run: git remote set-url origin "https://astro-astronomer:${GITHUB_TOKEN}@github.com/astronomer/${CIRCLE_PROJECT_REPONAME}.git"
@@ -102,34 +144,38 @@ jobs:
               ]
             }
 
-workflows:
-  test:
-    jobs:
-      - run_pre_commit
-  terraform_pipeline:
-    jobs:
-      - terraform_lint:
-          context:
-            - slack
-      - terraform_apply:
-          requires:
-            - terraform_lint
-          context:
-            - slack
-            - gcp-dev
-      - terraform_destroy:
-          requires:
-            - terraform_apply
-          context:
-            - slack
-            - gcp-dev
-      #- git_tag:
-      #    context:
-      #      - github-repo
-      #      - slack
-      #    requires:
-      #      - terraform_destroy
-      #    filters:
-      #      branches:
-      #        only:
-      #          - master
+commands:
+  pre-commit-commands:
+    description: "Execute the pre-commit"
+    steps:
+      - run:
+          name: Create pre-commit-cache-key.txt
+          command: |
+            cp .pre-commit-config.yaml /tmp/pre-commit-cache-key.txt
+            python --version --version | sed 's/^/# /' >> /tmp/pre-commit-cache-key.txt
+      - restore_cache:
+          keys:
+            - pre-commit-cache-{{ checksum "/tmp/pre-commit-cache-key.txt" }}
+      - terraform-install
+      - run:
+          name: Install terraform-docs
+          command: go install github.com/terraform-docs/[email protected]
+      - run:
+          name: Install pre-commit hooks
+          command: pre-commit install-hooks
+      - save_cache:
+          key: pre-commit-cache-{{ checksum "/tmp/pre-commit-cache-key.txt" }}
+          paths:
+            - ~/.cache/pre-commit
+      - run:
+          name: Run pre-commit
+          command: |
+            pre-commit run --all-files --show-diff-on-failure
+
+  terraform-install:
+    steps:
+      - run:
+          name: Install terraform
+          command: |
+            tfenv install
+            tfenv use
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,37 +1,38 @@
----
 repos:
+  - repo: https://github.com/antonbabenko/pre-commit-terraform
+    rev: v1.74.1
+    hooks:
+      - id: terraform_fmt
   - repo: https://github.com/codespell-project/codespell
     rev: v2.1.0
     hooks:
       - id: codespell
-  - repo: https://github.com/antonbabenko/pre-commit-terraform
-    rev: v1.13.0
-    hooks:
-      - id: terraform_fmt
-      - id: terraform_validate_no_variables
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: "v2.4.1"
+    rev: "v2.6.2"
     hooks:
       - id: prettier
         args: ["--print-width=135"]
-  - repo: https://github.com/detailyang/pre-commit-shell
-    rev: 1.0.5
-    hooks:
-      - id: shell-lint
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.0.1
+    rev: v4.2.0
     hooks:
       - id: check-case-conflict
       - id: check-executables-have-shebangs
+      - id: check-json
       - id: check-merge-conflict
+      - id: check-shebang-scripts-are-executable
+      - id: check-toml
+      - id: check-xml
       - id: check-yaml
         args: ["--allow-multiple-documents"]
+      - id: debug-statements
+      - id: detect-private-key
       - id: end-of-file-fixer
-      - id: file-contents-sorter
-        args: ["--ignore-case", "--unique"]
-        files: "^.gitignore$"
       - id: mixed-line-ending
         args: ["--fix=lf"]
+      - id: no-commit-to-branch
+        args: ["-b", "master"]
+      - id: requirements-txt-fixer
+      - id: sort-simple-yaml
       - id: trailing-whitespace
   - repo: https://github.com/astronomer/pre-commit-hooks
     rev: bd325c947efcba13c03b4f4c93d882f2f83ed6ff

diff --git a/.terraform-version b/.terraform-version
@@ -0,0 +1 @@
+1.2.5
diff --git a/db.tf b/db.tf
@@ -20,6 +20,8 @@ resource "google_service_account_key" "cloud_sql_admin" {
 resource "google_sql_database_instance" "instance" {
   count = var.deploy_db ? 1 : 0
 
+  deletion_protection = var.db_deletion_protection
+
   name             = "${var.deployment_id}-astro-db-${random_id.db_name_suffix.hex}"
   region           = local.region
   database_version = var.db_version

diff --git a/domain.tf b/domain.tf
@@ -37,7 +37,6 @@ resource "tls_private_key" "cert_private_key" {
 resource "tls_cert_request" "req" {
   count = var.lets_encrypt ? 1 : 0
 
-  key_algorithm   = "RSA"
   private_key_pem = tls_private_key.cert_private_key.0.private_key_pem
   dns_names       = ["*.${local.base_domain}"]
 

diff --git a/examples/from_scratch/main.tf b/examples/from_scratch/main.tf
@@ -17,5 +17,6 @@ module "astronomer_gcp" {
   management_endpoint        = "public"
   enable_gke_metered_billing = true
   db_max_connections         = 1000
-  db_version                 = "POSTGRES_12"
+  db_version                 = "POSTGRES_14"
+  db_deletion_protection     = false
 }
diff --git a/firewalls.tf b/firewalls.tf
@@ -61,3 +61,21 @@ resource "google_compute_firewall" "gke_iap_ssh_to_nodes" {
   source_ranges = var.iap_cidr_ranges
   target_tags   = local.gke_nodepool_network_tags
 }
+
+# Istio Firewall rule
+# ref: https://istio.io/latest/docs/setup/platform-setup/gke/
+resource "google_compute_firewall" "istio_firewall_rule" {
+
+  count = var.enable_istio ? 1 : 0
+
+  name        = google_container_cluster.primary.name
+  network     = google_compute_network.core.name
+  description = "Allow Istio to perform Pilot discovery validation webhook."
+
+  allow {
+    protocol = "tcp"
+    ports    = ["10250", "443", "15017"]
+  }
+
+  source_ranges = [google_container_cluster.primary.private_cluster_config.0.master_ipv4_cidr_block]
+}
diff --git a/iam.tf b/iam.tf
@@ -1,3 +1,5 @@
+data "google_project" "project" {}
+
 resource "google_service_account_key" "default_key" {
   service_account_id = google_service_account.k8s_registry.account_id
   public_key_type    = "TYPE_X509_PEM_FILE"
@@ -24,6 +26,7 @@ resource "google_project_iam_audit_config" "iap" {
   }
 
   service = "iap.googleapis.com"
+  project = data.google_project.project.project_id
 }
 
 resource "google_service_account_key" "velero" {
@@ -34,6 +37,8 @@ resource "google_project_iam_custom_role" "velero_server" {
   role_id = "velero.server.${var.deployment_id}"
   title   = "Velero Server"
 
+  project = data.google_project.project.project_id
+
   permissions = [
     "compute.disks.get",
     "compute.disks.create",
@@ -47,8 +52,9 @@ resource "google_project_iam_custom_role" "velero_server" {
 }
 
 resource "google_project_iam_member" "velero_server" {
-  member = "serviceAccount:${google_service_account.velero.email}"
-  role   = google_project_iam_custom_role.velero_server.id
+  member  = "serviceAccount:${google_service_account.velero.email}"
+  role    = google_project_iam_custom_role.velero_server.id
+  project = data.google_project.project.project_id
 }
 
 resource "google_storage_bucket_iam_member" "velero_server" {

diff --git a/locals.tf b/locals.tf
@@ -20,31 +20,7 @@ locals {
   cluster_name           = google_container_cluster.primary.name
   endpoint               = google_container_cluster.primary.endpoint
   cluster_ca_certificate = google_container_cluster.primary.master_auth[0].cluster_ca_certificate
-  # basic auth kubeconfig method will be removed in future release
-  # we will moving to token based auth method
-  kubeconfig   = <<EOF
-apiVersion: v1
-clusters:
-- cluster:
-    server: https://${google_container_cluster.primary.endpoint}
-    certificate-authority-data: ${google_container_cluster.primary.master_auth[0].cluster_ca_certificate}
-  name: cluster
-contexts:
-- context:
-    cluster: cluster
-    user: admin
-  name: context
-current-context: "context"
-kind: Config
-preferences: {}
-users:
-- name: "${google_container_cluster.primary.master_auth[0].username}"
-  user:
-    password: "${google_container_cluster.primary.master_auth[0].password}"
-    username: "${google_container_cluster.primary.master_auth[0].username}"
-
-  EOF
-  bastion_name = "${var.deployment_id}-bastion"
+  bastion_name           = "${var.deployment_id}-bastion"
   # the second ternary is due to a bug during terraform destroy that the random_string.postgres_airflow_password
   # is an empty array and causes an error.  this just checks and lets it keep going through destroy successfully.
   postgres_airflow_password = (

diff --git a/main.tf b/main.tf
@@ -17,6 +17,8 @@ resource "google_container_cluster" "primary" {
   provider = google-beta
   name     = "${var.deployment_id}-cluster"
 
+  project = data.google_project.project.project_id
+
   # "
   # We can't create a cluster with no node pool defined, but we want to only use
   # separately managed node pools. So we create the smallest possible default
@@ -59,7 +61,6 @@ resource "google_container_cluster" "primary" {
   enable_legacy_abac = false
 
   ip_allocation_policy {
-    use_ip_aliases                = true
     cluster_secondary_range_name  = google_compute_subnetwork.gke.secondary_ip_range[0].range_name
     services_secondary_range_name = google_compute_subnetwork.gke.secondary_ip_range[1].range_name
   }
@@ -74,7 +75,7 @@ resource "google_container_cluster" "primary" {
     cidr_blocks {
       # display_name = google_compute_subnetwork.bastion.name
       # either whitelist the caller's IP or only allow access from bastion
-      cidr_block = var.management_endpoint == "public" ? var.kube_api_whitelist_cidr == "" ? "${trimspace(data.http.local_ip.body)}/32" : var.kube_api_whitelist_cidr : google_compute_subnetwork.bastion[0].ip_cidr_range
+      cidr_block = var.management_endpoint == "public" ? var.kube_api_whitelist_cidr == "" ? "${trimspace(data.http.local_ip.response_body)}/32" : var.kube_api_whitelist_cidr : google_compute_subnetwork.bastion[0].ip_cidr_range
     }
 
   }
@@ -115,9 +116,6 @@ resource "google_container_cluster" "primary" {
   */
 
   master_auth {
-    username = ""
-    password = ""
-
     client_certificate_config {
       issue_client_certificate = false
     }
@@ -143,11 +141,6 @@ resource "random_id" "kubeconfig_suffix" {
   byte_length = 4
 }
 
-resource "local_file" "kubeconfig" {
-  sensitive_content = local.kubeconfig
-  filename          = "./kubeconfig-${random_id.kubeconfig_suffix.hex}"
-}
-
 resource "google_bigquery_dataset" "gke_metered_billing" {
   count                      = var.enable_gke_metered_billing ? 1 : 0
   dataset_id                 = "${var.deployment_id}_gke_usage_metering_dataset"

diff --git a/network.tf b/network.tf
@@ -87,7 +87,7 @@ resource "google_compute_global_address" "private_ip_address" {
   address_type  = "INTERNAL"
   prefix_length = 16
   network       = google_compute_network.core.self_link
-
+  project       = data.google_project.project.project_id
 }
 
 # Required for connecting the bastion subnetwork to the