diff --git a/.circleci/config.yml b/.circleci/config.yml index affde6b..6ab8830 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,27 +1,71 @@ --- version: 2.1 + orbs: slack: circleci/slack@4.1.3 + +executors: + pre-commit-executor: + docker: + - image: quay.io/astronomer/ci-pre-commit:2022-08 + environment: + TFENV_AUTO_INSTALL: true + terraform-executor: + docker: + - image: quay.io/astronomer/ci-terraform:2022-08 + environment: + TFENV_AUTO_INSTALL: true + +workflows: + test: + jobs: + - run_pre_commit + terraform_pipeline: + jobs: + - terraform_lint: + context: + - slack + - terraform_apply: + requires: + - terraform_lint + context: + - slack + - gcp-dev + - terraform_destroy: + requires: + - terraform_apply + context: + - slack + - gcp-dev + #- git_tag: + # context: + # - github-repo + # - slack + # requires: + # - terraform_destroy + # filters: + # branches: + # only: + # - master + jobs: run_pre_commit: - docker: - - image: quay.io/astronomer/ci-pre-commit:2021-09 + executor: pre-commit-executor steps: - checkout - - run: pre-commit run --all-files + - pre-commit-commands + terraform_lint: - docker: - - image: quay.io/astronomer/ci-terraform:2021-09 + executor: pre-commit-executor steps: - checkout - - run: pipeline/lint.sh + - pre-commit-commands - slack/notify: event: fail template: basic_fail_1 terraform_apply: - docker: - - image: quay.io/astronomer/ci-terraform:2021-09 + executor: terraform-executor steps: - checkout - run: EXAMPLE=from_scratch pipeline/run_terraform.sh @@ -30,8 +74,7 @@ jobs: template: basic_fail_1 terraform_destroy: - docker: - - image: quay.io/astronomer/ci-terraform:2021-09 + executor: terraform-executor steps: - checkout - run: DESTROY=1 EXAMPLE=from_scratch pipeline/run_terraform.sh @@ -40,8 +83,7 @@ jobs: template: basic_fail_1 git_tag: - docker: - - image: quay.io/astronomer/ci-terraform:2021-09 + executor: terraform-executor steps: - checkout - run: git remote set-url origin "https://astro-astronomer:${GITHUB_TOKEN}@github.com/astronomer/${CIRCLE_PROJECT_REPONAME}.git" @@ -102,34 +144,38 @@ jobs: ] } -workflows: - test: - jobs: - - run_pre_commit - terraform_pipeline: - jobs: - - terraform_lint: - context: - - slack - - terraform_apply: - requires: - - terraform_lint - context: - - slack - - gcp-dev - - terraform_destroy: - requires: - - terraform_apply - context: - - slack - - gcp-dev - #- git_tag: - # context: - # - github-repo - # - slack - # requires: - # - terraform_destroy - # filters: - # branches: - # only: - # - master +commands: + pre-commit-commands: + description: "Execute the pre-commit" + steps: + - run: + name: Create pre-commit-cache-key.txt + command: | + cp .pre-commit-config.yaml /tmp/pre-commit-cache-key.txt + python --version --version | sed 's/^/# /' >> /tmp/pre-commit-cache-key.txt + - restore_cache: + keys: + - pre-commit-cache-{{ checksum "/tmp/pre-commit-cache-key.txt" }} + - terraform-install + - run: + name: Install terraform-docs + command: go install github.com/terraform-docs/terraform-docs@v0.16.0 + - run: + name: Install pre-commit hooks + command: pre-commit install-hooks + - save_cache: + key: pre-commit-cache-{{ checksum "/tmp/pre-commit-cache-key.txt" }} + paths: + - ~/.cache/pre-commit + - run: + name: Run pre-commit + command: | + pre-commit run --all-files --show-diff-on-failure + + terraform-install: + steps: + - run: + name: Install terraform + command: | + tfenv install + tfenv use diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7f8758e..5c2c104 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,37 +1,38 @@ ---- repos: + - repo: https://github.com/antonbabenko/pre-commit-terraform + rev: v1.74.1 + hooks: + - id: terraform_fmt - repo: https://github.com/codespell-project/codespell rev: v2.1.0 hooks: - id: codespell - - repo: https://github.com/antonbabenko/pre-commit-terraform - rev: v1.13.0 - hooks: - - id: terraform_fmt - - id: terraform_validate_no_variables - repo: https://github.com/pre-commit/mirrors-prettier - rev: "v2.4.1" + rev: "v2.6.2" hooks: - id: prettier args: ["--print-width=135"] - - repo: https://github.com/detailyang/pre-commit-shell - rev: 1.0.5 - hooks: - - id: shell-lint - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.0.1 + rev: v4.2.0 hooks: - id: check-case-conflict - id: check-executables-have-shebangs + - id: check-json - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + - id: check-toml + - id: check-xml - id: check-yaml args: ["--allow-multiple-documents"] + - id: debug-statements + - id: detect-private-key - id: end-of-file-fixer - - id: file-contents-sorter - args: ["--ignore-case", "--unique"] - files: "^.gitignore$" - id: mixed-line-ending args: ["--fix=lf"] + - id: no-commit-to-branch + args: ["-b", "master"] + - id: requirements-txt-fixer + - id: sort-simple-yaml - id: trailing-whitespace - repo: https://github.com/astronomer/pre-commit-hooks rev: bd325c947efcba13c03b4f4c93d882f2f83ed6ff diff --git a/.terraform-version b/.terraform-version new file mode 100644 index 0000000..c813fe1 --- /dev/null +++ b/.terraform-version @@ -0,0 +1 @@ +1.2.5 diff --git a/db.tf b/db.tf index 645f4a6..fe23eb3 100644 --- a/db.tf +++ b/db.tf @@ -20,6 +20,8 @@ resource "google_service_account_key" "cloud_sql_admin" { resource "google_sql_database_instance" "instance" { count = var.deploy_db ? 1 : 0 + deletion_protection = var.db_deletion_protection + name = "${var.deployment_id}-astro-db-${random_id.db_name_suffix.hex}" region = local.region database_version = var.db_version diff --git a/domain.tf b/domain.tf index 70534b9..d036dee 100644 --- a/domain.tf +++ b/domain.tf @@ -37,7 +37,6 @@ resource "tls_private_key" "cert_private_key" { resource "tls_cert_request" "req" { count = var.lets_encrypt ? 1 : 0 - key_algorithm = "RSA" private_key_pem = tls_private_key.cert_private_key.0.private_key_pem dns_names = ["*.${local.base_domain}"] diff --git a/examples/from_scratch/main.tf b/examples/from_scratch/main.tf index aa02b26..54ee8c9 100644 --- a/examples/from_scratch/main.tf +++ b/examples/from_scratch/main.tf @@ -17,5 +17,6 @@ module "astronomer_gcp" { management_endpoint = "public" enable_gke_metered_billing = true db_max_connections = 1000 - db_version = "POSTGRES_12" + db_version = "POSTGRES_14" + db_deletion_protection = false } diff --git a/firewalls.tf b/firewalls.tf index c06c20a..438050a 100644 --- a/firewalls.tf +++ b/firewalls.tf @@ -61,3 +61,21 @@ resource "google_compute_firewall" "gke_iap_ssh_to_nodes" { source_ranges = var.iap_cidr_ranges target_tags = local.gke_nodepool_network_tags } + +# Istio Firewall rule +# ref: https://istio.io/latest/docs/setup/platform-setup/gke/ +resource "google_compute_firewall" "istio_firewall_rule" { + + count = var.enable_istio ? 1 : 0 + + name = google_container_cluster.primary.name + network = google_compute_network.core.name + description = "Allow Istio to perform Pilot discovery validation webhook." + + allow { + protocol = "tcp" + ports = ["10250", "443", "15017"] + } + + source_ranges = [google_container_cluster.primary.private_cluster_config.0.master_ipv4_cidr_block] +} diff --git a/iam.tf b/iam.tf index 1ca0f2a..e1820ef 100644 --- a/iam.tf +++ b/iam.tf @@ -1,3 +1,5 @@ +data "google_project" "project" {} + resource "google_service_account_key" "default_key" { service_account_id = google_service_account.k8s_registry.account_id public_key_type = "TYPE_X509_PEM_FILE" @@ -24,6 +26,7 @@ resource "google_project_iam_audit_config" "iap" { } service = "iap.googleapis.com" + project = data.google_project.project.project_id } resource "google_service_account_key" "velero" { @@ -34,6 +37,8 @@ resource "google_project_iam_custom_role" "velero_server" { role_id = "velero.server.${var.deployment_id}" title = "Velero Server" + project = data.google_project.project.project_id + permissions = [ "compute.disks.get", "compute.disks.create", @@ -47,8 +52,9 @@ resource "google_project_iam_custom_role" "velero_server" { } resource "google_project_iam_member" "velero_server" { - member = "serviceAccount:${google_service_account.velero.email}" - role = google_project_iam_custom_role.velero_server.id + member = "serviceAccount:${google_service_account.velero.email}" + role = google_project_iam_custom_role.velero_server.id + project = data.google_project.project.project_id } resource "google_storage_bucket_iam_member" "velero_server" { diff --git a/locals.tf b/locals.tf index 8aa61c9..4b1e1c6 100644 --- a/locals.tf +++ b/locals.tf @@ -20,31 +20,7 @@ locals { cluster_name = google_container_cluster.primary.name endpoint = google_container_cluster.primary.endpoint cluster_ca_certificate = google_container_cluster.primary.master_auth[0].cluster_ca_certificate - # basic auth kubeconfig method will be removed in future release - # we will moving to token based auth method - kubeconfig = < 0 ? "gcloud beta compute ssh --zone ${element(concat(google_compute_instance.bastion.*.zone, list("")), 0)} ${element(concat(google_compute_instance.bastion.*.name, list("")), 0)} --tunnel-through-iap --ssh-flag='-L 1234:127.0.0.1:8888 -C -N'" : "Not applicable - no bastion" + value = length(google_compute_instance.bastion) > 0 ? "gcloud beta compute ssh --zone ${element(concat(tolist(google_compute_instance.bastion.*.zone)), 0)} ${element(concat(tolist(google_compute_instance.bastion.*.name)), 0)} --tunnel-through-iap --ssh-flag='-L 1234:127.0.0.1:8888 -C -N'" : "Not applicable - no bastion" } output "db_connection_string" { - value = var.deploy_db ? "postgres://${element(concat(google_sql_user.airflow.*.name, list("")), 0)}:${local.postgres_airflow_password}@${element(concat(google_sql_database_instance.instance.*.private_ip_address, list("")), 0)}:5432" : "N/A: DB is not deployed with the terraform-google-astronomer-gcp module. Set deploy_db = true" + value = var.deploy_db ? "postgres://${element(concat(tolist(google_sql_user.airflow.*.name)), 0)}:${local.postgres_airflow_password}@${element(concat(tolist(google_sql_database_instance.instance.*.private_ip_address)), 0)}:5432" : "N/A: DB is not deployed with the terraform-google-astronomer-gcp module. Set deploy_db = true" sensitive = true } output "db_connection_user" { - value = var.deploy_db ? element(concat(google_sql_user.airflow.*.name, list("")), 0) : "N/A" + value = var.deploy_db ? element(concat(tolist(google_sql_user.airflow.*.name)), 0) : "N/A" } output "db_connection_password" { @@ -33,11 +33,11 @@ output "db_connection_password" { } output "db_instance_private_ip" { - value = var.deploy_db ? element(concat(google_sql_database_instance.instance.*.private_ip_address, list("")), 0) : "N/A" + value = var.deploy_db ? element(concat(tolist(google_sql_database_instance.instance.*.private_ip_address)), 0) : "N/A" } output "db_instance_name" { - value = var.deploy_db ? element(concat(google_sql_database_instance.instance.*.name, list("")), 0) : "N/A" + value = var.deploy_db ? element(concat(tolist(google_sql_database_instance.instance.*.name)), 0) : "N/A" } output "base_domain" { @@ -57,15 +57,6 @@ EOF sensitive = true } -output "kubeconfig" { - value = local.kubeconfig - sensitive = true -} - -output "kubeconfig_filename" { - value = local_file.kubeconfig.filename -} - output "container_registry_bucket_name" { value = google_storage_bucket.container_registry.name description = "Cloud Storage Bucket Name to be used for Container Registry" diff --git a/pipeline/run_terraform.sh b/pipeline/run_terraform.sh index 6b5f66a..a9b78f0 100755 --- a/pipeline/run_terraform.sh +++ b/pipeline/run_terraform.sh @@ -1,6 +1,6 @@ #!/bin/bash -TERRAFORM="${TERRAFORM:-terraform-0.13.7}" +TERRAFORM="${TERRAFORM:-terraform}" "${TERRAFORM}" -v diff --git a/providers.tf.example b/providers.tf.example index 9df0734..1ce5b1a 100644 --- a/providers.tf.example +++ b/providers.tf.example @@ -2,14 +2,12 @@ provider "google" { region = "us-east4" zone = "us-east4-a" project = "astronomer-cloud-dev-236021" - version = "~> 2.20" } provider "google-beta" { region = "us-east4" zone = "us-east4-a" project = "astronomer-cloud-dev-236021" - version = "~> 2.20" } provider "acme" { diff --git a/variables.tf b/variables.tf index 2b918a1..302d5c1 100644 --- a/variables.tf +++ b/variables.tf @@ -9,7 +9,7 @@ variable "dns_managed_zone" { } variable "kube_version_gke" { - default = "1.21.9-gke.1001" + default = "1.21.12-gke.2200" description = "The kubernetes version to use in GKE" } @@ -120,10 +120,16 @@ variable "deploy_db" { variable "db_version" { type = string # https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/sql_database_instance#argument-reference - default = "POSTGRES_9_6" + default = "POSTGRES_14" description = "define postgres database version" } +variable "db_deletion_protection" { + type = bool + default = true + description = "Turn on and off deletion_protection." +} + variable "db_max_connections" { type = number default = 0 @@ -465,3 +471,8 @@ variable "spotinist_token" { default = "12345" type = string } + +variable "enable_istio" { + default = "false" + type = string +} diff --git a/versions.tf b/versions.tf index be796c5..e3b6105 100644 --- a/versions.tf +++ b/versions.tf @@ -3,8 +3,7 @@ terraform { required_version = ">= 0.13" required_providers { acme = { - source = "vancluever/acme" - version = "~> 2.0" + source = "vancluever/acme" } google = { source = "hashicorp/google" @@ -13,28 +12,22 @@ terraform { source = "hashicorp/google-beta" } http = { - source = "hashicorp/http" - version = "~> 1.1" + source = "hashicorp/http" } local = { - source = "hashicorp/local" - version = "~> 1.3" + source = "hashicorp/local" } null = { - source = "hashicorp/null" - version = "~> 2.1" + source = "hashicorp/null" } random = { - source = "hashicorp/random" - version = "~> 2.2" + source = "hashicorp/random" } spotinst = { - source = "spotinst/spotinst" - version = "~> 1.17" + source = "spotinst/spotinst" } tls = { - source = "hashicorp/tls" - version = "~> 2.1" + source = "hashicorp/tls" } } }