diff --git a/deployment/aws-terraform/1-services/efs-csi.tf b/deployment/aws-terraform/1-services/efs-csi.tf new file mode 100644 index 0000000..f3e7844 --- /dev/null +++ b/deployment/aws-terraform/1-services/efs-csi.tf @@ -0,0 +1,31 @@ +resource "helm_release" "efs_csi_driver" { + namespace = "kube-system" + + name = "aws-efs-csi-driver" + repository = "https://kubernetes-sigs.github.io/aws-efs-csi-driver/" + chart = "aws-efs-csi-driver" + + set { + name = "controller.serviceAccount.name" + value = "efs-csi-controller-sa" + } + + set { + name = "node.serviceAccount.name" + value = "efs-csi-node-sa" + } + + set { + name = "image.repository" + value = "602401143452.dkr.ecr.${var.aws_region}.amazonaws.com/eks/aws-efs-csi-driver" + } +} + +resource "kubernetes_storage_class_v1" "efs_sc" { + metadata { + name = "efs-sc" + } + storage_provisioner = "efs.csi.aws.com" + + depends_on = [ helm_release.efs_csi_driver ] +} diff --git a/deployment/aws-terraform/1-services/irsa.tf b/deployment/aws-terraform/1-services/irsa.tf index 9f232d2..c7f7fd8 100644 --- a/deployment/aws-terraform/1-services/irsa.tf +++ b/deployment/aws-terraform/1-services/irsa.tf @@ -1,5 +1,6 @@ -# These EBS-CSI plugin configs are here because they require the Kubernetes TF -# plugin, which needs to be configured with information from the 0-hardware stage +# The EBS CSI plugin IRSA configs are here, and not in 0—hardware where the EBS +# CSI plugin was installed, because they require the Kubernetes TF provider, +# which needs to be configured with outputs from the 0-hardware stage module "ebs_csi_irsa" { source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" @@ -27,3 +28,63 @@ resource "kubernetes_annotations" "ebs_csi_iam_annotation" { "eks.amazonaws.com/role-arn": module.ebs_csi_irsa.iam_role_arn } } + +module "efs_csi_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + + role_name_prefix = "efs-csi-${local.cluster_name}" + attach_efs_csi_policy = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = [ + "kube-system:efs-csi-controller-sa" + ] + } + } + + tags = local.tags +} + +module "efs_csi_irsa_node" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + + role_name_prefix = "efs-csi-node-${local.cluster_name}" + attach_efs_csi_policy = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = [ + "kube-system:efs-csi-node-sa" + ] + } + } + + tags = local.tags +} + +resource "kubernetes_annotations" "efs_csi_iam_annotation" { + api_version = "v1" + kind = "ServiceAccount" + metadata { + name = "efs-csi-controller-sa" + namespace = "kube-system" + } + annotations = { + "eks.amazonaws.com/role-arn": module.efs_csi_irsa.iam_role_arn + } +} + +resource "kubernetes_annotations" "efs_csi_node_annotation" { + api_version = "v1" + kind = "ServiceAccount" + metadata { + name = "efs-csi-node-sa" + namespace = "kube-system" + } + annotations = { + "eks.amazonaws.com/role-arn": module.efs_csi_irsa_node.iam_role_arn + } +} diff --git a/deployment/aws-terraform/1-services/network.tf b/deployment/aws-terraform/1-services/network.tf new file mode 100644 index 0000000..5e1242e --- /dev/null +++ b/deployment/aws-terraform/1-services/network.tf @@ -0,0 +1,19 @@ +data "aws_vpc" "cluster_vpc" { + id = module.eks.vpc_id +} + +resource "aws_security_group" "efs" { + name = "EFS inbound" + description = "EFS inbound traffic" + vpc_id = module.eks.vpc_id + + ingress { + description = "NFS traffic" + from_port = 2049 + to_port = 2049 + protocol = "tcp" + cidr_blocks = [data.aws_vpc.cluster_vpc.cidr_block] + } + + tags = local.tags +} diff --git a/deployment/aws-terraform/application/noaa/cluster.tf b/deployment/aws-terraform/application/noaa/cluster.tf new file mode 100644 index 0000000..50f2db6 --- /dev/null +++ b/deployment/aws-terraform/application/noaa/cluster.tf @@ -0,0 +1,4 @@ +module "eks" { + source = "../../../../modules/aws/cluster" + cluster_name = local.cluster_name +} diff --git a/deployment/aws-terraform/application/noaa/config.tf b/deployment/aws-terraform/application/noaa/config.tf new file mode 100644 index 0000000..8379a6e --- /dev/null +++ b/deployment/aws-terraform/application/noaa/config.tf @@ -0,0 +1,44 @@ +terraform { + required_version = ">= 1.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.18.0" + } + } +} + + +provider "aws" {} + +terraform { + backend "s3" { + region = var.aws_region + encrypt = "true" + } +} + +provider "kubernetes" { + host = module.eks.endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.id] + } +} + +provider "helm" { + kubernetes { + host = module.eks.endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + exec { + api_version = "client.authentication.k8s.io/v1beta1" + args = ["eks", "get-token", "--cluster-name", local.cluster_name] + command = "aws" + } + } +} diff --git a/deployment/aws-terraform/application/noaa/efs.tf b/deployment/aws-terraform/application/noaa/efs.tf new file mode 100644 index 0000000..b72bbe6 --- /dev/null +++ b/deployment/aws-terraform/application/noaa/efs.tf @@ -0,0 +1,18 @@ +data "aws_vpc" "cluster_vpc" { + id = module.eks.vpc_id +} + +resource "aws_efs_file_system" "noaa" { + creation_token = "noaa-hydro-data" + + lifecycle_policy { + transition_to_ia = "AFTER_30_DAYS" + } +} + +resource "aws_efs_mount_target" "noaa" { + for_each = toset( module.eks.vpc_private_subnet_ids ) + file_system_id = aws_efs_file_system.noaa.id + subnet_id = each.key + security_groups = [ module.eks.cluster_security_group ] +} diff --git a/deployment/aws-terraform/application/noaa/locals.tf b/deployment/aws-terraform/application/noaa/locals.tf new file mode 100644 index 0000000..6d05d03 --- /dev/null +++ b/deployment/aws-terraform/application/noaa/locals.tf @@ -0,0 +1,10 @@ +locals { + cluster_name = "${var.project_prefix}-${var.environment}" + + tags = { + Name = var.project_prefix + Environment = var.environment + GithubRepo = "kubernetes" + GithubOrg = "azavea" + } +} diff --git a/deployment/aws-terraform/application/noaa/variables.tf b/deployment/aws-terraform/application/noaa/variables.tf new file mode 100644 index 0000000..29e4471 --- /dev/null +++ b/deployment/aws-terraform/application/noaa/variables.tf @@ -0,0 +1,14 @@ +variable "aws_region" { + type=string + description="The AWS region to deploy into. This will be set by wrapper scripts from the active profile, avoid setting in the .tfvars file" +} + +variable "environment" { + type=string + description="Name of target environment (e.g., production, staging, QA, etc.). This will be set by wrapper scripts from the active profile, avoid setting in the .tfvars file" +} + +variable "project_prefix" { + type=string + description="The project name prefix used to identify cluster resources. This will be set by wrapper scripts; avoid setting in the .tfvars file!" +} diff --git a/deployment/aws-terraform/application/noaa/volumes.tf b/deployment/aws-terraform/application/noaa/volumes.tf new file mode 100644 index 0000000..9898445 --- /dev/null +++ b/deployment/aws-terraform/application/noaa/volumes.tf @@ -0,0 +1,35 @@ +resource "kubernetes_persistent_volume_v1" "noaa" { + metadata { + name = "noaa-hydro-data" + } + spec { + access_modes = ["ReadWriteOnce", "ReadOnlyMany"] + storage_class_name = "efs-sc" + capacity = { + storage = "512Gi" + } + persistent_volume_source { + csi { + driver = "efs.csi.aws.com" + volume_handle = aws_efs_file_system.noaa.id + } + } + } +} + +resource "kubernetes_persistent_volume_claim_v1" "noaa" { + metadata { + name = "noaa-hydro-data" + namespace = "daskhub" + } + spec { + access_modes = ["ReadWriteOnce", "ReadOnlyMany"] + storage_class_name = "efs-sc" + resources { + requests = { + storage = "512Gi" + } + } + volume_name = kubernetes_persistent_volume_v1.noaa.metadata.0.name + } +} diff --git a/deployment/aws-terraform/application/template/cluster.tf b/deployment/aws-terraform/application/template/cluster.tf new file mode 100644 index 0000000..50f2db6 --- /dev/null +++ b/deployment/aws-terraform/application/template/cluster.tf @@ -0,0 +1,4 @@ +module "eks" { + source = "../../../../modules/aws/cluster" + cluster_name = local.cluster_name +} diff --git a/deployment/aws-terraform/application/template/config.tf b/deployment/aws-terraform/application/template/config.tf new file mode 100644 index 0000000..8379a6e --- /dev/null +++ b/deployment/aws-terraform/application/template/config.tf @@ -0,0 +1,44 @@ +terraform { + required_version = ">= 1.0.0" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 4.18.0" + } + } +} + + +provider "aws" {} + +terraform { + backend "s3" { + region = var.aws_region + encrypt = "true" + } +} + +provider "kubernetes" { + host = module.eks.endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + # This requires the awscli to be installed locally where Terraform is executed + args = ["eks", "get-token", "--cluster-name", module.eks.id] + } +} + +provider "helm" { + kubernetes { + host = module.eks.endpoint + cluster_ca_certificate = base64decode(module.eks.cluster_certificate_authority_data) + exec { + api_version = "client.authentication.k8s.io/v1beta1" + args = ["eks", "get-token", "--cluster-name", local.cluster_name] + command = "aws" + } + } +} diff --git a/deployment/aws-terraform/application/template/locals.tf b/deployment/aws-terraform/application/template/locals.tf new file mode 100644 index 0000000..6d05d03 --- /dev/null +++ b/deployment/aws-terraform/application/template/locals.tf @@ -0,0 +1,10 @@ +locals { + cluster_name = "${var.project_prefix}-${var.environment}" + + tags = { + Name = var.project_prefix + Environment = var.environment + GithubRepo = "kubernetes" + GithubOrg = "azavea" + } +} diff --git a/deployment/aws-terraform/application/template/variables.tf b/deployment/aws-terraform/application/template/variables.tf new file mode 100644 index 0000000..29e4471 --- /dev/null +++ b/deployment/aws-terraform/application/template/variables.tf @@ -0,0 +1,14 @@ +variable "aws_region" { + type=string + description="The AWS region to deploy into. This will be set by wrapper scripts from the active profile, avoid setting in the .tfvars file" +} + +variable "environment" { + type=string + description="Name of target environment (e.g., production, staging, QA, etc.). This will be set by wrapper scripts from the active profile, avoid setting in the .tfvars file" +} + +variable "project_prefix" { + type=string + description="The project name prefix used to identify cluster resources. This will be set by wrapper scripts; avoid setting in the .tfvars file!" +} diff --git a/modules/aws/infrastructure/eks.tf b/modules/aws/infrastructure/eks.tf index 6ee6ca0..8a0938e 100644 --- a/modules/aws/infrastructure/eks.tf +++ b/modules/aws/infrastructure/eks.tf @@ -1,5 +1,6 @@ module "eks" { - source = "terraform-aws-modules/eks/aws" + source = "terraform-aws-modules/eks/aws" + version = "18.31.2" cluster_name = local.cluster_name cluster_version = var.cluster_version