refactor: move clickhouseproxy to use generic ec2 module (#140)

This diff is the tf split of #119 and limits the terraform changes. We
introduce a new ec2 module and configure the clickhouse proxy instance
to use this module instead of having its own module.

Part of #110
Closes #141

ansible/deploy-clickhouse-proxy.yml

@@ -0,0 +1,13 @@
+---
+- name: Deploy clickhouse proxy
+  hosts:
+    - clickhouseproxy.dev.ooni.io
+  become: true
+  roles:
+    - role: bootstrap
+    - role: nginx
+      tags: nginx
+    - role: clickhouse_proxy
+      vars: 
+        clickhouse_url: "clickhouse3.prod.ooni.io"
+        clickhouse_port: 9000

ansible/group_vars/clickhouse/vars.yml

@@ -7,6 +7,8 @@ nftables_clickhouse_allow:
     ip: 168.119.7.188
   - fqdn: notebook.ooni.org
     ip: 138.201.19.39
+  - fqdn: backend-hel.ooni.org
+    ip: 65.108.192.151
   - fqdn: clickhouseproxy.dev.ooni.io
     ip: "{{ lookup('dig', 'clickhouseproxy.dev.ooni.io/A') }}"
 

ansible/inventory

@@ -31,3 +31,6 @@ ams-ps.ooni.nu
 # currently disabled due to them not supporting ed25519 keys
 #mia-echoth.ooni.nu
 #mia-httpth.ooni.nu
+
+[aws-proxy]
+clickhouseproxy.dev.ooni.io

ansible/roles/bootstrap/tasks/main.yml

@@ -68,6 +68,7 @@
 
 - name: install systemd-resolved
   tags: resolved
+  ignore_errors: yes # some ubuntu hosts do not have the systemd-resolved package
   ansible.builtin.apt:
     install_recommends: no
     cache_valid_time: 86400

ansible/roles/clickhouse_proxy/handlers/main.yml

@@ -0,0 +1,21 @@
+- name: test nginx config
+  command: /usr/sbin/nginx -t -c /etc/nginx/nginx.conf
+  listen:
+    - restart nginx
+    - reload nginx
+
+- name: restart nginx
+  service:
+    name: nginx
+    state: restarted
+
+- name: reload nginx
+  service:
+    name: nginx
+    state: reloaded
+
+- name: reload nftables
+  tags: nftables
+  ansible.builtin.systemd_service:
+    name: nftables
+    state: reloaded

ansible/roles/clickhouse_proxy/tasks/main.yml

@@ -0,0 +1,30 @@
+---
+- name: Allow traffic on port 9000
+  tags: clickhouse-proxy
+  blockinfile:
+    path: /etc/ooni/nftables/tcp/9000.nft
+    create: yes
+    block: |
+      add rule inet filter input tcp dport 9000 counter accept comment "clickhouse"
+  notify:
+    - reload nftables  
+
+- name: Create the modules-enabled directory if not exists
+  tags: webserv
+  ansible.builtin.file:
+    path: /etc/nginx/modules-enabled
+    state: directory
+    mode: 0755
+    owner: root
+    group: root
+
+- name: Add stream nginx config
+  tags: webserv
+  template:
+    src: templates/99-stream.conf
+    dest: /etc/nginx/modules-enabled/99-stream.conf
+    mode: 0755
+    owner: root
+  notify:
+    - reload nginx
+    - restart nginx

ansible/roles/clickhouse_proxy/templates/99-stream.conf

@@ -0,0 +1,13 @@
+stream {
+    upstream clickhouse_backend {
+        server {{ clickhouse_url }}:{{ clickhouse_port }};
+    }
+
+    server {
+        listen 9000;
+
+        proxy_pass clickhouse_backend; 
+    }
+
+    error_log /var/log/nginx/error.log;
+}

tf/environments/dev/main.tf

@@ -407,8 +407,8 @@ module "ooniapi_reverseproxy" {
   )
 }
 
-module "ooni_backendproxy" {
-  source = "../../modules/ooni_backendproxy"
+module "ooni_clickhouse_proxy" {
+  source = "../../modules/ec2"
 
   stage = local.environment
 
@@ -420,19 +420,55 @@ module "ooni_backendproxy" {
   key_name      = module.adm_iam_roles.oonidevops_key_name
   instance_type = "t3a.nano"
 
-  backend_url        = "https://backend-fsn.ooni.org/"
-  wcth_addresses     = module.ooni_th_droplet.droplet_ipv4_address
-  wcth_domain_suffix = "th.ooni.org"
-  clickhouse_url     = "clickhouse1.prod.ooni.io"
-  clickhouse_port    = "9000"
+  name = "oonickprx"
+  ingress_rules = [{
+    from_port = 22,
+    to_port = 22,
+    protocol = "tcp",
+    cidr_blocks = ["0.0.0.0/0"],
+  }, {
+    from_port = 80,
+    to_port = 80,
+    protocol = "tcp",
+    cidr_blocks = ["0.0.0.0/0"],
+  }, {
+    from_port = 9000,
+    to_port = 9000,
+    protocol = "tcp",
+    cidr_blocks = module.network.vpc_subnet_private[*].cidr_block,
+  }]
+
+  egress_rules = [{
+    from_port = 0,
+    to_port = 0,
+    protocol = "-1",
+    cidr_blocks = ["0.0.0.0/0"],
+  }, {
+    from_port = 0,
+    to_port = 0,
+    protocol = "-1",
+    ipv6_cidr_blocks = ["::/0"]
+  }]
+
+  sg_prefix = "oockprx"
+  tg_prefix = "ckpr"
 
   tags = merge(
     local.tags,
-    { Name = "ooni-tier0-backendproxy" }
+    { Name = "ooni-tier0-clickhouseproxy" }
   )
 }
 
+resource "aws_route53_record" "clickhouse_proxy_alias" {
+  zone_id = local.dns_zone_ooni_io
+  name    = "clickhouseproxy.${local.environment}.ooni.io"
+  type    = "CNAME"
+  ttl     = 300
 
+  records = [
+    module.ooni_clickhouse_proxy.aws_instance_public_dns
+  ]
+}
 
 #### OONI Run service
 

tf/modules/ec2/main.tf

@@ -0,0 +1,111 @@
+data "aws_ssm_parameter" "ubuntu_22_ami" {
+  name = "/aws/service/canonical/ubuntu/server/22.04/stable/current/amd64/hvm/ebs-gp2/ami-id"
+}
+
+# Important note about security groups:
+# https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group#recreating-a-security-group
+resource "aws_security_group" "ec2_sg" {
+  description = "security group for ec2"
+  name_prefix = var.sg_prefix
+
+  vpc_id = var.vpc_id
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  tags = var.tags
+}
+
+resource "aws_security_group_rule" "ec2_sg_ingress" {
+  count = length(var.ingress_rules)
+
+  type = "ingress"
+  from_port         = var.ingress_rules[count.index].from_port
+  to_port           = var.ingress_rules[count.index].to_port
+  protocol          = var.ingress_rules[count.index].protocol
+  cidr_blocks       = var.ingress_rules[count.index].cidr_blocks
+  ipv6_cidr_blocks  = var.ingress_rules[count.index].ipv6_cidr_blocks
+  security_group_id =  aws_security_group.ec2_sg.id
+}
+
+resource "aws_security_group_rule" "ec2_sg_egress" {
+  count = length(var.egress_rules)
+
+  type = "egress"
+  from_port         = var.egress_rules[count.index].from_port
+  to_port           = var.egress_rules[count.index].to_port
+  protocol          = var.egress_rules[count.index].protocol
+  cidr_blocks       = var.egress_rules[count.index].cidr_blocks
+  ipv6_cidr_blocks  = var.egress_rules[count.index].ipv6_cidr_blocks
+  security_group_id =  aws_security_group.ec2_sg.id
+}
+
+data "cloudinit_config" "ooni_ec2" {
+  base64_encode = true
+
+  part {
+    filename     = "init.cfg"
+    content_type = "text/cloud-config"
+    content = templatefile("${path.module}/templates/cloud-init.yml", {})
+  }
+
+}
+
+resource "aws_launch_template" "ooni_ec2" {
+  name_prefix   = "${var.name}-tmpl-"
+  image_id      = data.aws_ssm_parameter.ubuntu_22_ami.value
+  instance_type = var.instance_type
+  key_name      = var.key_name
+
+  user_data = data.cloudinit_config.ooni_ec2.rendered
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  network_interfaces {
+    delete_on_termination       = true
+    associate_public_ip_address = true
+    subnet_id                   = var.subnet_id
+    security_groups = [
+      aws_security_group.ec2_sg.id,
+    ]
+  }
+
+  tag_specifications {
+    resource_type = "instance"
+    tags          = var.tags
+  }
+}
+
+resource "aws_instance" "ooni_ec2" {
+  launch_template {
+    id      = aws_launch_template.ooni_ec2.id
+    version = "$Latest"
+  }
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  tags = var.tags
+}
+
+resource "aws_alb_target_group" "ooni_ec2" {
+  name_prefix = "oo${var.tg_prefix}"
+  port        = 80
+  protocol    = "HTTP"
+  vpc_id      = var.vpc_id
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  tags = var.tags
+}
+
+resource "aws_lb_target_group_attachment" "oonibackend_proxy" {
+  target_id        = aws_instance.ooni_ec2.id
+  target_group_arn = aws_alb_target_group.ooni_ec2.arn
+}

tf/modules/ec2/outputs.tf

@@ -0,0 +1,7 @@
+output "aws_instance_id" {
+  value = aws_instance.ooni_ec2.id
+}
+
+output "aws_instance_public_dns" {
+    value = aws_instance.ooni_ec2.public_dns
+}

tf/modules/ec2/variables.tf

@@ -0,0 +1,65 @@
+variable "vpc_id" {
+  description = "the id of the VPC to deploy the instance into"
+}
+
+variable "subnet_id" {
+  description = "the ids of the subnet to deploy the instance into"
+}
+
+variable "private_subnet_cidr" {
+  description = "the cidr block of the private subnet to allow traffic from for the clickhouse proxy"
+}
+
+ variable "tags" {
+  description = "tags to apply to the resources"
+  default     = {}
+  type        = map(string)
+}
+
+variable "key_name" {
+  description = "Name of AWS key pair"
+}
+
+variable "name" {
+  description = "Name of the resources"
+}
+
+variable "instance_type" {
+  default = "t2.micro"
+}
+
+variable "stage" {
+  default = "one of dev, stage, test, prod"
+}
+
+variable "dns_zone_ooni_io" {
+  description = "id of the DNS zone for ooni_io"
+}
+
+variable "sg_prefix" {
+    description = "security group prefix"
+}
+
+variable "ingress_rules" {
+  type = list(object({
+      from_port   = number
+      to_port     = number
+      protocol    = string
+      cidr_blocks = list(string)
+      ipv6_cidr_blocks = optional(list(string))
+    }))
+}
+
+variable "egress_rules" {
+  type = list(object({
+      from_port   = number
+      to_port     = number
+      protocol    = string
+      cidr_blocks = optional(list(string))
+      ipv6_cidr_blocks = optional(list(string))
+    }))
+}
+
+variable "tg_prefix" {
+    description = "target group prefix. Will be prefixed with `oo`, example: bkprx -> oobkprx"
+}

tf/modules/ooni_backendproxy/main.tf

@@ -14,7 +14,6 @@ resource "aws_security_group" "nginx_sg" {
     protocol    = "tcp"
     from_port   = 9000
     to_port     = 9000
-    cidr_blocks = var.private_subnet_cidr
   }
 
   ingress {